HDDS-3710. Merge archived jacoco coverage results

.github/coverage-report.sh

@@ -0,0 +1,56 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+#This script can merge the downloaded jacoco files from other artifacts
+#Works only in github environment where the jacoco .exec files are downloaded
+#from job artifacts
+DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)"
+cd "$DIR/.." || exit 1
+
+set -ex
+
+REPORT_DIR="$DIR/../target/coverage"
+
+mkdir -p "$REPORT_DIR"
+
+#Install jacoco cli
+mvn org.apache.maven.plugins:maven-dependency-plugin:3.1.2:copy -Dartifact=org.jacoco:org.jacoco.cli:0.8.5:jar:nodeps
+
+jacoco() {
+  java -jar target/dependency/org.jacoco.cli-0.8.5-nodeps.jar "$@"
+}
+
+#Merge all the jacoco.exec files
+jacoco merge $(find target/artifacts -name jacoco-combined.exec) --destfile "$REPORT_DIR/jacoco-all.exec"
+
+rm -rf target/coverage-classes || true
+mkdir -p target/coverage-classes
+
+#Unzip all the classes from the last build
+find hadoop-ozone/dist/target/*/share/ozone/lib -name "hadoop-*.jar" | \
+    grep -E 'hadoop-ozone-|hadoop-hdds-' | \
+    grep -v -E 'legacy|current|tests' | \
+    xargs -n1 unzip -o -q -d target/coverage-classes
+
+#Exclude some classes from the coverage
+find target/coverage-classes -name proto -type d | xargs rm -rf
+find target/coverage-classes -name generated -type d | xargs rm -rf
+find target/coverage-classes -name v1 -type d | xargs rm -rf
+find target/coverage-classes -name freon -type d | xargs rm -rf
+find target/coverage-classes -name genesis -type d | xargs rm -rf
+
+#generate the reports
+jacoco report "$REPORT_DIR/jacoco-all.exec" --classfiles target/coverage-classes --html "$REPORT_DIR/all" --xml "$REPORT_DIR/all.xml"

.github/workflows/post-commit.yml

@@ -58,13 +58,6 @@ jobs:
         - uses: ./.github/buildenv
           with:
              args: ./hadoop-ozone/dev-support/checks/unit.sh
-        - uses: ./.github/buildenv
-          if: github.repository == 'apache/hadoop-ozone'
-          with:
-            args: ./hadoop-ozone/dev-support/checks/sonar.sh
-          env:
-            SONAR_TOKEN: ${{ secrets.SONARCLOUD_TOKEN }}
-            GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         - uses: actions/upload-artifact@master
           if: always()
           with:
@@ -226,4 +219,35 @@ jobs:
           if: always()
           with:
             name: it-ozone
-            path: mnt/ozone/target/integration
+            path: mnt/ozone/target/integration
+  coverage:
+    name: coverage
+    runs-on: ubuntu-18.04
+    needs:
+       - it-ozone
+       - it-hdds-om
+       - it-client
+       - it-filesystem-contract
+       - it-filesystem
+       - it-freon
+       - unit
+    steps:
+       - uses: actions/checkout@v2
+       - uses: ./.github/buildenv
+         with:
+            args: ./hadoop-ozone/dev-support/checks/build.sh
+       - uses: actions/download-artifact@v2
+         with:
+            path: target/artifacts
+       - run: ./.github/coverage-report.sh
+       - uses: ./.github/buildenv
+         if: github.repository == 'apache/hadoop-ozone'
+         with:
+           args: ./hadoop-ozone/dev-support/checks/sonar.sh
+         env:
+           SONAR_TOKEN: ${{ secrets.SONARCLOUD_TOKEN }}
+           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+       - uses: actions/upload-artifact@master
+         with:
+           name: coverage
+           path: target/coverage

hadoop-ozone/dev-support/checks/sonar.sh

@@ -13,11 +13,21 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+set -x
 DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)"
-cd "$DIR/../../.." || exit 1
+PROJECT_DIR="$(realpath "$DIR"/../../..)"
+cd "$PROJECT_DIR" || exit 1
 
 if [ ! "$SONAR_TOKEN" ]; then
   echo "SONAR_TOKEN environment variable should be set"
   exit 1
 fi
+
+#Workaround: Sonar expects per-project Sonar XML report, but we have one, combined. Sonar seems to handle it well.
+# Only the classes from the current project will be used. We can copy the same, combined report to all the subprojects.
+if [ -f "$PROJECT_DIR/target/coverage/all.xml" ]; then
+   find "$PROJECT_DIR" -name pom.xml | grep -v target | xargs dirname | xargs -n1 -IDIR mkdir -p DIR/target/coverage/
+   find "$PROJECT_DIR" -name pom.xml | grep -v target | xargs dirname | xargs -n1 -IDIR cp "$PROJECT_DIR/target/coverage/all.xml" DIR/target/coverage/
+fi
+
 mvn -B verify -DskipShade -DskipTests -Dskip.yarn org.sonarsource.scanner.maven:sonar-maven-plugin:3.6.0.1398:sonar -Dsonar.host.url=https://sonarcloud.io -Dsonar.organization=apache -Dsonar.projectKey=hadoop-ozone