From a9beee6939371db3e484d4f3b44dee0e6ab052d9 Mon Sep 17 00:00:00 2001 From: Mahesh Raju Somalaraju Date: Tue, 15 Apr 2025 14:52:39 +0530 Subject: [PATCH] TEZ-4624: Replace findbugs plugin with spotbugs --- BUILDING.txt | 4 +- Jenkinsfile | 7 +-- build-tools/docker/Dockerfile | 13 +++-- build-tools/test-patch.sh | 68 +++++++++++------------ pom.xml | 100 ++++++++++++++-------------------- tez-api/pom.xml | 4 -- tez-dag/pom.xml | 5 -- tez-runtime-internals/pom.xml | 4 -- 8 files changed, 87 insertions(+), 118 deletions(-) diff --git a/BUILDING.txt b/BUILDING.txt index 57c9cd13c0..733dcb09cf 100644 --- a/BUILDING.txt +++ b/BUILDING.txt @@ -8,7 +8,7 @@ Requirements: * JDK 1.8+ * Maven 3.6.3 or later -* Findbugs 2.0.2 or later (if running findbugs) +* spotbugs 4.9.3 or later (if running spotbugs) * ProtocolBuffer 3.21.1 * Internet connection for first build (to fetch all dependencies) * Hadoop version should be 2.7.0 or higher. @@ -36,7 +36,7 @@ Maven build goals: * Compile : mvn compile * Run tests : mvn test * Create JAR : mvn package - * Run findbugs : mvn compile findbugs:findbugs + * Run spotbugs : mvn compile spotbugs:spotbugs * Run checkstyle : mvn compile checkstyle:checkstyle * Install JAR in M2 cache : mvn install * Deploy JAR to Maven repo : mvn deploy diff --git a/Jenkinsfile b/Jenkinsfile index 4ce236598d..9d2ec4a465 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -117,11 +117,10 @@ pipeline { YETUS_ARGS+=("--proclimit=5500") YETUS_ARGS+=("--dockermemlimit=20g") - # -1 findbugs issues that show up prior to the patch being applied - # YETUS_ARGS+=("--findbugs-strict-precheck") - + # -1 spotbugs issues that show up prior to the patch being applied + # YETUS_ARGS+=("--spotbugs-strict-precheck") # rsync these files back into the archive dir - YETUS_ARGS+=("--archive-list=checkstyle-errors.xml,findbugsXml.xml") + YETUS_ARGS+=("--archive-list=checkstyle-errors.xml,spotbugsXml.xml") # URL for user-side presentation in reports and such to our artifacts # (needs to match the archive bits below) diff --git a/build-tools/docker/Dockerfile b/build-tools/docker/Dockerfile index 5b681067ae..cf49e01b3a 100644 --- a/build-tools/docker/Dockerfile +++ b/build-tools/docker/Dockerfile @@ -190,13 +190,14 @@ RUN java -version 2>&1 | grep "1.8.0" && rm -f /usr/lib/jvm/default-java \ ENV JAVA_HOME /usr/lib/jvm/java-8-openjdk-amd64 ###### -# Install findbugs +# Install SpotBugs 4.2.2 ###### -# hadolint ignore=DL3008 -RUN apt-get -q update && apt-get -q install --no-install-recommends -y findbugs \ - && apt-get clean \ - && rm -rf /var/lib/apt/lists/* -ENV FINDBUGS_HOME /usr +RUN mkdir -p /opt/spotbugs \ + && curl -L -s -S https://github.com/spotbugs/spotbugs/releases/download/4.2.2/spotbugs-4.2.2.tgz \ + -o /opt/spotbugs.tgz \ + && tar xzf /opt/spotbugs.tgz --strip-components 1 -C /opt/spotbugs \ + && chmod +x /opt/spotbugs/bin/* +ENV SPOTBUGS_HOME /opt/spotbugs ###### # Install maven diff --git a/build-tools/test-patch.sh b/build-tools/test-patch.sh index 9b457af221..feb9644893 100755 --- a/build-tools/test-patch.sh +++ b/build-tools/test-patch.sh @@ -40,7 +40,7 @@ DIFF=${DIFF:-diff} JIRACLI=${JIRA:-jira} SED=${SED:-sed} CURL=${CURL:-curl} -FINDBUGS_HOME=${FINDBUGS_HOME} +SPOTBUGS_HOME=${SPOTBUGS_HOME} ############################################################################### printUsage() { @@ -60,7 +60,7 @@ printUsage() { echo "--grep-cmd= The 'grep' command to use (default 'grep')" echo "--patch-cmd= The 'patch' command to use (default 'patch')" echo "--diff-cmd= The 'diff' command to use (default 'diff')" - echo "--findbugs-home= Findbugs home directory (default FINDBUGS_HOME environment variable)" + echo "--spotbugs-home= Spotbugs home directory (default SPOTBUGS_HOME environment variable)" echo "--dirty-workspace Allow the local git workspace to have uncommitted changes" echo "--run-tests Run all tests below the base directory" echo @@ -115,8 +115,8 @@ parseArgs() { --jira-password=*) JIRA_PASSWD=${i#*=} ;; - --findbugs-home=*) - FINDBUGS_HOME=${i#*=} + --spotbugs-home=*) + SPOTBUGS_HOME=${i#*=} ;; --dirty-workspace) DIRTY_WORKSPACE=true @@ -507,7 +507,7 @@ $JIRA_COMMENT_FOOTER" ############################################################################### -### Install the new jars so tests and findbugs can find all of the updated jars +### Install the new jars so tests and spotbugs can find all of the updated jars buildAndInstall () { echo "" echo "" @@ -525,67 +525,67 @@ buildAndInstall () { ############################################################################### -### Check there are no changes in the number of Findbugs warnings -checkFindbugsWarnings () { +### Check there are no changes in the number of Spotbugs warnings +checkSpotbugsWarnings () { echo "" echo "" echo "======================================================================" echo "======================================================================" - echo " Determining number of patched Findbugs warnings." + echo " Determining number of patched Spotbugs warnings." echo "======================================================================" echo "======================================================================" echo "" echo "" rc=0 - echo " Running findbugs " - echo "$MVN clean test findbugs:findbugs -DskipTests < /dev/null > $PATCH_DIR/patchFindBugsOutput.txt 2>&1" - $MVN clean test findbugs:findbugs -DskipTests < /dev/null > $PATCH_DIR/patchFindBugsOutput.txt 2>&1 + echo " Running spotbugs " + echo "$MVN clean test spotbugs:spotbugs -DskipTests < /dev/null > $PATCH_DIR/patchSpotBugsOutput.txt 2>&1" + $MVN clean test spotbugs:spotbugs -DskipTests < /dev/null > $PATCH_DIR/patchSpotBugsOutput.txt 2>&1 rc=$? - findbugs_version=$(${AWK} 'match($0, /findbugs-maven-plugin:[^:]*:findbugs/) { print substr($0, RSTART + 22, RLENGTH - 31); exit }' "${PATCH_DIR}/patchFindBugsOutput.txt") + spotbugs_version=$(${AWK} 'match($0, /spotbugs-maven-plugin:[^:]*:spotbugs/) { print substr($0, RSTART + 22, RLENGTH - 31); exit }' "${PATCH_DIR}/patchSpotBugsOutput.txt") if [ $rc != 0 ] ; then JIRA_COMMENT="$JIRA_COMMENT - {color:red}-1 findbugs{color}. The patch appears to cause Findbugs (version ${findbugs_version}) to fail." + {color:red}-1 spotbugs{color}. The patch appears to cause Spotbugs (version ${spotbugs_version}) to fail." return 1 fi - findbugsWarnings=0 - for file in $(find $BASEDIR -name findbugsXml.xml) + spotbugsWarnings=0 + for file in $(find $BASEDIR -name spotbugsXml.xml) do relative_file=${file#$BASEDIR/} # strip leading $BASEDIR prefix - if [ ! $relative_file == "target/findbugsXml.xml" ]; then - module_suffix=${relative_file%/target/findbugsXml.xml} # strip trailing path + if [ ! $relative_file == "target/spotbugsXml.xml" ]; then + module_suffix=${relative_file%/target/spotbugsXml.xml} # strip trailing path module_suffix=`basename ${module_suffix}` fi - cp $file $PATCH_DIR/patchFindbugsWarnings${module_suffix}.xml - $FINDBUGS_HOME/bin/setBugDatabaseInfo -timestamp "01/01/2000" \ - $PATCH_DIR/patchFindbugsWarnings${module_suffix}.xml \ - $PATCH_DIR/patchFindbugsWarnings${module_suffix}.xml - newFindbugsWarnings=`$FINDBUGS_HOME/bin/filterBugs -first "01/01/2000" $PATCH_DIR/patchFindbugsWarnings${module_suffix}.xml \ - $PATCH_DIR/newPatchFindbugsWarnings${module_suffix}.xml | $AWK '{print $1}'` - echo "Found $newFindbugsWarnings Findbugs warnings ($file)" - findbugsWarnings=$((findbugsWarnings+newFindbugsWarnings)) - $FINDBUGS_HOME/bin/convertXmlToText -html \ - $PATCH_DIR/newPatchFindbugsWarnings${module_suffix}.xml \ - $PATCH_DIR/newPatchFindbugsWarnings${module_suffix}.html - if [[ $newFindbugsWarnings > 0 ]] ; then - JIRA_COMMENT_FOOTER="Findbugs warnings: $BUILD_URL/artifact/patchprocess/newPatchFindbugsWarnings${module_suffix}.html + cp $file $PATCH_DIR/patchSpotbugsWarnings${module_suffix}.xml + $SPOTBUGS_HOME/bin/setBugDatabaseInfo -timestamp "01/01/2000" \ + $PATCH_DIR/patchSpotbugsWarnings${module_suffix}.xml \ + $PATCH_DIR/patchSpotbugsWarnings${module_suffix}.xml + newSpotbugsWarnings=`$SPOTBUGS_HOME/bin/filterBugs -first "01/01/2000" $PATCH_DIR/patchSpotbugsWarnings${module_suffix}.xml \ + $PATCH_DIR/newPatchSpotbugsWarnings${module_suffix}.xml | $AWK '{print $1}'` + echo "Found $newSpotbugsWarnings Spotbugs warnings ($file)" + spotbugsWarnings=$((spotbugsWarnings+newSpotbugsWarnings)) + $SPOTBUGS_HOME/bin/convertXmlToText -html \ + $PATCH_DIR/newPatchSpotbugsWarnings${module_suffix}.xml \ + $PATCH_DIR/newPatchSpotbugsWarnings${module_suffix}.html + if [[ $newSpotbugsWarnings > 0 ]] ; then + JIRA_COMMENT_FOOTER="Spotbugs warnings: $BUILD_URL/artifact/patchprocess/newPatchSpotbugsWarnings${module_suffix}.html $JIRA_COMMENT_FOOTER" fi done - if [[ $findbugsWarnings -gt 0 ]] ; then + if [[ $spotbugsWarnings -gt 0 ]] ; then JIRA_COMMENT="$JIRA_COMMENT - {color:red}-1 findbugs{color}. The patch appears to introduce $findbugsWarnings new Findbugs (version ${findbugs_version}) warnings." + {color:red}-1 spotbugs{color}. The patch appears to introduce $spotbugsWarnings new Spotbugs (version ${spotbugs_version}) warnings." return 1 fi JIRA_COMMENT="$JIRA_COMMENT - {color:green}+1 findbugs{color}. The patch does not introduce any new Findbugs (version ${findbugs_version}) warnings." + {color:green}+1 spotbugs{color}. The patch does not introduce any new Spotbugs (version ${spotbugs_version}) warnings." return 0 } @@ -805,7 +805,7 @@ fi checkJavadocWarnings (( RESULT = RESULT + $? )) buildAndInstall -checkFindbugsWarnings +checkSpotbugsWarnings (( RESULT = RESULT + $? )) checkReleaseAuditWarnings (( RESULT = RESULT + $? )) diff --git a/pom.xml b/pom.xml index 0ab151d9d0..49312a899c 100644 --- a/pom.xml +++ b/pom.xml @@ -70,7 +70,8 @@ 2.6 ${user.home}/clover.license 3.2.0 - 3.0.5 + 4.9.3 + 4.9.3.0 1.8.0 32.0.1-jre 3.4.1 @@ -770,11 +771,6 @@ jettison ${jettison.version} - - com.google.code.findbugs - jsr305 - ${jsr305.version} - com.sun.jersey jersey-client @@ -970,9 +966,20 @@ ${dependency-check-maven.version} - org.codehaus.mojo - findbugs-maven-plugin - ${findbugs-maven-plugin.version} + com.github.spotbugs + spotbugs-maven-plugin + ${spotbugs-maven-plugin.version} + + + com.github.spotbugs + spotbugs + ${spotbugs.version} + + + + ${basedir}/findbugs-exclude.xml + false + org.codehaus.mojo @@ -1119,11 +1126,8 @@ build-helper-maven-plugin - org.codehaus.mojo - findbugs-maven-plugin - - ${basedir}/findbugs-exclude.xml - + com.github.spotbugs + spotbugs-maven-plugin + target/site + ${basedir}/findbugs-exclude.xml + + + + + diff --git a/tez-api/pom.xml b/tez-api/pom.xml index 4e6c27e18f..08eaa80549 100644 --- a/tez-api/pom.xml +++ b/tez-api/pom.xml @@ -85,10 +85,6 @@ junit junit - - com.google.code.findbugs - jsr305 - com.sun.jersey jersey-client diff --git a/tez-dag/pom.xml b/tez-dag/pom.xml index 779f7ed82f..93ab9c51b6 100644 --- a/tez-dag/pom.xml +++ b/tez-dag/pom.xml @@ -111,11 +111,6 @@ commons-cli commons-cli - - com.google.code.findbugs - jsr305 - compile - org.apache.commons commons-collections4 diff --git a/tez-runtime-internals/pom.xml b/tez-runtime-internals/pom.xml index e235e933cd..d95990c6d4 100644 --- a/tez-runtime-internals/pom.xml +++ b/tez-runtime-internals/pom.xml @@ -86,10 +86,6 @@ junit junit - - com.google.code.findbugs - jsr305 - org.mockito mockito-core