TEZ-4386: Upgrade from Findbugs to Spotbugs

BUILDING.txt

@@ -8,7 +8,7 @@ Requirements:
 
 * JDK 1.8+
 * Maven 3.1 or later
-* Findbugs 2.0.2 or later (if running findbugs)
+* Spotbugs 4.2.2 or later (if running spotbugs)
 * ProtocolBuffer 2.5.0
 * Internet connection for first build (to fetch all dependencies)
 * Hadoop version should be 2.7.0 or higher.
@@ -36,7 +36,7 @@ Maven build goals:
  * Compile                   : mvn compile
  * Run tests                 : mvn test
  * Create JAR                : mvn package
- * Run findbugs              : mvn compile findbugs:findbugs
+ * Run spotbugs              : mvn compile spotbugs:spotbugs
  * Run checkstyle            : mvn compile checkstyle:checkstyle
  * Install JAR in M2 cache   : mvn install
  * Deploy JAR to Maven repo  : mvn deploy

Jenkinsfile

@@ -117,11 +117,11 @@ pipeline {
                         YETUS_ARGS+=("--proclimit=5500")
                         YETUS_ARGS+=("--dockermemlimit=20g")
 
-                        # -1 findbugs issues that show up prior to the patch being applied
-                        # YETUS_ARGS+=("--findbugs-strict-precheck")
+                        # -1 spotbugs issues that show up prior to the patch being applied
+                        # YETUS_ARGS+=("--spotbugs-strict-precheck")
 
                         # rsync these files back into the archive dir
-                        YETUS_ARGS+=("--archive-list=checkstyle-errors.xml,findbugsXml.xml")
+                        YETUS_ARGS+=("--archive-list=checkstyle-errors.xml,spotbugsXml.xml")
 
                         # URL for user-side presentation in reports and such to our artifacts
                         # (needs to match the archive bits below)

build-tools/docker/Dockerfile

@@ -186,13 +186,14 @@ RUN apt-get -q update && apt-get -q install --no-install-recommends -y default-j
 ENV JAVA_HOME /usr/lib/jvm/java-8-openjdk-amd64
 
 ######
-# Install findbugs
+# Install SpotBugs 4.2.2
 ######
-# hadolint ignore=DL3008
-RUN apt-get -q update && apt-get -q install --no-install-recommends -y findbugs \
-    && apt-get clean \
-    && rm -rf /var/lib/apt/lists/*
-ENV FINDBUGS_HOME /usr
+RUN mkdir -p /opt/spotbugs \
+    && curl -L -s -S https://github.com/spotbugs/spotbugs/releases/download/4.2.2/spotbugs-4.2.2.tgz \
+      -o /opt/spotbugs.tgz \
+    && tar xzf /opt/spotbugs.tgz --strip-components 1 -C /opt/spotbugs \
+    && chmod +x /opt/spotbugs/bin/*
+ENV SPOTBUGS_HOME /opt/spotbugs
 
 ######
 # Install maven

build-tools/test-patch.sh

@@ -40,7 +40,7 @@ DIFF=${DIFF:-diff}
 JIRACLI=${JIRA:-jira}
 SED=${SED:-sed}
 CURL=${CURL:-curl}
-FINDBUGS_HOME=${FINDBUGS_HOME}
+SPOTBUGS_HOME=${SPOTBUGS_HOME}
 
 ###############################################################################
 printUsage() {
@@ -60,7 +60,7 @@ printUsage() {
   echo "--grep-cmd=<cmd>       The 'grep' command to use (default 'grep')"
   echo "--patch-cmd=<cmd>      The 'patch' command to use (default 'patch')"
   echo "--diff-cmd=<cmd>       The 'diff' command to use (default 'diff')"
-  echo "--findbugs-home=<path> Findbugs home directory (default FINDBUGS_HOME environment variable)"
+  echo "--spotbugs-home=<path> Spotbugs home directory (default SPOTBUGS_HOME environment variable)"
   echo "--dirty-workspace      Allow the local git workspace to have uncommitted changes"
   echo "--run-tests            Run all tests below the base directory"
   echo
@@ -115,8 +115,8 @@ parseArgs() {
     --jira-password=*)
       JIRA_PASSWD=${i#*=}
       ;;
-    --findbugs-home=*)
-      FINDBUGS_HOME=${i#*=}
+    --spotbugs-home=*)
+      SPOTBUGS_HOME=${i#*=}
       ;;
     --dirty-workspace)
       DIRTY_WORKSPACE=true
@@ -507,7 +507,7 @@ $JIRA_COMMENT_FOOTER"
 
 
 ###############################################################################
-### Install the new jars so tests and findbugs can find all of the updated jars
+### Install the new jars so tests and spotbugs can find all of the updated jars
 buildAndInstall () {
   echo ""
   echo ""
@@ -525,67 +525,67 @@ buildAndInstall () {
 
 
 ###############################################################################
-### Check there are no changes in the number of Findbugs warnings
-checkFindbugsWarnings () {
+### Check there are no changes in the number of Spotbugs warnings
+checkSpotbugsWarnings () {
   echo ""
   echo ""
   echo "======================================================================"
   echo "======================================================================"
-  echo "    Determining number of patched Findbugs warnings."
+  echo "    Determining number of patched Spotbugs warnings."
   echo "======================================================================"
   echo "======================================================================"
   echo ""
   echo ""
 
   rc=0
-  echo " Running findbugs "
-  echo "$MVN clean test findbugs:findbugs -DskipTests < /dev/null > $PATCH_DIR/patchFindBugsOutput.txt 2>&1"
-  $MVN clean test findbugs:findbugs -DskipTests < /dev/null > $PATCH_DIR/patchFindBugsOutput.txt 2>&1
+  echo " Running spotbugs "
+  echo "$MVN clean test spotbugs:spotbugs -DskipTests < /dev/null > $PATCH_DIR/patchSpotBugsOutput.txt 2>&1"
+  $MVN clean test spotbugs:spotbugs -DskipTests < /dev/null > $PATCH_DIR/patchSpotBugsOutput.txt 2>&1
   rc=$?
-  findbugs_version=$(${AWK} 'match($0, /findbugs-maven-plugin:[^:]*:findbugs/) { print substr($0, RSTART + 22, RLENGTH - 31); exit }' "${PATCH_DIR}/patchFindBugsOutput.txt")
+  spotbugs_version=$(${AWK} 'match($0, /spotbugs-maven-plugin:[^:]*:spotbugs/) { print substr($0, RSTART + 22, RLENGTH - 31); exit }' "${PATCH_DIR}/patchSpotBugsOutput.txt")
 
   if [ $rc != 0 ] ; then
     JIRA_COMMENT="$JIRA_COMMENT
 
-    {color:red}-1 findbugs{color}.  The patch appears to cause Findbugs (version ${findbugs_version}) to fail."
+    {color:red}-1 spotbugs{color}.  The patch appears to cause Spotbugs (version ${spotbugs_version}) to fail."
     return 1
   fi
 
-  findbugsWarnings=0
-  for file in $(find $BASEDIR -name findbugsXml.xml)
+  spotbugsWarnings=0
+  for file in $(find $BASEDIR -name spotbugsXml.xml)
   do
     relative_file=${file#$BASEDIR/} # strip leading $BASEDIR prefix
-    if [ ! $relative_file == "target/findbugsXml.xml" ]; then
-      module_suffix=${relative_file%/target/findbugsXml.xml} # strip trailing path
+    if [ ! $relative_file == "target/spotbugsXml.xml" ]; then
+      module_suffix=${relative_file%/target/spotbugsXml.xml} # strip trailing path
       module_suffix=`basename ${module_suffix}`
     fi
 
-    cp $file $PATCH_DIR/patchFindbugsWarnings${module_suffix}.xml
-    $FINDBUGS_HOME/bin/setBugDatabaseInfo -timestamp "01/01/2000" \
-      $PATCH_DIR/patchFindbugsWarnings${module_suffix}.xml \
-      $PATCH_DIR/patchFindbugsWarnings${module_suffix}.xml
-    newFindbugsWarnings=`$FINDBUGS_HOME/bin/filterBugs -first "01/01/2000" $PATCH_DIR/patchFindbugsWarnings${module_suffix}.xml \
-      $PATCH_DIR/newPatchFindbugsWarnings${module_suffix}.xml | $AWK '{print $1}'`
-    echo "Found $newFindbugsWarnings Findbugs warnings ($file)"
-    findbugsWarnings=$((findbugsWarnings+newFindbugsWarnings))
-    $FINDBUGS_HOME/bin/convertXmlToText -html \
-      $PATCH_DIR/newPatchFindbugsWarnings${module_suffix}.xml \
-      $PATCH_DIR/newPatchFindbugsWarnings${module_suffix}.html
-    if [[ $newFindbugsWarnings > 0 ]] ; then
-      JIRA_COMMENT_FOOTER="Findbugs warnings: $BUILD_URL/artifact/patchprocess/newPatchFindbugsWarnings${module_suffix}.html
+    cp $file $PATCH_DIR/patchSpotbugsWarnings${module_suffix}.xml
+    $SPOTBUGS_HOME/bin/setBugDatabaseInfo -timestamp "01/01/2000" \
+      $PATCH_DIR/patchSpotbugsWarnings${module_suffix}.xml \
+      $PATCH_DIR/patchSpotbugsWarnings${module_suffix}.xml
+    newSpotbugsWarnings=`$SPOTBUGS_HOME/bin/filterBugs -first "01/01/2000" $PATCH_DIR/patchSpotbugsWarnings${module_suffix}.xml \
+      $PATCH_DIR/newPatchSpotbugsWarnings${module_suffix}.xml | $AWK '{print $1}'`
+    echo "Found $newSpotbugsWarnings Spotbugs warnings ($file)"
+    spotbugsWarnings=$((spotbugsWarnings+newSpotbugsWarnings))
+    $SPOTBUGS_HOME/bin/convertXmlToText -html \
+      $PATCH_DIR/newPatchSpotbugsWarnings${module_suffix}.xml \
+      $PATCH_DIR/newPatchSpotbugsWarnings${module_suffix}.html
+    if [[ $newSpotbugsWarnings > 0 ]] ; then
+      JIRA_COMMENT_FOOTER="Spotbugs warnings: $BUILD_URL/artifact/patchprocess/newPatchSpotbugsWarnings${module_suffix}.html
 $JIRA_COMMENT_FOOTER"
     fi
   done
 
-  if [[ $findbugsWarnings -gt 0 ]] ; then
+  if [[ $spotbugsWarnings -gt 0 ]] ; then
     JIRA_COMMENT="$JIRA_COMMENT
 
-    {color:red}-1 findbugs{color}.  The patch appears to introduce $findbugsWarnings new Findbugs (version ${findbugs_version}) warnings."
+    {color:red}-1 spotbugs{color}.  The patch appears to introduce $spotbugsWarnings new Spotbugs (version ${spotbugs_version}) warnings."
     return 1
   fi
   JIRA_COMMENT="$JIRA_COMMENT
 
-    {color:green}+1 findbugs{color}.  The patch does not introduce any new Findbugs (version ${findbugs_version}) warnings."
+    {color:green}+1 spotbugs{color}.  The patch does not introduce any new Spotbugs (version ${spotbugs_version}) warnings."
   return 0
 }
 
@@ -805,7 +805,7 @@ fi
 checkJavadocWarnings
 (( RESULT = RESULT + $? ))
 buildAndInstall
-checkFindbugsWarnings
+checkSpotbugsWarnings
 (( RESULT = RESULT + $? ))
 checkReleaseAuditWarnings
 (( RESULT = RESULT + $? ))

pom.xml

@@ -68,7 +68,8 @@
     <protoc.path>${env.PROTOC_PATH}</protoc.path>
     <scm.url>scm:git:https://gitbox.apache.org/repos/asf/tez.git</scm.url>
     <frontend-maven-plugin.version>1.4</frontend-maven-plugin.version>
-    <findbugs-maven-plugin.version>3.0.5</findbugs-maven-plugin.version>
+    <spotbugs.version>4.2.2</spotbugs.version>
+    <spotbugs-maven-plugin.version>4.2.0</spotbugs-maven-plugin.version>
     <maven-checkstyle-plugin.version>3.1.1</maven-checkstyle-plugin.version>
     <checkstyle.version>8.35</checkstyle.version>
     <dependency-check-maven.version>1.3.6</dependency-check-maven.version>
@@ -738,11 +739,6 @@
         <artifactId>jettison</artifactId>
         <version>1.3.4</version>
       </dependency>
-      <dependency>
-        <groupId>com.google.code.findbugs</groupId>
-        <artifactId>jsr305</artifactId>
-        <version>3.0.0</version>
-      </dependency>
       <dependency>
         <groupId>com.sun.jersey</groupId>
         <artifactId>jersey-client</artifactId>
@@ -922,9 +918,20 @@
           <version>${dependency-check-maven.version}</version>
         </plugin>
         <plugin>
-          <groupId>org.codehaus.mojo</groupId>
-          <artifactId>findbugs-maven-plugin</artifactId>
-          <version>${findbugs-maven-plugin.version}</version>
+          <groupId>com.github.spotbugs</groupId>
+          <artifactId>spotbugs-maven-plugin</artifactId>
+          <version>${spotbugs-maven-plugin.version}</version>
+          <dependencies>
+            <dependency>
+              <groupId>com.github.spotbugs</groupId>
+              <artifactId>spotbugs</artifactId>
+              <version>${spotbugs.version}</version>
+            </dependency>
+          </dependencies>
+          <configuration>
+            <excludeFilterFile>${basedir}/spotbugs-exclude.xml</excludeFilterFile>
+            <failOnError>false</failOnError>
+          </configuration>
         </plugin>
         <plugin>
           <groupId>org.codehaus.mojo</groupId>
@@ -1046,11 +1053,8 @@
         <artifactId>build-helper-maven-plugin</artifactId>
       </plugin>
       <plugin>
-        <groupId>org.codehaus.mojo</groupId>
-        <artifactId>findbugs-maven-plugin</artifactId>
-        <configuration>
-          <excludeFilterFile>${basedir}/findbugs-exclude.xml</excludeFilterFile>
-        </configuration>
+        <groupId>com.github.spotbugs</groupId>
+        <artifactId>spotbugs-maven-plugin</artifactId>
       </plugin>
       <plugin>
        <!-- OWASP's dependency-check plugin will scan the third party
@@ -1142,45 +1146,23 @@
       </build>
     </profile>
     <profile>
-      <id>findbugs</id>
+      <id>spotbugs</id>
       <activation>
         <activeByDefault>false</activeByDefault>
         <property>
-          <name>findbugs</name>
+          <name>spotbugs</name>
         </property>
       </activation>
-      <build>
-        <plugins>
-          <plugin>
-            <groupId>org.codehaus.mojo</groupId>
-            <artifactId>findbugs-maven-plugin</artifactId>
-            <executions>
-              <execution>
-                <id>site</id>
-                <phase>pre-site</phase>
-                <goals>
-                  <goal>findbugs</goal>
-                </goals>
-              </execution>
-            </executions>
-            <configuration>
-              <excludeFilterFile>${basedir}/findbugs-exclude.xml</excludeFilterFile>
-            </configuration>
-          </plugin>
-        </plugins>
-      </build>
       <reporting>
         <plugins>
           <plugin>
-            <groupId>org.codehaus.mojo</groupId>
-            <artifactId>findbugs-maven-plugin</artifactId>
+            <groupId>com.github.spotbugs</groupId>
+            <artifactId>spotbugs-maven-plugin</artifactId>
             <configuration>
-              <formats>
-                <format>xml</format>
-                <format>html</format>
-              </formats>
-              <canGenerate>true</canGenerate>
-              <excludeFilterFile>${basedir}/findbugs-exclude.xml</excludeFilterFile>
+              <xmlOutput>true</xmlOutput>
+              <!-- Optional directory to put spotbugs xdoc xml report -->
+              <xmlOutputDirectory>target/site</xmlOutputDirectory>
+              <excludeFilterFile>${basedir}/spotbugs-exclude.xml</excludeFilterFile>
             </configuration>
           </plugin>
         </plugins>

tez-api/pom.xml

@@ -81,10 +81,6 @@
      <groupId>junit</groupId>
      <artifactId>junit</artifactId>
     </dependency>
-    <dependency>
-      <groupId>com.google.code.findbugs</groupId>
-      <artifactId>jsr305</artifactId>
-    </dependency>
     <dependency>
       <groupId>com.sun.jersey</groupId>
       <artifactId>jersey-client</artifactId>

tez-api/src/main/java/org/apache/tez/client/TezClientUtils.java

@@ -1010,11 +1010,11 @@ public static String maybeAddDefaultMemoryJavaOpts(String javaOpts, Resource res
       return javaOpts;
     }
 
-    if ((maxHeapFactor <= 0 && Double.valueOf("-1") != maxHeapFactor) || maxHeapFactor >= 1) {
+    if ((maxHeapFactor <= 0 && Double.parseDouble("-1") != maxHeapFactor) || maxHeapFactor >= 1) {
       return javaOpts;
     }
 
-    if (Double.valueOf("-1") == maxHeapFactor) {
+    if (Double.parseDouble("-1") == maxHeapFactor) {
       maxHeapFactor = resource.getMemory() < TezConstants.TEZ_CONTAINER_SMALL_SLAB_BOUND_MB
         ? TezConstants.TEZ_CONTAINER_MAX_JAVA_HEAP_FRACTION_SMALL_SLAB
         : TezConstants.TEZ_CONTAINER_MAX_JAVA_HEAP_FRACTION_LARGE_SLAB;

tez-dag/pom.xml

@@ -107,11 +107,6 @@
       <groupId>commons-cli</groupId>
       <artifactId>commons-cli</artifactId>
     </dependency>
-    <dependency>
-      <groupId>com.google.code.findbugs</groupId>
-      <artifactId>jsr305</artifactId>
-      <scope>compile</scope>
-    </dependency>
     <dependency>
      <groupId>org.apache.commons</groupId>
      <artifactId>commons-collections4</artifactId>

tez-dag/findbugs-exclude.xml → tez-dag/spotbugs-exclude.xml

@@ -13,6 +13,11 @@
 -->
 <FindBugsFilter>
 
+  <Match>
+    <Class name="org.apache.tez.dag.app.launcher.LocalContainerLauncher$RunningTaskCallback"/>
+    <Bug pattern="NP_PARAMETER_MUST_BE_NONNULL_BUT_MARKED_AS_NULLABLE"/>
+  </Match>
+
   <Match>
     <Class name="org.apache.tez.dag.app.rm.node.AMNodeTracker" />
     <Or>

tez-dag/src/main/java/org/apache/tez/dag/app/TaskCommunicatorContextImpl.java

@@ -22,7 +22,6 @@
 import java.util.Objects;
 
 import com.google.common.annotations.VisibleForTesting;
-import com.google.common.base.Function;
 import org.apache.tez.common.Preconditions;
 import com.google.common.collect.Iterables;
 import org.apache.hadoop.classification.InterfaceAudience;
@@ -181,12 +180,7 @@ public Iterable<String> getInputVertexNames(String vertexName) {
     DAG dag = getDag();
     Vertex vertex = dag.getVertex(vertexName);
     Set<Vertex> sources = vertex.getInputVertices().keySet();
-    return Iterables.transform(sources, new Function<Vertex, String>() {
-      @Override
-      public String apply(Vertex input) {
-        return input.getName();
-      }
-    });
+    return Iterables.transform(sources, input -> input.getName());
   }
 
   @Override