diff --git a/BUILDING.txt b/BUILDING.txt index 875bf3e097..9c5bfa9996 100644 --- a/BUILDING.txt +++ b/BUILDING.txt @@ -8,7 +8,7 @@ Requirements: * JDK 1.8+ * Maven 3.1 or later -* Findbugs 2.0.2 or later (if running findbugs) +* Spotbugs 4.2.2 or later (if running spotbugs) * ProtocolBuffer 2.5.0 * Internet connection for first build (to fetch all dependencies) * Hadoop version should be 2.7.0 or higher. @@ -36,7 +36,7 @@ Maven build goals: * Compile : mvn compile * Run tests : mvn test * Create JAR : mvn package - * Run findbugs : mvn compile findbugs:findbugs + * Run spotbugs : mvn compile spotbugs:spotbugs * Run checkstyle : mvn compile checkstyle:checkstyle * Install JAR in M2 cache : mvn install * Deploy JAR to Maven repo : mvn deploy diff --git a/Jenkinsfile b/Jenkinsfile index 4ce236598d..db270323b5 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -117,11 +117,11 @@ pipeline { YETUS_ARGS+=("--proclimit=5500") YETUS_ARGS+=("--dockermemlimit=20g") - # -1 findbugs issues that show up prior to the patch being applied - # YETUS_ARGS+=("--findbugs-strict-precheck") + # -1 spotbugs issues that show up prior to the patch being applied + # YETUS_ARGS+=("--spotbugs-strict-precheck") # rsync these files back into the archive dir - YETUS_ARGS+=("--archive-list=checkstyle-errors.xml,findbugsXml.xml") + YETUS_ARGS+=("--archive-list=checkstyle-errors.xml,spotbugsXml.xml") # URL for user-side presentation in reports and such to our artifacts # (needs to match the archive bits below) diff --git a/build-tools/docker/Dockerfile b/build-tools/docker/Dockerfile index f51cc3bd2b..92ab1ba22f 100644 --- a/build-tools/docker/Dockerfile +++ b/build-tools/docker/Dockerfile @@ -186,13 +186,14 @@ RUN apt-get -q update && apt-get -q install --no-install-recommends -y default-j ENV JAVA_HOME /usr/lib/jvm/java-8-openjdk-amd64 ###### -# Install findbugs +# Install SpotBugs 4.2.2 ###### -# hadolint ignore=DL3008 -RUN apt-get -q update && apt-get -q install --no-install-recommends -y findbugs \ - && apt-get clean \ - && rm -rf /var/lib/apt/lists/* -ENV FINDBUGS_HOME /usr +RUN mkdir -p /opt/spotbugs \ + && curl -L -s -S https://github.com/spotbugs/spotbugs/releases/download/4.2.2/spotbugs-4.2.2.tgz \ + -o /opt/spotbugs.tgz \ + && tar xzf /opt/spotbugs.tgz --strip-components 1 -C /opt/spotbugs \ + && chmod +x /opt/spotbugs/bin/* +ENV SPOTBUGS_HOME /opt/spotbugs ###### # Install maven diff --git a/build-tools/test-patch.sh b/build-tools/test-patch.sh index 9b457af221..feb9644893 100755 --- a/build-tools/test-patch.sh +++ b/build-tools/test-patch.sh @@ -40,7 +40,7 @@ DIFF=${DIFF:-diff} JIRACLI=${JIRA:-jira} SED=${SED:-sed} CURL=${CURL:-curl} -FINDBUGS_HOME=${FINDBUGS_HOME} +SPOTBUGS_HOME=${SPOTBUGS_HOME} ############################################################################### printUsage() { @@ -60,7 +60,7 @@ printUsage() { echo "--grep-cmd= The 'grep' command to use (default 'grep')" echo "--patch-cmd= The 'patch' command to use (default 'patch')" echo "--diff-cmd= The 'diff' command to use (default 'diff')" - echo "--findbugs-home= Findbugs home directory (default FINDBUGS_HOME environment variable)" + echo "--spotbugs-home= Spotbugs home directory (default SPOTBUGS_HOME environment variable)" echo "--dirty-workspace Allow the local git workspace to have uncommitted changes" echo "--run-tests Run all tests below the base directory" echo @@ -115,8 +115,8 @@ parseArgs() { --jira-password=*) JIRA_PASSWD=${i#*=} ;; - --findbugs-home=*) - FINDBUGS_HOME=${i#*=} + --spotbugs-home=*) + SPOTBUGS_HOME=${i#*=} ;; --dirty-workspace) DIRTY_WORKSPACE=true @@ -507,7 +507,7 @@ $JIRA_COMMENT_FOOTER" ############################################################################### -### Install the new jars so tests and findbugs can find all of the updated jars +### Install the new jars so tests and spotbugs can find all of the updated jars buildAndInstall () { echo "" echo "" @@ -525,67 +525,67 @@ buildAndInstall () { ############################################################################### -### Check there are no changes in the number of Findbugs warnings -checkFindbugsWarnings () { +### Check there are no changes in the number of Spotbugs warnings +checkSpotbugsWarnings () { echo "" echo "" echo "======================================================================" echo "======================================================================" - echo " Determining number of patched Findbugs warnings." + echo " Determining number of patched Spotbugs warnings." echo "======================================================================" echo "======================================================================" echo "" echo "" rc=0 - echo " Running findbugs " - echo "$MVN clean test findbugs:findbugs -DskipTests < /dev/null > $PATCH_DIR/patchFindBugsOutput.txt 2>&1" - $MVN clean test findbugs:findbugs -DskipTests < /dev/null > $PATCH_DIR/patchFindBugsOutput.txt 2>&1 + echo " Running spotbugs " + echo "$MVN clean test spotbugs:spotbugs -DskipTests < /dev/null > $PATCH_DIR/patchSpotBugsOutput.txt 2>&1" + $MVN clean test spotbugs:spotbugs -DskipTests < /dev/null > $PATCH_DIR/patchSpotBugsOutput.txt 2>&1 rc=$? - findbugs_version=$(${AWK} 'match($0, /findbugs-maven-plugin:[^:]*:findbugs/) { print substr($0, RSTART + 22, RLENGTH - 31); exit }' "${PATCH_DIR}/patchFindBugsOutput.txt") + spotbugs_version=$(${AWK} 'match($0, /spotbugs-maven-plugin:[^:]*:spotbugs/) { print substr($0, RSTART + 22, RLENGTH - 31); exit }' "${PATCH_DIR}/patchSpotBugsOutput.txt") if [ $rc != 0 ] ; then JIRA_COMMENT="$JIRA_COMMENT - {color:red}-1 findbugs{color}. The patch appears to cause Findbugs (version ${findbugs_version}) to fail." + {color:red}-1 spotbugs{color}. The patch appears to cause Spotbugs (version ${spotbugs_version}) to fail." return 1 fi - findbugsWarnings=0 - for file in $(find $BASEDIR -name findbugsXml.xml) + spotbugsWarnings=0 + for file in $(find $BASEDIR -name spotbugsXml.xml) do relative_file=${file#$BASEDIR/} # strip leading $BASEDIR prefix - if [ ! $relative_file == "target/findbugsXml.xml" ]; then - module_suffix=${relative_file%/target/findbugsXml.xml} # strip trailing path + if [ ! $relative_file == "target/spotbugsXml.xml" ]; then + module_suffix=${relative_file%/target/spotbugsXml.xml} # strip trailing path module_suffix=`basename ${module_suffix}` fi - cp $file $PATCH_DIR/patchFindbugsWarnings${module_suffix}.xml - $FINDBUGS_HOME/bin/setBugDatabaseInfo -timestamp "01/01/2000" \ - $PATCH_DIR/patchFindbugsWarnings${module_suffix}.xml \ - $PATCH_DIR/patchFindbugsWarnings${module_suffix}.xml - newFindbugsWarnings=`$FINDBUGS_HOME/bin/filterBugs -first "01/01/2000" $PATCH_DIR/patchFindbugsWarnings${module_suffix}.xml \ - $PATCH_DIR/newPatchFindbugsWarnings${module_suffix}.xml | $AWK '{print $1}'` - echo "Found $newFindbugsWarnings Findbugs warnings ($file)" - findbugsWarnings=$((findbugsWarnings+newFindbugsWarnings)) - $FINDBUGS_HOME/bin/convertXmlToText -html \ - $PATCH_DIR/newPatchFindbugsWarnings${module_suffix}.xml \ - $PATCH_DIR/newPatchFindbugsWarnings${module_suffix}.html - if [[ $newFindbugsWarnings > 0 ]] ; then - JIRA_COMMENT_FOOTER="Findbugs warnings: $BUILD_URL/artifact/patchprocess/newPatchFindbugsWarnings${module_suffix}.html + cp $file $PATCH_DIR/patchSpotbugsWarnings${module_suffix}.xml + $SPOTBUGS_HOME/bin/setBugDatabaseInfo -timestamp "01/01/2000" \ + $PATCH_DIR/patchSpotbugsWarnings${module_suffix}.xml \ + $PATCH_DIR/patchSpotbugsWarnings${module_suffix}.xml + newSpotbugsWarnings=`$SPOTBUGS_HOME/bin/filterBugs -first "01/01/2000" $PATCH_DIR/patchSpotbugsWarnings${module_suffix}.xml \ + $PATCH_DIR/newPatchSpotbugsWarnings${module_suffix}.xml | $AWK '{print $1}'` + echo "Found $newSpotbugsWarnings Spotbugs warnings ($file)" + spotbugsWarnings=$((spotbugsWarnings+newSpotbugsWarnings)) + $SPOTBUGS_HOME/bin/convertXmlToText -html \ + $PATCH_DIR/newPatchSpotbugsWarnings${module_suffix}.xml \ + $PATCH_DIR/newPatchSpotbugsWarnings${module_suffix}.html + if [[ $newSpotbugsWarnings > 0 ]] ; then + JIRA_COMMENT_FOOTER="Spotbugs warnings: $BUILD_URL/artifact/patchprocess/newPatchSpotbugsWarnings${module_suffix}.html $JIRA_COMMENT_FOOTER" fi done - if [[ $findbugsWarnings -gt 0 ]] ; then + if [[ $spotbugsWarnings -gt 0 ]] ; then JIRA_COMMENT="$JIRA_COMMENT - {color:red}-1 findbugs{color}. The patch appears to introduce $findbugsWarnings new Findbugs (version ${findbugs_version}) warnings." + {color:red}-1 spotbugs{color}. The patch appears to introduce $spotbugsWarnings new Spotbugs (version ${spotbugs_version}) warnings." return 1 fi JIRA_COMMENT="$JIRA_COMMENT - {color:green}+1 findbugs{color}. The patch does not introduce any new Findbugs (version ${findbugs_version}) warnings." + {color:green}+1 spotbugs{color}. The patch does not introduce any new Spotbugs (version ${spotbugs_version}) warnings." return 0 } @@ -805,7 +805,7 @@ fi checkJavadocWarnings (( RESULT = RESULT + $? )) buildAndInstall -checkFindbugsWarnings +checkSpotbugsWarnings (( RESULT = RESULT + $? )) checkReleaseAuditWarnings (( RESULT = RESULT + $? )) diff --git a/hadoop-shim-impls/findbugs-exclude.xml b/hadoop-shim-impls/hadoop-shim-2.7/spotbugs-exclude.xml similarity index 100% rename from hadoop-shim-impls/findbugs-exclude.xml rename to hadoop-shim-impls/hadoop-shim-2.7/spotbugs-exclude.xml diff --git a/hadoop-shim-impls/hadoop-shim-2.7/findbugs-exclude.xml b/hadoop-shim-impls/hadoop-shim-2.8/spotbugs-exclude.xml similarity index 100% rename from hadoop-shim-impls/hadoop-shim-2.7/findbugs-exclude.xml rename to hadoop-shim-impls/hadoop-shim-2.8/spotbugs-exclude.xml diff --git a/hadoop-shim-impls/hadoop-shim-2.8/findbugs-exclude.xml b/hadoop-shim-impls/spotbugs-exclude.xml similarity index 100% rename from hadoop-shim-impls/hadoop-shim-2.8/findbugs-exclude.xml rename to hadoop-shim-impls/spotbugs-exclude.xml diff --git a/hadoop-shim/findbugs-exclude.xml b/hadoop-shim/spotbugs-exclude.xml similarity index 100% rename from hadoop-shim/findbugs-exclude.xml rename to hadoop-shim/spotbugs-exclude.xml diff --git a/pom.xml b/pom.xml index aff586bad9..1d32147225 100644 --- a/pom.xml +++ b/pom.xml @@ -68,7 +68,8 @@ ${env.PROTOC_PATH} scm:git:https://gitbox.apache.org/repos/asf/tez.git 1.4 - 3.0.5 + 4.2.2 + 4.2.0 3.1.1 8.35 1.3.6 @@ -738,11 +739,6 @@ jettison 1.3.4 - - com.google.code.findbugs - jsr305 - 3.0.0 - com.sun.jersey jersey-client @@ -922,9 +918,20 @@ ${dependency-check-maven.version} - org.codehaus.mojo - findbugs-maven-plugin - ${findbugs-maven-plugin.version} + com.github.spotbugs + spotbugs-maven-plugin + ${spotbugs-maven-plugin.version} + + + com.github.spotbugs + spotbugs + ${spotbugs.version} + + + + ${basedir}/spotbugs-exclude.xml + false + org.codehaus.mojo @@ -1046,11 +1053,8 @@ build-helper-maven-plugin - org.codehaus.mojo - findbugs-maven-plugin - - ${basedir}/findbugs-exclude.xml - + com.github.spotbugs + spotbugs-maven-plugin + target/site + ${basedir}/spotbugs-exclude.xml diff --git a/tez-api/pom.xml b/tez-api/pom.xml index 8cab40c172..972fc09cb7 100644 --- a/tez-api/pom.xml +++ b/tez-api/pom.xml @@ -81,10 +81,6 @@ junit junit - - com.google.code.findbugs - jsr305 - com.sun.jersey jersey-client diff --git a/tez-api/findbugs-exclude.xml b/tez-api/spotbugs-exclude.xml similarity index 100% rename from tez-api/findbugs-exclude.xml rename to tez-api/spotbugs-exclude.xml diff --git a/tez-api/src/main/java/org/apache/tez/client/TezClientUtils.java b/tez-api/src/main/java/org/apache/tez/client/TezClientUtils.java index d0fc374a81..0337c8b90f 100644 --- a/tez-api/src/main/java/org/apache/tez/client/TezClientUtils.java +++ b/tez-api/src/main/java/org/apache/tez/client/TezClientUtils.java @@ -1010,11 +1010,11 @@ public static String maybeAddDefaultMemoryJavaOpts(String javaOpts, Resource res return javaOpts; } - if ((maxHeapFactor <= 0 && Double.valueOf("-1") != maxHeapFactor) || maxHeapFactor >= 1) { + if ((maxHeapFactor <= 0 && Double.parseDouble("-1") != maxHeapFactor) || maxHeapFactor >= 1) { return javaOpts; } - if (Double.valueOf("-1") == maxHeapFactor) { + if (Double.parseDouble("-1") == maxHeapFactor) { maxHeapFactor = resource.getMemory() < TezConstants.TEZ_CONTAINER_SMALL_SLAB_BOUND_MB ? TezConstants.TEZ_CONTAINER_MAX_JAVA_HEAP_FRACTION_SMALL_SLAB : TezConstants.TEZ_CONTAINER_MAX_JAVA_HEAP_FRACTION_LARGE_SLAB; diff --git a/tez-common/findbugs-exclude.xml b/tez-common/spotbugs-exclude.xml similarity index 100% rename from tez-common/findbugs-exclude.xml rename to tez-common/spotbugs-exclude.xml diff --git a/tez-dag/pom.xml b/tez-dag/pom.xml index 723082c54e..5fad248784 100644 --- a/tez-dag/pom.xml +++ b/tez-dag/pom.xml @@ -107,11 +107,6 @@ commons-cli commons-cli - - com.google.code.findbugs - jsr305 - compile - org.apache.commons commons-collections4 diff --git a/tez-dag/findbugs-exclude.xml b/tez-dag/spotbugs-exclude.xml similarity index 97% rename from tez-dag/findbugs-exclude.xml rename to tez-dag/spotbugs-exclude.xml index e8755832f4..a857898cf3 100644 --- a/tez-dag/findbugs-exclude.xml +++ b/tez-dag/spotbugs-exclude.xml @@ -13,6 +13,11 @@ --> + + + + + diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/TaskCommunicatorContextImpl.java b/tez-dag/src/main/java/org/apache/tez/dag/app/TaskCommunicatorContextImpl.java index 4f9e8723aa..8b08227f79 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/TaskCommunicatorContextImpl.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/TaskCommunicatorContextImpl.java @@ -22,7 +22,6 @@ import java.util.Objects; import com.google.common.annotations.VisibleForTesting; -import com.google.common.base.Function; import org.apache.tez.common.Preconditions; import com.google.common.collect.Iterables; import org.apache.hadoop.classification.InterfaceAudience; @@ -181,12 +180,7 @@ public Iterable getInputVertexNames(String vertexName) { DAG dag = getDag(); Vertex vertex = dag.getVertex(vertexName); Set sources = vertex.getInputVertices().keySet(); - return Iterables.transform(sources, new Function() { - @Override - public String apply(Vertex input) { - return input.getName(); - } - }); + return Iterables.transform(sources, input -> input.getName()); } @Override diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/TaskImpl.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/TaskImpl.java index d2933c5b86..4da1964990 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/TaskImpl.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/TaskImpl.java @@ -33,7 +33,6 @@ import java.util.concurrent.locks.ReentrantReadWriteLock; import org.apache.tez.common.Preconditions; -import com.google.common.base.Predicate; import com.google.common.collect.Maps; import org.apache.commons.lang.StringUtils; @@ -954,12 +953,7 @@ private void addDiagnosticInfo(String diag) { int getUncompletedAttemptsCount() { try { readLock.lock(); - return Maps.filterValues(taskAttemptStatus, new Predicate() { - @Override - public boolean apply(Boolean state) { - return !state; - } - }).size(); + return Maps.filterValues(taskAttemptStatus, state -> !state).size(); } finally { readLock.unlock(); } @@ -969,12 +963,7 @@ public boolean apply(Boolean state) { int getFinishedAttemptsCount() { try { readLock.lock(); - return Maps.filterValues(taskAttemptStatus, new Predicate() { - @Override - public boolean apply(Boolean state) { - return state; - } - }).size(); + return Maps.filterValues(taskAttemptStatus, state -> state).size(); } finally { readLock.unlock(); } diff --git a/tez-dag/src/main/java/org/apache/tez/dag/history/events/VertexGroupCommitFinishedEvent.java b/tez-dag/src/main/java/org/apache/tez/dag/history/events/VertexGroupCommitFinishedEvent.java index c9d5aae770..8334c8c481 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/history/events/VertexGroupCommitFinishedEvent.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/history/events/VertexGroupCommitFinishedEvent.java @@ -33,7 +33,6 @@ import org.apache.tez.dag.recovery.records.RecoveryProtos.SummaryEventProto; import org.apache.tez.dag.recovery.records.RecoveryProtos.VertexGroupCommitFinishedProto; -import com.google.common.base.Function; import com.google.common.collect.Collections2; public class VertexGroupCommitFinishedEvent implements HistoryEvent, SummaryEvent { @@ -70,12 +69,7 @@ public boolean isHistoryEvent() { } public VertexGroupCommitFinishedProto toProto() { - Collection vertexIdsStr = Collections2.transform(vertexIds, new Function(){ - @Override - public String apply(TezVertexID vertexId) { - return vertexId.toString(); - } - }); + Collection vertexIdsStr = Collections2.transform(vertexIds, vertexId -> vertexId.toString()); return VertexGroupCommitFinishedProto.newBuilder() .setDagId(dagID.toString()) .setVertexGroupName(vertexGroupName) @@ -86,12 +80,7 @@ public String apply(TezVertexID vertexId) { public void fromProto(VertexGroupCommitFinishedProto proto) { this.dagID = TezDAGID.fromString(proto.getDagId()); this.vertexGroupName = proto.getVertexGroupName(); - this.vertexIds = Collections2.transform(proto.getVertexIdsList(), new Function() { - @Override - public TezVertexID apply(String input) { - return TezVertexID.fromString(input); - } - }); + this.vertexIds = Collections2.transform(proto.getVertexIdsList(), input -> TezVertexID.fromString(input)); } @Override diff --git a/tez-dag/src/main/java/org/apache/tez/dag/history/events/VertexGroupCommitStartedEvent.java b/tez-dag/src/main/java/org/apache/tez/dag/history/events/VertexGroupCommitStartedEvent.java index cdd11bcc4c..cc9b666dc1 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/history/events/VertexGroupCommitStartedEvent.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/history/events/VertexGroupCommitStartedEvent.java @@ -33,7 +33,6 @@ import org.apache.tez.dag.recovery.records.RecoveryProtos.SummaryEventProto; import org.apache.tez.dag.recovery.records.RecoveryProtos.VertexGroupCommitStartedProto; -import com.google.common.base.Function; import com.google.common.collect.Collections2; public class VertexGroupCommitStartedEvent implements HistoryEvent, SummaryEvent { @@ -70,12 +69,7 @@ public boolean isHistoryEvent() { } public VertexGroupCommitStartedProto toProto() { - Collection vertexIdsStr = Collections2.transform(vertexIds, new Function(){ - @Override - public String apply(TezVertexID vertexId) { - return vertexId.toString(); - } - }); + Collection vertexIdsStr = Collections2.transform(vertexIds, vertexId -> vertexId.toString()); return VertexGroupCommitStartedProto.newBuilder() .setDagId(dagID.toString()) .setVertexGroupName(vertexGroupName) @@ -86,12 +80,7 @@ public String apply(TezVertexID vertexId) { public void fromProto(VertexGroupCommitStartedProto proto) { this.dagID = TezDAGID.fromString(proto.getDagId()); this.vertexGroupName = proto.getVertexGroupName(); - this.vertexIds = Collections2.transform(proto.getVertexIdsList(), new Function() { - @Override - public TezVertexID apply(String input) { - return TezVertexID.fromString(input); - } - }); + this.vertexIds = Collections2.transform(proto.getVertexIdsList(), input -> TezVertexID.fromString(input)); } @Override diff --git a/tez-examples/findbugs-exclude.xml b/tez-examples/spotbugs-exclude.xml similarity index 100% rename from tez-examples/findbugs-exclude.xml rename to tez-examples/spotbugs-exclude.xml diff --git a/tez-ext-service-tests/findbugs-exclude.xml b/tez-ext-service-tests/spotbugs-exclude.xml similarity index 100% rename from tez-ext-service-tests/findbugs-exclude.xml rename to tez-ext-service-tests/spotbugs-exclude.xml diff --git a/tez-mapreduce/findbugs-exclude.xml b/tez-mapreduce/spotbugs-exclude.xml similarity index 100% rename from tez-mapreduce/findbugs-exclude.xml rename to tez-mapreduce/spotbugs-exclude.xml diff --git a/tez-mapreduce/src/main/java/org/apache/hadoop/mapred/split/TezMapredSplitsGrouper.java b/tez-mapreduce/src/main/java/org/apache/hadoop/mapred/split/TezMapredSplitsGrouper.java index 38a213ec9a..6f7339d373 100644 --- a/tez-mapreduce/src/main/java/org/apache/hadoop/mapred/split/TezMapredSplitsGrouper.java +++ b/tez-mapreduce/src/main/java/org/apache/hadoop/mapred/split/TezMapredSplitsGrouper.java @@ -23,9 +23,7 @@ import java.util.List; import java.util.Objects; -import com.google.common.base.Function; import com.google.common.collect.Lists; -import org.apache.tez.mapreduce.grouper.GroupedSplitContainer; import org.apache.tez.mapreduce.grouper.MapredSplitContainer; import org.apache.tez.mapreduce.grouper.SplitContainer; import org.apache.tez.mapreduce.grouper.SplitLocationProviderWrapperMapred; @@ -69,36 +67,21 @@ public InputSplit[] getGroupedSplits(Configuration conf, Objects.requireNonNull(originalSplits, "Splits must be specified"); List originalSplitContainers = Lists.transform(Arrays.asList(originalSplits), - new Function() { - @Override - public SplitContainer apply(InputSplit input) { - return new MapredSplitContainer(input); - } - }); + input -> new MapredSplitContainer(input)); try { - List resultList = Lists.transform(super - .getGroupedSplits(conf, originalSplitContainers, desiredNumSplits, - wrappedInputFormatName, estimator == null ? null : - new SplitSizeEstimatorWrapperMapred(estimator), - locationProvider == null ? null : - new SplitLocationProviderWrapperMapred(locationProvider)), - new Function() { - @Override - public InputSplit apply(GroupedSplitContainer input) { - List underlyingSplits = Lists.transform(input.getWrappedSplitContainers(), - new Function() { - @Override - public InputSplit apply(SplitContainer input) { - return ((MapredSplitContainer) input).getRawSplit(); - } - }); + List resultList = + Lists.transform(super.getGroupedSplits(conf, originalSplitContainers, desiredNumSplits, + wrappedInputFormatName, estimator == null ? null : new SplitSizeEstimatorWrapperMapred(estimator), + locationProvider == null ? null : new SplitLocationProviderWrapperMapred(locationProvider)), + input -> { + List underlyingSplits = Lists.transform(input.getWrappedSplitContainers(), + input1 -> ((MapredSplitContainer) input1).getRawSplit()); - - return new TezGroupedSplit(underlyingSplits, input.getWrappedInputFormatName(), - input.getLocations(), input.getRack(), input.getLength()); - } - }); + return new TezGroupedSplit(underlyingSplits, input.getWrappedInputFormatName(), + input.getLocations(), input.getRack(), input.getLength()); + } + ); InputSplit[] resultArr = resultList.toArray(new InputSplit[resultList.size()]); return resultArr; } catch (InterruptedException e) { diff --git a/tez-mapreduce/src/main/java/org/apache/hadoop/mapreduce/split/TezMapReduceSplitsGrouper.java b/tez-mapreduce/src/main/java/org/apache/hadoop/mapreduce/split/TezMapReduceSplitsGrouper.java index 4f638e1400..5622e91b03 100644 --- a/tez-mapreduce/src/main/java/org/apache/hadoop/mapreduce/split/TezMapReduceSplitsGrouper.java +++ b/tez-mapreduce/src/main/java/org/apache/hadoop/mapreduce/split/TezMapReduceSplitsGrouper.java @@ -24,9 +24,7 @@ import javax.annotation.Nullable; -import com.google.common.base.Function; import com.google.common.collect.Lists; -import org.apache.tez.mapreduce.grouper.GroupedSplitContainer; import org.apache.tez.mapreduce.grouper.MapReduceSplitContainer; import org.apache.tez.mapreduce.grouper.SplitContainer; import org.apache.tez.mapreduce.grouper.SplitSizeEstimatorWrapperMapReduce; @@ -156,37 +154,21 @@ public List getGroupedSplits(Configuration conf, InterruptedException { Objects.requireNonNull(originalSplits, "Splits must be specified"); List originalSplitContainers = Lists.transform(originalSplits, - new Function() { - @Override - public SplitContainer apply(InputSplit input) { - return new MapReduceSplitContainer(input); - } - }); + input -> new MapReduceSplitContainer(input)); - return Lists.transform(super - .getGroupedSplits(conf, originalSplitContainers, desiredNumSplits, - wrappedInputFormatName, estimator == null ? null : - new SplitSizeEstimatorWrapperMapReduce(estimator), - locationProvider == null ? null : - new SplitLocationProviderMapReduce(locationProvider)), - new Function() { - @Override - public InputSplit apply(GroupedSplitContainer input) { + return Lists.transform( + super.getGroupedSplits(conf, originalSplitContainers, desiredNumSplits, wrappedInputFormatName, + estimator == null ? null : new SplitSizeEstimatorWrapperMapReduce(estimator), + locationProvider == null ? null : new SplitLocationProviderMapReduce(locationProvider)), + input -> { + List underlyingSplits = Lists.transform(input.getWrappedSplitContainers(), + input1 -> ((MapReduceSplitContainer) input1).getRawSplit()); - List underlyingSplits = Lists.transform(input.getWrappedSplitContainers(), - new Function() { - @Override - public InputSplit apply(SplitContainer input) { - return ((MapReduceSplitContainer) input).getRawSplit(); - } - }); - - return new TezGroupedSplit(underlyingSplits, input.getWrappedInputFormatName(), - input.getLocations(), input.getRack(), input.getLength()); - - } - }); + return new TezGroupedSplit(underlyingSplits, input.getWrappedInputFormatName(), input.getLocations(), + input.getRack(), input.getLength()); + } + ); } /** diff --git a/tez-plugins/tez-aux-services/findbugs-exclude.xml b/tez-plugins/tez-aux-services/spotbugs-exclude.xml similarity index 100% rename from tez-plugins/tez-aux-services/findbugs-exclude.xml rename to tez-plugins/tez-aux-services/spotbugs-exclude.xml diff --git a/tez-plugins/tez-history-parser/findbugs-exclude.xml b/tez-plugins/tez-history-parser/spotbugs-exclude.xml similarity index 100% rename from tez-plugins/tez-history-parser/findbugs-exclude.xml rename to tez-plugins/tez-history-parser/spotbugs-exclude.xml diff --git a/tez-plugins/tez-history-parser/src/main/java/org/apache/tez/history/parser/datamodel/BaseParser.java b/tez-plugins/tez-history-parser/src/main/java/org/apache/tez/history/parser/datamodel/BaseParser.java index 9f3881c8b2..497783405d 100644 --- a/tez-plugins/tez-history-parser/src/main/java/org/apache/tez/history/parser/datamodel/BaseParser.java +++ b/tez-plugins/tez-history-parser/src/main/java/org/apache/tez/history/parser/datamodel/BaseParser.java @@ -19,7 +19,6 @@ package org.apache.tez.history.parser.datamodel; import org.apache.tez.common.Preconditions; -import com.google.common.base.Predicate; import com.google.common.collect.Iterables; import com.google.common.collect.Lists; import org.apache.tez.dag.records.TezTaskAttemptID; @@ -127,11 +126,9 @@ protected void linkParsedContents() { * @param referenceTime */ private void setReferenceTime(List eventList, final long referenceTime) { - Iterables.all(eventList, new Predicate() { - @Override public boolean apply(Event input) { - input.setReferenceTime(referenceTime); - return false; - } + Iterables.all(eventList, input -> { + input.setReferenceTime(referenceTime); + return false; }); } diff --git a/tez-plugins/tez-history-parser/src/main/java/org/apache/tez/history/parser/datamodel/DagInfo.java b/tez-plugins/tez-history-parser/src/main/java/org/apache/tez/history/parser/datamodel/DagInfo.java index 5067ec7aa8..35ed963ee5 100644 --- a/tez-plugins/tez-history-parser/src/main/java/org/apache/tez/history/parser/datamodel/DagInfo.java +++ b/tez-plugins/tez-history-parser/src/main/java/org/apache/tez/history/parser/datamodel/DagInfo.java @@ -19,8 +19,6 @@ package org.apache.tez.history.parser.datamodel; import org.apache.tez.common.Preconditions; -import com.google.common.base.Predicate; -import com.google.common.collect.Iterables; import com.google.common.collect.LinkedHashMultimap; import com.google.common.collect.LinkedListMultimap; import com.google.common.collect.Lists; @@ -47,6 +45,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.stream.Collectors; import static org.apache.hadoop.classification.InterfaceAudience.Public; import static org.apache.hadoop.classification.InterfaceStability.Evolving; @@ -545,13 +544,9 @@ public final List getSuccessfullVertices() { * @return Collection */ public final List getVertices(final VertexState state) { - return Collections.unmodifiableList(Lists.newLinkedList(Iterables.filter(Lists.newLinkedList - (vertexNameMap.values()), new Predicate() { - @Override public boolean apply(VertexInfo input) { - return input.getStatus() != null && input.getStatus().equals(state.toString()); - } - } - ) + return Collections.unmodifiableList(Lists.newLinkedList(Lists.newLinkedList(vertexNameMap.values()).stream() + .filter(input -> input.getStatus() != null && input.getStatus().equals(state.toString())) + .collect(Collectors.toList()) ) ); } @@ -561,13 +556,9 @@ public final Map getVertexMapping() { } private Ordering getVertexOrdering() { - return Ordering.from(new Comparator() { - @Override public int compare(VertexInfo o1, VertexInfo o2) { - return (o1.getTimeTaken() < o2.getTimeTaken()) ? -1 : - ((o1.getTimeTaken() == o2.getTimeTaken()) ? - 0 : 1); - } - }); + return Ordering.from((o1, o2) -> (o1.getTimeTaken() < o2.getTimeTaken()) ? -1 : + ((o1.getTimeTaken() == o2.getTimeTaken()) ? + 0 : 1)); } /** diff --git a/tez-plugins/tez-history-parser/src/main/java/org/apache/tez/history/parser/datamodel/TaskInfo.java b/tez-plugins/tez-history-parser/src/main/java/org/apache/tez/history/parser/datamodel/TaskInfo.java index fd01093244..ee09501a16 100644 --- a/tez-plugins/tez-history-parser/src/main/java/org/apache/tez/history/parser/datamodel/TaskInfo.java +++ b/tez-plugins/tez-history-parser/src/main/java/org/apache/tez/history/parser/datamodel/TaskInfo.java @@ -19,9 +19,7 @@ package org.apache.tez.history.parser.datamodel; import org.apache.tez.common.Preconditions; -import com.google.common.base.Predicate; import com.google.common.base.Strings; -import com.google.common.collect.Iterables; import com.google.common.collect.LinkedHashMultimap; import com.google.common.collect.Lists; import com.google.common.collect.Maps; @@ -42,6 +40,7 @@ import java.util.List; import java.util.Map; import java.util.Objects; +import java.util.stream.Collectors; import static org.apache.hadoop.classification.InterfaceAudience.Public; import static org.apache.hadoop.classification.InterfaceStability.Evolving; @@ -208,14 +207,9 @@ public final List getSuccessfulTaskAttempts() { * @return Collection */ public final List getTaskAttempts(final TaskAttemptState state) { - return Collections.unmodifiableList(Lists.newLinkedList(Iterables.filter(Lists.newLinkedList - (attemptInfoMap.values()), new Predicate() { - @Override - public boolean apply(TaskAttemptInfo input) { - return input.getStatus() != null && input.getStatus().equals(state.toString()); - } - } - ) + return Collections.unmodifiableList(Lists.newLinkedList(Lists.newLinkedList(attemptInfoMap.values()).stream() + .filter(input -> input.getStatus() != null && input.getStatus().equals(state.toString())) + .collect(Collectors.toList()) ) ); } diff --git a/tez-plugins/tez-history-parser/src/main/java/org/apache/tez/history/parser/datamodel/VertexInfo.java b/tez-plugins/tez-history-parser/src/main/java/org/apache/tez/history/parser/datamodel/VertexInfo.java index 106f2bc729..0bf135186f 100644 --- a/tez-plugins/tez-history-parser/src/main/java/org/apache/tez/history/parser/datamodel/VertexInfo.java +++ b/tez-plugins/tez-history-parser/src/main/java/org/apache/tez/history/parser/datamodel/VertexInfo.java @@ -19,8 +19,6 @@ package org.apache.tez.history.parser.datamodel; import org.apache.tez.common.Preconditions; -import com.google.common.base.Predicate; -import com.google.common.collect.Iterables; import com.google.common.collect.LinkedHashMultimap; import com.google.common.collect.Lists; import com.google.common.collect.Maps; @@ -42,6 +40,7 @@ import java.util.List; import java.util.Map; import java.util.Objects; +import java.util.stream.Collectors; import static org.apache.hadoop.classification.InterfaceAudience.Public; import static org.apache.hadoop.classification.InterfaceStability.Evolving; @@ -408,13 +407,9 @@ public final List getSuccessfulTasks() { * @return List */ public final List getTasks(final TaskState state) { - return Collections.unmodifiableList(Lists.newLinkedList(Iterables.filter(Lists.newLinkedList - (taskInfoMap.values()), new Predicate() { - @Override public boolean apply(TaskInfo input) { - return input.getStatus() != null && input.getStatus().equals(state.toString()); - } - } - ) + return Collections.unmodifiableList(Lists.newLinkedList(Lists.newLinkedList(taskInfoMap.values()).stream() + .filter(input -> input.getStatus() != null && input.getStatus().equals(state.toString())) + .collect(Collectors.toList()) ) ); } diff --git a/tez-plugins/tez-protobuf-history-plugin/findbugs-exclude.xml b/tez-plugins/tez-protobuf-history-plugin/spotbugs-exclude.xml similarity index 100% rename from tez-plugins/tez-protobuf-history-plugin/findbugs-exclude.xml rename to tez-plugins/tez-protobuf-history-plugin/spotbugs-exclude.xml diff --git a/tez-plugins/tez-yarn-timeline-cache-plugin/findbugs-exclude.xml b/tez-plugins/tez-yarn-timeline-cache-plugin/spotbugs-exclude.xml similarity index 100% rename from tez-plugins/tez-yarn-timeline-cache-plugin/findbugs-exclude.xml rename to tez-plugins/tez-yarn-timeline-cache-plugin/spotbugs-exclude.xml diff --git a/tez-plugins/tez-yarn-timeline-history-with-acls/findbugs-exclude.xml b/tez-plugins/tez-yarn-timeline-history-with-acls/spotbugs-exclude.xml similarity index 100% rename from tez-plugins/tez-yarn-timeline-history-with-acls/findbugs-exclude.xml rename to tez-plugins/tez-yarn-timeline-history-with-acls/spotbugs-exclude.xml diff --git a/tez-plugins/tez-yarn-timeline-history-with-fs/findbugs-exclude.xml b/tez-plugins/tez-yarn-timeline-history-with-fs/spotbugs-exclude.xml similarity index 100% rename from tez-plugins/tez-yarn-timeline-history-with-fs/findbugs-exclude.xml rename to tez-plugins/tez-yarn-timeline-history-with-fs/spotbugs-exclude.xml diff --git a/tez-plugins/tez-yarn-timeline-history/findbugs-exclude.xml b/tez-plugins/tez-yarn-timeline-history/spotbugs-exclude.xml similarity index 100% rename from tez-plugins/tez-yarn-timeline-history/findbugs-exclude.xml rename to tez-plugins/tez-yarn-timeline-history/spotbugs-exclude.xml diff --git a/tez-runtime-internals/pom.xml b/tez-runtime-internals/pom.xml index f7e14dae59..1a900ee3f6 100644 --- a/tez-runtime-internals/pom.xml +++ b/tez-runtime-internals/pom.xml @@ -65,10 +65,6 @@ junit junit - - com.google.code.findbugs - jsr305 - org.mockito mockito-all diff --git a/tez-runtime-internals/findbugs-exclude.xml b/tez-runtime-internals/spotbugs-exclude.xml similarity index 92% rename from tez-runtime-internals/findbugs-exclude.xml rename to tez-runtime-internals/spotbugs-exclude.xml index 2b308de838..f33c285c60 100644 --- a/tez-runtime-internals/findbugs-exclude.xml +++ b/tez-runtime-internals/spotbugs-exclude.xml @@ -13,6 +13,10 @@ --> + + + + diff --git a/tez-runtime-internals/src/main/java/org/apache/tez/runtime/common/resources/MemoryDistributor.java b/tez-runtime-internals/src/main/java/org/apache/tez/runtime/common/resources/MemoryDistributor.java index d8ba1f71ff..61b3e9d44a 100644 --- a/tez-runtime-internals/src/main/java/org/apache/tez/runtime/common/resources/MemoryDistributor.java +++ b/tez-runtime-internals/src/main/java/org/apache/tez/runtime/common/resources/MemoryDistributor.java @@ -43,7 +43,6 @@ import org.apache.tez.runtime.api.TaskContext; import com.google.common.annotations.VisibleForTesting; -import com.google.common.base.Function; import org.apache.tez.common.Preconditions; import com.google.common.collect.Iterables; @@ -123,19 +122,11 @@ public void makeInitialAllocations() throws TezException { logInitialRequests(requestList); Iterable requestContexts = Iterables.transform(requestList, - new Function() { - public InitialMemoryRequestContext apply(RequestorInfo requestInfo) { - return requestInfo.getRequestContext(); - } - }); + requestInfo -> requestInfo.getRequestContext()); Iterable allocations = null; if (!isEnabled) { - allocations = Iterables.transform(requestList, new Function() { - public Long apply(RequestorInfo requestInfo) { - return requestInfo.getRequestContext().getRequestedSize(); - } - }); + allocations = Iterables.transform(requestList, requestInfo -> requestInfo.getRequestContext().getRequestedSize()); } else { InitialMemoryAllocator allocator = ReflectionUtils.createClazzInstance(allocatorClassName); allocator.setConf(conf); diff --git a/tez-runtime-internals/src/main/java/org/apache/tez/runtime/common/resources/ScalingAllocator.java b/tez-runtime-internals/src/main/java/org/apache/tez/runtime/common/resources/ScalingAllocator.java index e045abd9d2..1cb9d5d03f 100644 --- a/tez-runtime-internals/src/main/java/org/apache/tez/runtime/common/resources/ScalingAllocator.java +++ b/tez-runtime-internals/src/main/java/org/apache/tez/runtime/common/resources/ScalingAllocator.java @@ -28,7 +28,6 @@ import org.apache.hadoop.conf.Configuration; import org.apache.tez.dag.api.TezConfiguration; -import com.google.common.base.Function; import org.apache.tez.common.Preconditions; import com.google.common.collect.Iterables; import com.google.common.collect.Lists; @@ -67,12 +66,7 @@ public Iterable assignMemory(long availableForAllocation, int numTotalInpu if (totalRequested < availableForAllocation || totalRequested == 0) { // Not scaling up requests. Assuming things were setup correctly by // users in this case, keeping Processor, caching etc in mind. - return Lists.newArrayList(Iterables.transform(requests, - new Function() { - public Long apply(InitialMemoryRequestContext requestContext) { - return requestContext.getRequestedSize(); - } - })); + return Lists.newArrayList(Iterables.transform(requests, requestContext -> requestContext.getRequestedSize())); } List allocations = Lists.newArrayListWithCapacity(numRequests); diff --git a/tez-runtime-internals/src/main/java/org/apache/tez/runtime/task/TezChild.java b/tez-runtime-internals/src/main/java/org/apache/tez/runtime/task/TezChild.java index bc8c2d8391..c46e1468f0 100644 --- a/tez-runtime-internals/src/main/java/org/apache/tez/runtime/task/TezChild.java +++ b/tez-runtime-internals/src/main/java/org/apache/tez/runtime/task/TezChild.java @@ -20,7 +20,6 @@ import java.io.IOException; import java.net.InetSocketAddress; -import java.net.URI; import java.net.URL; import java.nio.ByteBuffer; import java.security.PrivilegedExceptionAction; @@ -74,7 +73,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import com.google.common.base.Function; import org.apache.tez.common.Preconditions; import org.apache.tez.common.TezClassLoader; @@ -346,12 +344,7 @@ private void handleNewTaskLocalResources(ContainerTask containerTask, @Override public List run() throws Exception { return RelocalizationUtils.processAdditionalResources( - Maps.transformValues(additionalResources, new Function() { - @Override - public URI apply(TezLocalResource input) { - return input.getUri(); - } - }), defaultConf, workingDir); + Maps.transformValues(additionalResources, input -> input.getUri()), defaultConf, workingDir); } }); RelocalizationUtils.addUrlsToClassPath(downloadedUrls); diff --git a/tez-runtime-library/findbugs-exclude.xml b/tez-runtime-library/spotbugs-exclude.xml similarity index 92% rename from tez-runtime-library/findbugs-exclude.xml rename to tez-runtime-library/spotbugs-exclude.xml index caa46c5527..ff224a3c46 100644 --- a/tez-runtime-library/findbugs-exclude.xml +++ b/tez-runtime-library/spotbugs-exclude.xml @@ -13,6 +13,22 @@ --> + + + + + + + + + + + + + + + + @@ -184,13 +200,13 @@ - + - + diff --git a/tez-runtime-library/src/main/java/org/apache/tez/dag/library/vertexmanager/ShuffleVertexManagerBase.java b/tez-runtime-library/src/main/java/org/apache/tez/dag/library/vertexmanager/ShuffleVertexManagerBase.java index 05e2d8ce78..5374c9d273 100644 --- a/tez-runtime-library/src/main/java/org/apache/tez/dag/library/vertexmanager/ShuffleVertexManagerBase.java +++ b/tez-runtime-library/src/main/java/org/apache/tez/dag/library/vertexmanager/ShuffleVertexManagerBase.java @@ -20,8 +20,6 @@ import com.google.common.annotations.VisibleForTesting; import org.apache.tez.common.Preconditions; -import com.google.common.base.Predicate; -import com.google.common.collect.Iterables; import com.google.common.collect.Lists; import com.google.common.collect.Maps; import com.google.common.collect.Sets; @@ -67,6 +65,7 @@ import java.util.List; import java.util.Map; import java.util.Set; +import java.util.stream.Collectors; import java.util.zip.Inflater; /** @@ -553,13 +552,9 @@ SourceVertexInfo getSourceVertexInfo(String vertextName) { } Iterable> getBipartiteInfo() { - return Iterables.filter(srcVertexInfo.entrySet(), - new Predicate>() { - public boolean apply(Map.Entry input) { - return (input.getValue().edgeProperty.getDataMovementType() == - DataMovementType.SCATTER_GATHER); - } - }); + return srcVertexInfo.entrySet().stream() + .filter(input -> (input.getValue().edgeProperty.getDataMovementType() == DataMovementType.SCATTER_GATHER)) + .collect(Collectors.toList()); } /** diff --git a/tez-tests/findbugs-exclude.xml b/tez-tests/spotbugs-exclude.xml similarity index 100% rename from tez-tests/findbugs-exclude.xml rename to tez-tests/spotbugs-exclude.xml diff --git a/tez-tools/analyzers/job-analyzer/findbugs-exclude.xml b/tez-tools/analyzers/job-analyzer/spotbugs-exclude.xml similarity index 100% rename from tez-tools/analyzers/job-analyzer/findbugs-exclude.xml rename to tez-tools/analyzers/job-analyzer/spotbugs-exclude.xml diff --git a/tez-tools/analyzers/job-analyzer/src/main/java/org/apache/tez/analyzer/plugins/SlowNodeAnalyzer.java b/tez-tools/analyzers/job-analyzer/src/main/java/org/apache/tez/analyzer/plugins/SlowNodeAnalyzer.java index 9e573c2033..7cc39e1095 100644 --- a/tez-tools/analyzers/job-analyzer/src/main/java/org/apache/tez/analyzer/plugins/SlowNodeAnalyzer.java +++ b/tez-tools/analyzers/job-analyzer/src/main/java/org/apache/tez/analyzer/plugins/SlowNodeAnalyzer.java @@ -18,8 +18,6 @@ package org.apache.tez.analyzer.plugins; -import com.google.common.base.Predicate; -import com.google.common.collect.Iterables; import com.google.common.collect.Lists; import com.google.common.collect.Multimap; @@ -40,7 +38,7 @@ import java.util.Collection; import java.util.List; - +import java.util.stream.Collectors; /** * This will provide the set of nodes participated in the DAG in descending order of task execution @@ -107,12 +105,8 @@ public void analyze(DagInfo dagInfo) throws TezException { private Iterable getFilteredTaskAttempts(Collection taskAttemptInfos, final TaskAttemptState status) { - return Iterables.filter(taskAttemptInfos, new - Predicate() { - @Override public boolean apply(TaskAttemptInfo input) { - return input.getStatus().equalsIgnoreCase(status.toString()); - } - }); + return taskAttemptInfos.stream().filter(input -> input.getStatus().equalsIgnoreCase(status.toString())) + .collect(Collectors.toList()); } private float getAvgTaskExecutionTime(Iterable taskAttemptInfos) { diff --git a/tez-tools/tez-javadoc-tools/findbugs-exclude.xml b/tez-tools/tez-javadoc-tools/spotbugs-exclude.xml similarity index 100% rename from tez-tools/tez-javadoc-tools/findbugs-exclude.xml rename to tez-tools/tez-javadoc-tools/spotbugs-exclude.xml diff --git a/tez-ui/findbugs-exclude.xml b/tez-ui/spotbugs-exclude.xml similarity index 100% rename from tez-ui/findbugs-exclude.xml rename to tez-ui/spotbugs-exclude.xml