Merging from apache trunk

BUILDING.txt

@@ -4,7 +4,7 @@ Build instructions for Hadoop
 Requirements:
 
 * Unix System
-* JDK 1.6+
+* JDK 1.7+
 * Maven 3.0 or later
 * Findbugs 1.3.9 (if running findbugs)
 * ProtocolBuffer 2.5.0
@@ -204,15 +204,19 @@ Building on Windows
 Requirements:
 
 * Windows System
-* JDK 1.6+
+* JDK 1.7+
 * Maven 3.0 or later
 * Findbugs 1.3.9 (if running findbugs)
 * ProtocolBuffer 2.5.0
 * CMake 2.6 or newer
 * Windows SDK or Visual Studio 2010 Professional
-* Unix command-line tools from GnuWin32 or Cygwin: sh, mkdir, rm, cp, tar, gzip
 * zlib headers (if building native code bindings for zlib)
 * Internet connection for first build (to fetch all Maven and Hadoop dependencies)
+* Unix command-line tools from GnuWin32: sh, mkdir, rm, cp, tar, gzip. These
+  tools must be present on your PATH.
+
+Unix command-line tools are also included with the Windows Git package which
+can be downloaded from http://git-scm.com/download/win.
 
 If using Visual Studio, it must be Visual Studio 2010 Professional (not 2012).
 Do not use Visual Studio Express.  It does not support compiling for 64-bit,
@@ -221,6 +225,8 @@ download here:
 
 http://www.microsoft.com/en-us/download/details.aspx?id=8279
 
+Cygwin is neither required nor supported.
+
 ----------------------------------------------------------------------------------
 Building:
 

dev-support/test-patch.sh

@@ -857,74 +857,6 @@ findModules () {
   rm $TMP_MODULES
   echo $CHANGED_MODULES
 }
-###############################################################################
-### Run the test-contrib target
-runContribTests () {
-  echo ""
-  echo ""
-  echo "======================================================================"
-  echo "======================================================================"
-  echo "    Running contrib tests."
-  echo "======================================================================"
-  echo "======================================================================"
-  echo ""
-  echo ""
-
-  if [[ `$GREP -c 'test-contrib' build.xml` == 0 ]] ; then
-    echo "No contrib tests in this project."
-    return 0
-  fi
-
-  ### Kill any rogue build processes from the last attempt
-  $PS auxwww | $GREP ${PROJECT_NAME}PatchProcess | $AWK '{print $2}' | /usr/bin/xargs -t -I {} /bin/kill -9 {} > /dev/null
-
-  #echo "$ANT_HOME/bin/ant -Dversion="${VERSION}" $ECLIPSE_PROPERTY -DHadoopPatchProcess= -Dtest.junit.output.format=xml -Dtest.output=no test-contrib"
-  #$ANT_HOME/bin/ant -Dversion="${VERSION}" $ECLIPSE_PROPERTY -DHadoopPatchProcess= -Dtest.junit.output.format=xml -Dtest.output=no test-contrib
-  echo "NOP"
-  if [[ $? != 0 ]] ; then
-    JIRA_COMMENT="$JIRA_COMMENT
-
-    {color:red}-1 contrib tests{color}.  The patch failed contrib unit tests."
-    return 1
-  fi
-  JIRA_COMMENT="$JIRA_COMMENT
-
-    {color:green}+1 contrib tests{color}.  The patch passed contrib unit tests."
-  return 0
-}
-
-###############################################################################
-### Run the inject-system-faults target
-checkInjectSystemFaults () {
-  echo ""
-  echo ""
-  echo "======================================================================"
-  echo "======================================================================"
-  echo "    Checking the integrity of system test framework code."
-  echo "======================================================================"
-  echo "======================================================================"
-  echo ""
-  echo ""
-
-  ### Kill any rogue build processes from the last attempt
-  $PS auxwww | $GREP ${PROJECT_NAME}PatchProcess | $AWK '{print $2}' | /usr/bin/xargs -t -I {} /bin/kill -9 {} > /dev/null
-
-  #echo "$ANT_HOME/bin/ant -Dversion="${VERSION}" -DHadoopPatchProcess= -Dtest.junit.output.format=xml -Dtest.output=no -Dcompile.c++=yes -Dforrest.home=$FORREST_HOME inject-system-faults"
-  #$ANT_HOME/bin/ant -Dversion="${VERSION}" -DHadoopPatchProcess= -Dtest.junit.output.format=xml -Dtest.output=no -Dcompile.c++=yes -Dforrest.home=$FORREST_HOME inject-system-faults
-  echo "NOP"
-  return 0
-  if [[ $? != 0 ]] ; then
-    JIRA_COMMENT="$JIRA_COMMENT
-
-    {color:red}-1 system test framework{color}.  The patch failed system test framework compile."
-    return 1
-  fi
-  JIRA_COMMENT="$JIRA_COMMENT
-
-    {color:green}+1 system test framework{color}.  The patch passed system test framework compile."
-  return 0
-}
-
 ###############################################################################
 ### Submit a comment to the defect's Jira
 submitJiraComment () {
@@ -1059,10 +991,7 @@ checkReleaseAuditWarnings
 if [[ $JENKINS == "true" || $RUN_TESTS == "true" ]] ; then
   runTests
   (( RESULT = RESULT + $? ))
-  runContribTests
-  (( RESULT = RESULT + $? ))
 fi
-checkInjectSystemFaults
 (( RESULT = RESULT + $? ))
 JIRA_COMMENT_FOOTER="Test results: $BUILD_URL/testReport/
 $JIRA_COMMENT_FOOTER"

hadoop-assemblies/pom.xml

@@ -45,10 +45,10 @@
         <configuration>
           <rules>
             <requireMavenVersion>
-              <version>[3.0.0,)</version>
+              <version>${enforced.maven.version}</version>
             </requireMavenVersion>
             <requireJavaVersion>
-              <version>1.6</version>
+              <version>${enforced.java.version}</version>
             </requireJavaVersion>
           </rules>
         </configuration>

hadoop-client/pom.xml

@@ -13,7 +13,7 @@
  limitations under the License. See accompanying LICENSE file.
 -->
 <project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
-         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
   <modelVersion>4.0.0</modelVersion>
 <parent>
    <groupId>org.apache.hadoop</groupId>

hadoop-common-project/hadoop-annotations/pom.xml

@@ -39,23 +39,6 @@
   </dependencies>
 
   <profiles>
-    <profile>
-      <id>os.linux</id>
-      <activation>
-        <os>
-          <family>!Mac</family>
-        </os>
-      </activation>
-      <dependencies>
-        <dependency>
-          <groupId>jdk.tools</groupId>
-          <artifactId>jdk.tools</artifactId>
-          <version>1.6</version>
-          <scope>system</scope>
-          <systemPath>${java.home}/../lib/tools.jar</systemPath>
-        </dependency>
-      </dependencies>
-    </profile>
     <profile>
       <id>jdk1.7</id>
       <activation>

hadoop-common-project/hadoop-auth-examples/src/main/java/org/apache/hadoop/security/authentication/examples/WhoClient.java

@@ -19,6 +19,7 @@
 import java.io.InputStreamReader;
 import java.net.HttpURLConnection;
 import java.net.URL;
+import java.nio.charset.Charset;
 
 /**
  * Example that uses <code>AuthenticatedURL</code>.
@@ -39,7 +40,9 @@ public static void main(String[] args) {
       System.out.println("Status code: " + conn.getResponseCode() + " " + conn.getResponseMessage());
       System.out.println();
       if (conn.getResponseCode() == HttpURLConnection.HTTP_OK) {
-        BufferedReader reader = new BufferedReader(new InputStreamReader(conn.getInputStream()));
+        BufferedReader reader = new BufferedReader(
+            new InputStreamReader(
+                conn.getInputStream(), Charset.forName("UTF-8")));
         String line = reader.readLine();
         while (line != null) {
           System.out.println(line);

hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/client/AuthenticatedURL.java

@@ -24,19 +24,18 @@
 /**
  * The {@link AuthenticatedURL} class enables the use of the JDK {@link URL} class
  * against HTTP endpoints protected with the {@link AuthenticationFilter}.
- * <p/>
+ * <p>
  * The authentication mechanisms supported by default are Hadoop Simple  authentication
  * (also known as pseudo authentication) and Kerberos SPNEGO authentication.
- * <p/>
+ * <p>
  * Additional authentication mechanisms can be supported via {@link Authenticator} implementations.
- * <p/>
+ * <p>
  * The default {@link Authenticator} is the {@link KerberosAuthenticator} class which supports
  * automatic fallback from Kerberos SPNEGO to Hadoop Simple authentication.
- * <p/>
+ * <p>
  * <code>AuthenticatedURL</code> instances are not thread-safe.
- * <p/>
+ * <p>
  * The usage pattern of the {@link AuthenticatedURL} is:
- * <p/>
  * <pre>
  *
  * // establishing an initial connection
@@ -240,7 +239,7 @@ public static void injectToken(HttpURLConnection conn, Token token) {
 
   /**
    * Helper method that extracts an authentication token received from a connection.
-   * <p/>
+   * <p>
    * This method is used by {@link Authenticator} implementations.
    *
    * @param conn connection to extract the authentication token from.

hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/client/Authenticator.java

@@ -19,7 +19,7 @@
 
 /**
  * Interface for client authentication mechanisms.
- * <p/>
+ * <p>
  * Implementations are use-once instances, they don't need to be thread safe.
  */
 public interface Authenticator {

hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/client/KerberosAuthenticator.java

@@ -23,6 +23,8 @@
 import org.slf4j.LoggerFactory;
 
 import javax.security.auth.Subject;
+import javax.security.auth.kerberos.KerberosKey;
+import javax.security.auth.kerberos.KerberosTicket;
 import javax.security.auth.login.AppConfigurationEntry;
 import javax.security.auth.login.Configuration;
 import javax.security.auth.login.LoginContext;
@@ -41,9 +43,9 @@
 
 /**
  * The {@link KerberosAuthenticator} implements the Kerberos SPNEGO authentication sequence.
- * <p/>
+ * <p>
  * It uses the default principal for the Kerberos cache (normally set via kinit).
- * <p/>
+ * <p>
  * It falls back to the {@link PseudoAuthenticator} if the HTTP endpoint does not trigger an SPNEGO authentication
  * sequence.
  */
@@ -160,9 +162,9 @@ public void setConnectionConfigurator(ConnectionConfigurator configurator) {
 
   /**
    * Performs SPNEGO authentication against the specified URL.
-   * <p/>
+   * <p>
    * If a token is given it does a NOP and returns the given token.
-   * <p/>
+   * <p>
    * If no token is given, it will perform the SPNEGO authentication sequence using an
    * HTTP <code>OPTIONS</code> request.
    *
@@ -209,7 +211,7 @@ public void authenticate(URL url, AuthenticatedURL.Token token)
 
   /**
    * If the specified URL does not support SPNEGO authentication, a fallback {@link Authenticator} will be used.
-   * <p/>
+   * <p>
    * This implementation returns a {@link PseudoAuthenticator}.
    *
    * @return the fallback {@link Authenticator}.
@@ -247,7 +249,9 @@ private void doSpnegoSequence(AuthenticatedURL.Token token) throws IOException,
     try {
       AccessControlContext context = AccessController.getContext();
       Subject subject = Subject.getSubject(context);
-      if (subject == null) {
+      if (subject == null
+          || (subject.getPrivateCredentials(KerberosKey.class).isEmpty()
+              && subject.getPrivateCredentials(KerberosTicket.class).isEmpty())) {
         LOG.debug("No subject in context, logging in");
         subject = new Subject();
         LoginContext login = new LoginContext("", subject,

hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/client/PseudoAuthenticator.java

@@ -20,7 +20,7 @@
 /**
  * The {@link PseudoAuthenticator} implementation provides an authentication equivalent to Hadoop's
  * Simple authentication, it trusts the value of the 'user.name' Java System property.
- * <p/>
+ * <p>
  * The 'user.name' value is propagated using an additional query string parameter {@link #USER_NAME} ('user.name').
  */
 public class PseudoAuthenticator implements Authenticator {
@@ -47,13 +47,13 @@ public void setConnectionConfigurator(ConnectionConfigurator configurator) {
 
   /**
    * Performs simple authentication against the specified URL.
-   * <p/>
+   * <p>
    * If a token is given it does a NOP and returns the given token.
-   * <p/>
+   * <p>
    * If no token is given, it will perform an HTTP <code>OPTIONS</code> request injecting an additional
    * parameter {@link #USER_NAME} in the query string with the value returned by the {@link #getUserName()}
    * method.
-   * <p/>
+   * <p>
    * If the response is successful it will update the authentication token.
    *
    * @param url the URl to authenticate against.
@@ -79,7 +79,7 @@ public void authenticate(URL url, AuthenticatedURL.Token token) throws IOExcepti
 
   /**
    * Returns the current user name.
-   * <p/>
+   * <p>
    * This implementation returns the value of the Java system property 'user.name'
    *
    * @return the current user name.

hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/server/AltKerberosAuthenticationHandler.java

@@ -28,7 +28,6 @@
  * to allow a developer to implement their own custom authentication for browser
  * access.  The alternateAuthenticate method will be called whenever a request
  * comes from a browser.
- * <p/>
  */
 public abstract class AltKerberosAuthenticationHandler
                         extends KerberosAuthenticationHandler {
@@ -52,7 +51,6 @@ public abstract class AltKerberosAuthenticationHandler
   /**
    * Returns the authentication type of the authentication handler,
    * 'alt-kerberos'.
-   * <p/>
    *
    * @return the authentication type of the authentication handler,
    * 'alt-kerberos'.
@@ -80,7 +78,6 @@ public void init(Properties config) throws ServletException {
    * completed successfully (in the case of Java access) and only after the
    * custom authentication implemented by the subclass in alternateAuthenticate
    * has completed successfully (in the case of browser access).
-   * <p/>
    *
    * @param request the HTTP client request.
    * @param response the HTTP client response.
@@ -109,7 +106,7 @@ public AuthenticationToken authenticate(HttpServletRequest request,
    * refers to a browser.  If its not a browser, then Kerberos authentication
    * will be used; if it is a browser, alternateAuthenticate from the subclass
    * will be used.
-   * <p/>
+   * <p>
    * A User-Agent String is considered to be a browser if it does not contain
    * any of the values from alt-kerberos.non-browser.user-agents; the default
    * behavior is to consider everything a browser unless it contains one of: