sync

hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/client/KerberosAuthenticator.java

@@ -183,8 +183,9 @@ public void authenticate(URL url, AuthenticatedURL.Token token)
     if (!token.isSet()) {
       this.url = url;
       base64 = new Base64(0);
+      HttpURLConnection conn = null;
       try {
-        HttpURLConnection conn = token.openConnection(url, connConfigurator);
+        conn = token.openConnection(url, connConfigurator);
         conn.setRequestMethod(AUTH_HTTP_METHOD);
         conn.connect();
 
@@ -218,6 +219,10 @@ public void authenticate(URL url, AuthenticatedURL.Token token)
       } catch (AuthenticationException ex){
         throw wrapExceptionWithMessage(ex,
             "Error while authenticating with endpoint: " + url);
+      } finally {
+        if (conn != null) {
+          conn.disconnect();
+        }
       }
     }
   }

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/StorageType.java

@@ -24,6 +24,7 @@
 
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.util.StringUtils;
 
 /**
@@ -100,4 +101,20 @@ private static List<StorageType> getNonTransientTypes() {
     }
     return nonTransientTypes;
   }
+
+  // The configuration header for different StorageType.
+  public static final String CONF_KEY_HEADER =
+      "dfs.datanode.storagetype.";
+
+  /**
+   * Get the configured values for different StorageType.
+   * @param conf - absolute or fully qualified path
+   * @param t - the StorageType
+   * @param name - the sub-name of key
+   * @return the file system of the path
+   */
+  public static String getConf(Configuration conf,
+                               StorageType t, String name) {
+    return conf.get(CONF_KEY_HEADER + t.toString() + "." + name);
+  }
 }

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/ElasticByteBufferPool.java

@@ -96,6 +96,7 @@ public synchronized ByteBuffer getBuffer(boolean direct, int length) {
                       ByteBuffer.allocate(length);
     }
     tree.remove(entry.getKey());
+    entry.getValue().clear();
     return entry.getValue();
   }
 

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Client.java

@@ -18,6 +18,7 @@
 
 package org.apache.hadoop.ipc;
 
+import org.apache.hadoop.security.AccessControlException;
 import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting;
 import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
 import org.apache.hadoop.thirdparty.com.google.common.util.concurrent.ThreadFactoryBuilder;
@@ -857,7 +858,8 @@ public AuthMethod run()
               }
             } else if (UserGroupInformation.isSecurityEnabled()) {
               if (!fallbackAllowed) {
-                throw new IOException("Server asks us to fall back to SIMPLE " +
+                throw new AccessControlException(
+                    "Server asks us to fall back to SIMPLE " +
                     "auth, but this client is configured to only allow secure " +
                     "connections.");
               }

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java

@@ -2202,7 +2202,7 @@ private void doSaslReply(Message message) throws IOException {
     private void doSaslReply(Exception ioe) throws IOException {
       setupResponse(authFailedCall,
           RpcStatusProto.FATAL, RpcErrorCodeProto.FATAL_UNAUTHORIZED,
-          null, ioe.getClass().getName(), ioe.toString());
+          null, ioe.getClass().getName(), ioe.getMessage());
       sendResponse(authFailedCall);
     }
 
@@ -2597,8 +2597,7 @@ private void processOneRpc(ByteBuffer bb)
         final RpcCall call = new RpcCall(this, callId, retry);
         setupResponse(call,
             rse.getRpcStatusProto(), rse.getRpcErrorCodeProto(), null,
-            t.getClass().getName(),
-            t.getMessage() != null ? t.getMessage() : t.toString());
+            t.getClass().getName(), t.getMessage());
         sendResponse(call);
       }
     }

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetUtils.java

@@ -46,6 +46,7 @@
 
 import javax.net.SocketFactory;
 
+import org.apache.hadoop.security.AccessControlException;
 import org.apache.hadoop.thirdparty.com.google.common.cache.Cache;
 import org.apache.hadoop.thirdparty.com.google.common.cache.CacheBuilder;
 
@@ -874,6 +875,11 @@ public static IOException wrapException(final String destHost,
                 + " failed on socket exception: " + exception
                 + ";"
                 + see("SocketException"));
+      } else if (exception instanceof AccessControlException) {
+        return wrapWithMessage(exception,
+            "Call From "
+                + localHost + " to " + destHost + ":" + destPort
+                + " failed: " + exception.getMessage());
       } else {
         // 1. Return instance of same type with exception msg if Exception has a
         // String constructor.

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/web/DelegationTokenAuthenticator.java

@@ -312,8 +312,9 @@ private Map doDelegationTokenOperation(URL url,
       dt = ((DelegationTokenAuthenticatedURL.Token) token).getDelegationToken();
       ((DelegationTokenAuthenticatedURL.Token) token).setDelegationToken(null);
     }
+    HttpURLConnection conn = null;
     try {
-      HttpURLConnection conn = aUrl.openConnection(url, token);
+      conn = aUrl.openConnection(url, token);
       conn.setRequestMethod(operation.getHttpMethod());
       HttpExceptionUtils.validateResponse(conn, HttpURLConnection.HTTP_OK);
       if (hasResponse) {
@@ -339,6 +340,9 @@ private Map doDelegationTokenOperation(URL url,
       if (dt != null) {
         ((DelegationTokenAuthenticatedURL.Token) token).setDelegationToken(dt);
       }
+      if (conn != null) {
+        conn.disconnect();
+      }
     }
     return ret;
   }

hadoop-common-project/hadoop-common/src/main/native/native.vcxproj

@@ -128,10 +128,6 @@
   <ItemGroup>
     <ClCompile Include="src\org\apache\hadoop\io\compress\zlib\ZlibCompressor.c" Condition="Exists('$(ZLIB_HOME)')" />
     <ClCompile Include="src\org\apache\hadoop\io\compress\zlib\ZlibDecompressor.c" Condition="Exists('$(ZLIB_HOME)')" />
-    <ClCompile Include="src\org\apache\hadoop\io\compress\lz4\lz4.c" />
-    <ClCompile Include="src\org\apache\hadoop\io\compress\lz4\lz4hc.c" />
-    <ClCompile Include="src\org\apache\hadoop\io\compress\lz4\Lz4Compressor.c" />
-    <ClCompile Include="src\org\apache\hadoop\io\compress\lz4\Lz4Decompressor.c" />
     <ClCompile Include="src\org\apache\hadoop\io\nativeio\file_descriptor.c" />
     <ClCompile Include="src\org\apache\hadoop\io\nativeio\NativeIO.c" />
     <ClCompile Include="src\org\apache\hadoop\security\JniBasedUnixGroupsMappingWin.c" />

hadoop-common-project/hadoop-common/src/site/markdown/ClusterSetup.md

@@ -156,7 +156,7 @@ This section deals with important parameters to be specified in the given config
 | `yarn.nodemanager.remote-app-log-dir` | */logs* | HDFS directory where the application logs are moved on application completion. Need to set appropriate permissions. Only applicable if log-aggregation is enabled. |
 | `yarn.nodemanager.remote-app-log-dir-suffix` | *logs* | Suffix appended to the remote log dir. Logs will be aggregated to ${yarn.nodemanager.remote-app-log-dir}/${user}/${thisParam} Only applicable if log-aggregation is enabled. |
 | `yarn.nodemanager.aux-services` | mapreduce\_shuffle | Shuffle service that needs to be set for Map Reduce applications. |
-| `yarn.nodemanager.env-whitelist` | Environment properties to be inherited by containers from NodeManagers | For mapreduce application in addition to the default values HADOOP\_MAPRED_HOME should to be added. Property value should JAVA\_HOME,HADOOP\_COMMON\_HOME,HADOOP\_HDFS\_HOME,HADOOP\_CONF\_DIR,CLASSPATH\_PREPEND\_DISTCACHE,HADOOP\_YARN\_HOME,HADOOP\_MAPRED\_HOME |
+| `yarn.nodemanager.env-whitelist` | Environment properties to be inherited by containers from NodeManagers | For mapreduce application in addition to the default values HADOOP\_MAPRED_HOME should to be added. Property value should JAVA\_HOME,HADOOP\_COMMON\_HOME,HADOOP\_HDFS\_HOME,HADOOP\_CONF\_DIR,CLASSPATH\_PREPEND\_DISTCACHE,HADOOP\_YARN\_HOME,HADOOP\_HOME,PATH,LANG,TZ,HADOOP\_MAPRED\_HOME |
 
   * Configurations for History Server (Needs to be moved elsewhere):
 

hadoop-common-project/hadoop-common/src/site/markdown/SingleCluster.md.vm

@@ -206,7 +206,7 @@ The following instructions assume that 1. ~ 4. steps of [the above instructions]
             </property>
             <property>
                 <name>yarn.nodemanager.env-whitelist</name>
-                <value>JAVA_HOME,HADOOP_COMMON_HOME,HADOOP_HDFS_HOME,HADOOP_CONF_DIR,CLASSPATH_PREPEND_DISTCACHE,HADOOP_YARN_HOME,HADOOP_MAPRED_HOME</value>
+                <value>JAVA_HOME,HADOOP_COMMON_HOME,HADOOP_HDFS_HOME,HADOOP_CONF_DIR,CLASSPATH_PREPEND_DISTCACHE,HADOOP_YARN_HOME,HADOOP_HOME,PATH,LANG,TZ,HADOOP_MAPRED_HOME</value>
             </property>
         </configuration>
 

hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestSaslRPC.java

@@ -533,24 +533,27 @@ public void handle(Callback[] callbacks)
   }
 
   private static Pattern BadToken =
-      Pattern.compile(".*DIGEST-MD5: digest response format violation.*");
+      Pattern.compile("^" + RemoteException.class.getName() +
+          "\\("+ SaslException.class.getName() + "\\): " +
+          "DIGEST-MD5: digest response format violation.*");
   private static Pattern KrbFailed =
       Pattern.compile(".*Failed on local exception:.* " +
                       "Failed to specify server's Kerberos principal name.*");
   private static Pattern Denied(AuthMethod method) {
-      return Pattern.compile(".*RemoteException.*AccessControlException.*: "
-          + method + " authentication is not enabled.*");
+    return Pattern.compile("^" + RemoteException.class.getName() +
+        "\\(" + AccessControlException.class.getName() + "\\): "
+        + method + " authentication is not enabled.*");
   }
   private static Pattern No(AuthMethod ... method) {
     String methods = StringUtils.join(method, ",\\s*");
     return Pattern.compile(".*Failed on local exception:.* " +
         "Client cannot authenticate via:\\[" + methods + "\\].*");
   }
   private static Pattern NoTokenAuth =
-      Pattern.compile(".*IllegalArgumentException: " +
+      Pattern.compile("^" + IllegalArgumentException.class.getName() + ": " +
                       "TOKEN authentication requires a secret manager");
   private static Pattern NoFallback = 
-      Pattern.compile(".*Failed on local exception:.* " +
+      Pattern.compile("^" + AccessControlException.class.getName() + ":.* " +
           "Server asks us to fall back to SIMPLE auth, " +
           "but this client is configured to only allow secure connections.*");
 

hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/net/ServerSocketUtil.java

@@ -22,6 +22,7 @@
 import org.slf4j.LoggerFactory;
 
 import java.io.IOException;
+import java.net.InetAddress;
 import java.net.ServerSocket;
 import java.util.Random;
 
@@ -49,7 +50,8 @@ public static int getPort(int port, int retries) throws IOException {
       if (tryPort == 0) {
         continue;
       }
-      try (ServerSocket s = new ServerSocket(tryPort)) {
+      try (ServerSocket s = new ServerSocket(tryPort, 50,
+          InetAddress.getLoopbackAddress())) {
         LOG.info("Using port " + tryPort);
         return tryPort;
       } catch (IOException e) {

hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSAudit.java

@@ -191,7 +191,7 @@ private void logEvent(final OpStatus status, AuditEvent event) {
   private void op(final OpStatus opStatus, final Object op,
       final UserGroupInformation ugi, final String key, final String remoteHost,
       final String extraMsg) {
-    final String user = ugi == null ? null: ugi.getShortUserName();
+    final String user = ugi == null ? null: ugi.getUserName();
     if (!Strings.isNullOrEmpty(user) && !Strings.isNullOrEmpty(key)
         && (op != null)
         && AGGREGATE_OPS_WHITELIST.contains(op)) {

hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSAuditLogger.java

@@ -77,7 +77,7 @@ class AuditEvent {
         this.user = null;
         this.impersonator = null;
       } else {
-        this.user = ugi.getShortUserName();
+        this.user = ugi.getUserName();
         if (ugi.getAuthenticationMethod()
             == UserGroupInformation.AuthenticationMethod.PROXY) {
           this.impersonator = ugi.getRealUser().getUserName();

hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMSAudit.java

@@ -40,7 +40,6 @@
 import org.junit.Rule;
 import org.junit.Test;
 import org.junit.rules.Timeout;
-import org.mockito.Mockito;
 
 public class TestKMSAudit {
 
@@ -50,6 +49,8 @@ public class TestKMSAudit {
   private PrintStream capturedOut;
 
   private KMSAudit kmsAudit;
+  private UserGroupInformation luser =
+      UserGroupInformation.createUserForTesting("luser@REALM", new String[0]);
 
   private static class FilterOut extends FilterOutputStream {
     public FilterOut(OutputStream out) {
@@ -95,10 +96,7 @@ private String getAndResetLogOutput() {
   }
 
   @Test
-  @SuppressWarnings("checkstyle:linelength")
   public void testAggregation() throws Exception {
-    UserGroupInformation luser = Mockito.mock(UserGroupInformation.class);
-    Mockito.when(luser.getShortUserName()).thenReturn("luser");
     kmsAudit.ok(luser, KMSOp.DECRYPT_EEK, "k1", "testmsg");
     kmsAudit.ok(luser, KMSOp.DECRYPT_EEK, "k1", "testmsg");
     kmsAudit.ok(luser, KMSOp.DECRYPT_EEK, "k1", "testmsg");
@@ -120,27 +118,30 @@ public void testAggregation() throws Exception {
     kmsAudit.evictCacheForTesting();
     String out = getAndResetLogOutput();
     System.out.println(out);
-    Assert.assertTrue(
-        out.matches(
-            "OK\\[op=DECRYPT_EEK, key=k1, user=luser, accessCount=1, interval=[^m]{1,4}ms\\] testmsg"
+    boolean doesMatch = out.matches(
+        "OK\\[op=DECRYPT_EEK, key=k1, user=luser@REALM, accessCount=1, "
+            + "interval=[^m]{1,4}ms\\] testmsg"
             // Not aggregated !!
-            + "OK\\[op=DELETE_KEY, key=k1, user=luser\\] testmsg"
-            + "OK\\[op=ROLL_NEW_VERSION, key=k1, user=luser\\] testmsg"
-            + "OK\\[op=INVALIDATE_CACHE, key=k1, user=luser\\] testmsg"
+            + "OK\\[op=DELETE_KEY, key=k1, user=luser@REALM\\] testmsg"
+            + "OK\\[op=ROLL_NEW_VERSION, key=k1, user=luser@REALM\\] testmsg"
+            + "OK\\[op=INVALIDATE_CACHE, key=k1, user=luser@REALM\\] testmsg"
             // Aggregated
-            + "OK\\[op=DECRYPT_EEK, key=k1, user=luser, accessCount=6, interval=[^m]{1,4}ms\\] testmsg"
-            + "OK\\[op=DECRYPT_EEK, key=k1, user=luser, accessCount=1, interval=[^m]{1,4}ms\\] testmsg"
-            + "OK\\[op=REENCRYPT_EEK, key=k1, user=luser, accessCount=1, interval=[^m]{1,4}ms\\] testmsg"
-            + "OK\\[op=REENCRYPT_EEK, key=k1, user=luser, accessCount=3, interval=[^m]{1,4}ms\\] testmsg"
-            + "OK\\[op=REENCRYPT_EEK_BATCH, key=k1, user=luser\\] testmsg"
-            + "OK\\[op=REENCRYPT_EEK_BATCH, key=k1, user=luser\\] testmsg"));
+            + "OK\\[op=DECRYPT_EEK, key=k1, user=luser@REALM, accessCount=6, "
+            + "interval=[^m]{1,4}ms\\] testmsg"
+            + "OK\\[op=DECRYPT_EEK, key=k1, user=luser@REALM, accessCount=1, "
+            + "interval=[^m]{1,4}ms\\] testmsg"
+            + "OK\\[op=REENCRYPT_EEK, key=k1, user=luser@REALM, "
+            + "accessCount=1, interval=[^m]{1,4}ms\\] testmsg"
+            + "OK\\[op=REENCRYPT_EEK, key=k1, user=luser@REALM, "
+            + "accessCount=3, interval=[^m]{1,4}ms\\] testmsg"
+            + "OK\\[op=REENCRYPT_EEK_BATCH, key=k1, user=luser@REALM\\] testmsg"
+            + "OK\\[op=REENCRYPT_EEK_BATCH, key=k1, user=luser@REALM\\] "
+            + "testmsg");
+    Assert.assertTrue(doesMatch);
   }
 
   @Test
-  @SuppressWarnings("checkstyle:linelength")
   public void testAggregationUnauth() throws Exception {
-    UserGroupInformation luser = Mockito.mock(UserGroupInformation.class);
-    Mockito.when(luser.getShortUserName()).thenReturn("luser");
     kmsAudit.unauthorized(luser, KMSOp.GENERATE_EEK, "k2");
     kmsAudit.evictCacheForTesting();
     kmsAudit.ok(luser, KMSOp.GENERATE_EEK, "k3", "testmsg");
@@ -159,25 +160,29 @@ public void testAggregationUnauth() throws Exception {
     // The UNAUTHORIZED will trigger cache invalidation, which then triggers
     // the aggregated OK (accessCount=5). But the order of the UNAUTHORIZED and
     // the aggregated OK is arbitrary - no correctness concerns, but flaky here.
-    Assert.assertTrue(out.matches(
-        "UNAUTHORIZED\\[op=GENERATE_EEK, key=k2, user=luser\\] "
-            + "OK\\[op=GENERATE_EEK, key=k3, user=luser, accessCount=1, interval=[^m]{1,4}ms\\] testmsg"
-            + "OK\\[op=GENERATE_EEK, key=k3, user=luser, accessCount=5, interval=[^m]{1,4}ms\\] testmsg"
-            + "UNAUTHORIZED\\[op=GENERATE_EEK, key=k3, user=luser\\] "
-            + "OK\\[op=GENERATE_EEK, key=k3, user=luser, accessCount=1, interval=[^m]{1,4}ms\\] testmsg")
-        || out.matches(
-        "UNAUTHORIZED\\[op=GENERATE_EEK, key=k2, user=luser\\] "
-            + "OK\\[op=GENERATE_EEK, key=k3, user=luser, accessCount=1, interval=[^m]{1,4}ms\\] testmsg"
-            + "UNAUTHORIZED\\[op=GENERATE_EEK, key=k3, user=luser\\] "
-            + "OK\\[op=GENERATE_EEK, key=k3, user=luser, accessCount=5, interval=[^m]{1,4}ms\\] testmsg"
-            + "OK\\[op=GENERATE_EEK, key=k3, user=luser, accessCount=1, interval=[^m]{1,4}ms\\] testmsg"));
+    boolean doesMatch = out.matches(
+        "UNAUTHORIZED\\[op=GENERATE_EEK, key=k2, user=luser@REALM\\] "
+            + "OK\\[op=GENERATE_EEK, key=k3, user=luser@REALM, accessCount=1,"
+            + " interval=[^m]{1,4}ms\\] testmsg"
+            + "OK\\[op=GENERATE_EEK, key=k3, user=luser@REALM, accessCount=5,"
+            + " interval=[^m]{1,4}ms\\] testmsg"
+            + "UNAUTHORIZED\\[op=GENERATE_EEK, key=k3, user=luser@REALM\\] "
+            + "OK\\[op=GENERATE_EEK, key=k3, user=luser@REALM, accessCount=1,"
+            + " interval=[^m]{1,4}ms\\] testmsg");
+    doesMatch = doesMatch || out.matches(
+        "UNAUTHORIZED\\[op=GENERATE_EEK, key=k2, user=luser@REALM\\] "
+            + "OK\\[op=GENERATE_EEK, key=k3, user=luser@REALM, accessCount=1,"
+            + " interval=[^m]{1,4}ms\\] testmsg"
+            + "UNAUTHORIZED\\[op=GENERATE_EEK, key=k3, user=luser@REALM\\] "
+            + "OK\\[op=GENERATE_EEK, key=k3, user=luser@REALM, accessCount=5,"
+            + " interval=[^m]{1,4}ms\\] testmsg"
+            + "OK\\[op=GENERATE_EEK, key=k3, user=luser@REALM, accessCount=1,"
+            + " interval=[^m]{1,4}ms\\] testmsg");
+    Assert.assertTrue(doesMatch);
   }
 
   @Test
-  @SuppressWarnings("checkstyle:linelength")
   public void testAuditLogFormat() throws Exception {
-    UserGroupInformation luser = Mockito.mock(UserGroupInformation.class);
-    Mockito.when(luser.getShortUserName()).thenReturn("luser");
     kmsAudit.ok(luser, KMSOp.GENERATE_EEK, "k4", "testmsg");
     kmsAudit.ok(luser, KMSOp.GENERATE_EEK, "testmsg");
     kmsAudit.evictCacheForTesting();
@@ -187,12 +192,15 @@ public void testAuditLogFormat() throws Exception {
     String out = getAndResetLogOutput();
     System.out.println(out);
     Assert.assertTrue(out.matches(
-        "OK\\[op=GENERATE_EEK, key=k4, user=luser, accessCount=1, interval=[^m]{1,4}ms\\] testmsg"
-            + "OK\\[op=GENERATE_EEK, user=luser\\] testmsg"
-            + "OK\\[op=GENERATE_EEK, key=k4, user=luser, accessCount=1, interval=[^m]{1,4}ms\\] testmsg"
-            + "UNAUTHORIZED\\[op=DECRYPT_EEK, key=k4, user=luser\\] "
-            + "ERROR\\[user=luser\\] Method:'method' Exception:'testmsg'"
-            + "UNAUTHENTICATED RemoteHost:remotehost Method:method URL:url ErrorMsg:'testmsg'"));
+        "OK\\[op=GENERATE_EEK, key=k4, user=luser@REALM, accessCount=1, "
+            + "interval=[^m]{1,4}ms\\] testmsg"
+            + "OK\\[op=GENERATE_EEK, user=luser@REALM\\] testmsg"
+            + "OK\\[op=GENERATE_EEK, key=k4, user=luser@REALM, accessCount=1,"
+            + " interval=[^m]{1,4}ms\\] testmsg"
+            + "UNAUTHORIZED\\[op=DECRYPT_EEK, key=k4, user=luser@REALM\\] "
+            + "ERROR\\[user=luser@REALM\\] Method:'method' Exception:'testmsg'"
+            + "UNAUTHENTICATED RemoteHost:remotehost Method:method URL:url "
+            + "ErrorMsg:'testmsg'"));
   }
 
   @SuppressWarnings("unchecked")

hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/hdfs.proto

@@ -198,7 +198,7 @@ message StorageTypeQuotaInfosProto {
 }
 
 message StorageTypeQuotaInfoProto {
-  required StorageTypeProto type = 1;
+  optional StorageTypeProto type = 1 [default = DISK];
   required uint64 quota = 2;
   required uint64 consumed = 3;
 }