* MiniKdc sets 2 System properties when started and un-sets them when stopped:
*
* - java.security.krb5.conf: set to the MiniKDC real/host/port
@@ -92,7 +92,7 @@
* For example, running testcases in parallel that start a KDC each. To
* accomplish this a single MiniKdc should be used for all testcases running
* in parallel.
- *
+ *
* MiniKdc default configuration values are:
*
* - org.name=EXAMPLE (used to create the REALM)
@@ -106,7 +106,6 @@
* - debug=false
*
* The generated krb5.conf forces TCP connections.
- *
*/
public class MiniKdc {
@@ -218,7 +217,7 @@ public void run() {
/**
* Convenience method that returns MiniKdc default configuration.
- *
+ *
* The returned configuration is a copy, it can be customized before using
* it to create a MiniKdc.
* @return a MiniKdc default configuration.
@@ -484,7 +483,6 @@ private void initKDCServer() throws Exception {
/**
* Stops the MiniKdc
- * @throws Exception
*/
public synchronized void stop() {
if (kdc != null) {
diff --git a/hadoop-common-project/hadoop-minikdc/src/test/java/org/apache/hadoop/minikdc/TestMiniKdc.java b/hadoop-common-project/hadoop-minikdc/src/test/java/org/apache/hadoop/minikdc/TestMiniKdc.java
index c052bb1425afa..fac7f0fbd0d7e 100644
--- a/hadoop-common-project/hadoop-minikdc/src/test/java/org/apache/hadoop/minikdc/TestMiniKdc.java
+++ b/hadoop-common-project/hadoop-minikdc/src/test/java/org/apache/hadoop/minikdc/TestMiniKdc.java
@@ -37,7 +37,8 @@
import java.util.Arrays;
public class TestMiniKdc extends KerberosSecurityTestcase {
-
+ private static final boolean IBM_JAVA = System.getProperty("java.vendor")
+ .contains("IBM");
@Test
public void testMiniKdcStart() {
MiniKdc kdc = getKdc();
@@ -94,15 +95,20 @@ private static String getKrb5LoginModuleName() {
@Override
public AppConfigurationEntry[] getAppConfigurationEntry(String name) {
Map options = new HashMap();
- options.put("keyTab", keytab);
options.put("principal", principal);
- options.put("useKeyTab", "true");
- options.put("storeKey", "true");
- options.put("doNotPrompt", "true");
- options.put("useTicketCache", "true");
- options.put("renewTGT", "true");
options.put("refreshKrb5Config", "true");
- options.put("isInitiator", Boolean.toString(isInitiator));
+ if (IBM_JAVA) {
+ options.put("useKeytab", keytab);
+ options.put("credsType", "both");
+ } else {
+ options.put("keyTab", keytab);
+ options.put("useKeyTab", "true");
+ options.put("storeKey", "true");
+ options.put("doNotPrompt", "true");
+ options.put("useTicketCache", "true");
+ options.put("renewTGT", "true");
+ options.put("isInitiator", Boolean.toString(isInitiator));
+ }
String ticketCache = System.getenv("KRB5CCNAME");
if (ticketCache != null) {
options.put("ticketCache", ticketCache);
diff --git a/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/mount/MountResponse.java b/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/mount/MountResponse.java
index 5e101c1de8973..889d45a444f81 100644
--- a/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/mount/MountResponse.java
+++ b/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/mount/MountResponse.java
@@ -19,6 +19,7 @@
import java.util.List;
+import org.apache.commons.io.Charsets;
import org.apache.hadoop.nfs.NfsExports;
import org.apache.hadoop.oncrpc.RpcAcceptedReply;
import org.apache.hadoop.oncrpc.XDR;
@@ -76,7 +77,7 @@ public static XDR writeExportList(XDR xdr, int xid, List exports,
if (hostGroups.length > 0) {
for (int j = 0; j < hostGroups.length; j++) {
xdr.writeBoolean(true); // Value follows - yes
- xdr.writeVariableOpaque(hostGroups[j].getBytes());
+ xdr.writeVariableOpaque(hostGroups[j].getBytes(Charsets.UTF_8));
}
}
xdr.writeBoolean(false); // Value follows - no more group
diff --git a/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/nfs/nfs3/FileHandle.java b/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/nfs/nfs3/FileHandle.java
index 6bc4686157697..415b459f68ffd 100644
--- a/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/nfs/nfs3/FileHandle.java
+++ b/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/nfs/nfs3/FileHandle.java
@@ -22,6 +22,7 @@
import java.security.NoSuchAlgorithmException;
import java.util.Arrays;
+import org.apache.commons.io.Charsets;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.oncrpc.XDR;
@@ -72,10 +73,8 @@ public FileHandle(String s) {
return;
}
- byte[] in = s.getBytes();
- for (int i = 0; i < in.length; i++) {
- digest.update(in[i]);
- }
+ byte[] in = s.getBytes(Charsets.UTF_8);
+ digest.update(in);
byte[] digestbytes = digest.digest();
for (int i = 0; i < 16; i++) {
diff --git a/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/nfs/nfs3/request/CREATE3Request.java b/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/nfs/nfs3/request/CREATE3Request.java
index 473d527646310..e75ce15b52a39 100644
--- a/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/nfs/nfs3/request/CREATE3Request.java
+++ b/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/nfs/nfs3/request/CREATE3Request.java
@@ -19,6 +19,7 @@
import java.io.IOException;
+import org.apache.commons.io.Charsets;
import org.apache.hadoop.nfs.nfs3.FileHandle;
import org.apache.hadoop.nfs.nfs3.Nfs3Constant;
import org.apache.hadoop.oncrpc.XDR;
@@ -78,7 +79,7 @@ public long getVerf() {
public void serialize(XDR xdr) {
handle.serialize(xdr);
xdr.writeInt(name.length());
- xdr.writeFixedOpaque(name.getBytes(), name.length());
+ xdr.writeFixedOpaque(name.getBytes(Charsets.UTF_8), name.length());
xdr.writeInt(mode);
objAttr.serialize(xdr);
}
diff --git a/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/nfs/nfs3/request/LINK3Request.java b/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/nfs/nfs3/request/LINK3Request.java
index 2e959f59f90d3..1dcb85d3d46ef 100644
--- a/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/nfs/nfs3/request/LINK3Request.java
+++ b/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/nfs/nfs3/request/LINK3Request.java
@@ -19,6 +19,7 @@
import java.io.IOException;
+import org.apache.commons.io.Charsets;
import org.apache.hadoop.nfs.nfs3.FileHandle;
import org.apache.hadoop.oncrpc.XDR;
@@ -56,6 +57,6 @@ public void serialize(XDR xdr) {
handle.serialize(xdr);
fromDirHandle.serialize(xdr);
xdr.writeInt(fromName.length());
- xdr.writeFixedOpaque(fromName.getBytes(), fromName.length());
+ xdr.writeFixedOpaque(fromName.getBytes(Charsets.UTF_8), fromName.length());
}
}
diff --git a/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/nfs/nfs3/request/LOOKUP3Request.java b/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/nfs/nfs3/request/LOOKUP3Request.java
index 4661821a68b4c..0435483e83781 100644
--- a/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/nfs/nfs3/request/LOOKUP3Request.java
+++ b/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/nfs/nfs3/request/LOOKUP3Request.java
@@ -19,6 +19,7 @@
import java.io.IOException;
+import org.apache.commons.io.Charsets;
import org.apache.hadoop.nfs.nfs3.FileHandle;
import org.apache.hadoop.oncrpc.XDR;
@@ -53,7 +54,7 @@ public void setName(String name) {
@VisibleForTesting
public void serialize(XDR xdr) {
handle.serialize(xdr);
- xdr.writeInt(name.getBytes().length);
- xdr.writeFixedOpaque(name.getBytes());
+ xdr.writeInt(name.getBytes(Charsets.UTF_8).length);
+ xdr.writeFixedOpaque(name.getBytes(Charsets.UTF_8));
}
}
\ No newline at end of file
diff --git a/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/nfs/nfs3/request/MKDIR3Request.java b/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/nfs/nfs3/request/MKDIR3Request.java
index b3ef828a7ec3f..bba26eeb3015c 100644
--- a/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/nfs/nfs3/request/MKDIR3Request.java
+++ b/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/nfs/nfs3/request/MKDIR3Request.java
@@ -19,6 +19,7 @@
import java.io.IOException;
+import org.apache.commons.io.Charsets;
import org.apache.hadoop.nfs.nfs3.FileHandle;
import org.apache.hadoop.oncrpc.XDR;
@@ -54,8 +55,8 @@ public SetAttr3 getObjAttr() {
@Override
public void serialize(XDR xdr) {
handle.serialize(xdr);
- xdr.writeInt(name.getBytes().length);
- xdr.writeFixedOpaque(name.getBytes());
+ xdr.writeInt(name.getBytes(Charsets.UTF_8).length);
+ xdr.writeFixedOpaque(name.getBytes(Charsets.UTF_8));
objAttr.serialize(xdr);
}
}
\ No newline at end of file
diff --git a/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/nfs/nfs3/request/MKNOD3Request.java b/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/nfs/nfs3/request/MKNOD3Request.java
index 4a13f879ea3c8..0659dd10c046e 100644
--- a/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/nfs/nfs3/request/MKNOD3Request.java
+++ b/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/nfs/nfs3/request/MKNOD3Request.java
@@ -19,6 +19,7 @@
import java.io.IOException;
+import org.apache.commons.io.Charsets;
import org.apache.hadoop.nfs.NfsFileType;
import org.apache.hadoop.nfs.nfs3.FileHandle;
import org.apache.hadoop.nfs.nfs3.Nfs3FileAttributes.Specdata3;
@@ -79,7 +80,7 @@ public Specdata3 getSpec() {
public void serialize(XDR xdr) {
handle.serialize(xdr);
xdr.writeInt(name.length());
- xdr.writeFixedOpaque(name.getBytes(), name.length());
+ xdr.writeFixedOpaque(name.getBytes(Charsets.UTF_8), name.length());
objAttr.serialize(xdr);
if (spec != null) {
xdr.writeInt(spec.getSpecdata1());
diff --git a/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/nfs/nfs3/request/REMOVE3Request.java b/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/nfs/nfs3/request/REMOVE3Request.java
index ffd47b0e5dcf0..9ad156d90ea12 100644
--- a/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/nfs/nfs3/request/REMOVE3Request.java
+++ b/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/nfs/nfs3/request/REMOVE3Request.java
@@ -19,6 +19,7 @@
import java.io.IOException;
+import org.apache.commons.io.Charsets;
import org.apache.hadoop.nfs.nfs3.FileHandle;
import org.apache.hadoop.oncrpc.XDR;
@@ -46,7 +47,7 @@ public String getName() {
@Override
public void serialize(XDR xdr) {
handle.serialize(xdr);
- xdr.writeInt(name.getBytes().length);
- xdr.writeFixedOpaque(name.getBytes());
+ xdr.writeInt(name.getBytes(Charsets.UTF_8).length);
+ xdr.writeFixedOpaque(name.getBytes(Charsets.UTF_8));
}
}
\ No newline at end of file
diff --git a/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/nfs/nfs3/request/RENAME3Request.java b/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/nfs/nfs3/request/RENAME3Request.java
index 5144e8a4910c1..c54a8e2da2d84 100644
--- a/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/nfs/nfs3/request/RENAME3Request.java
+++ b/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/nfs/nfs3/request/RENAME3Request.java
@@ -19,6 +19,7 @@
import java.io.IOException;
+import org.apache.commons.io.Charsets;
import org.apache.hadoop.nfs.nfs3.FileHandle;
import org.apache.hadoop.oncrpc.XDR;
@@ -66,10 +67,10 @@ public String getToName() {
@Override
public void serialize(XDR xdr) {
fromDirHandle.serialize(xdr);
- xdr.writeInt(fromName.getBytes().length);
- xdr.writeFixedOpaque(fromName.getBytes());
+ xdr.writeInt(fromName.getBytes(Charsets.UTF_8).length);
+ xdr.writeFixedOpaque(fromName.getBytes(Charsets.UTF_8));
toDirHandle.serialize(xdr);
- xdr.writeInt(toName.getBytes().length);
- xdr.writeFixedOpaque(toName.getBytes());
+ xdr.writeInt(toName.getBytes(Charsets.UTF_8).length);
+ xdr.writeFixedOpaque(toName.getBytes(Charsets.UTF_8));
}
}
\ No newline at end of file
diff --git a/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/nfs/nfs3/request/RMDIR3Request.java b/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/nfs/nfs3/request/RMDIR3Request.java
index e9977fa5488ff..6ae0de89fbf09 100644
--- a/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/nfs/nfs3/request/RMDIR3Request.java
+++ b/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/nfs/nfs3/request/RMDIR3Request.java
@@ -19,6 +19,7 @@
import java.io.IOException;
+import org.apache.commons.io.Charsets;
import org.apache.hadoop.nfs.nfs3.FileHandle;
import org.apache.hadoop.oncrpc.XDR;
@@ -46,7 +47,7 @@ public String getName() {
@Override
public void serialize(XDR xdr) {
handle.serialize(xdr);
- xdr.writeInt(name.getBytes().length);
- xdr.writeFixedOpaque(name.getBytes());
+ xdr.writeInt(name.getBytes(Charsets.UTF_8).length);
+ xdr.writeFixedOpaque(name.getBytes(Charsets.UTF_8));
}
}
\ No newline at end of file
diff --git a/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/nfs/nfs3/request/SYMLINK3Request.java b/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/nfs/nfs3/request/SYMLINK3Request.java
index 288079449dc69..59188e2b50e71 100644
--- a/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/nfs/nfs3/request/SYMLINK3Request.java
+++ b/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/nfs/nfs3/request/SYMLINK3Request.java
@@ -19,6 +19,7 @@
import java.io.IOException;
+import org.apache.commons.io.Charsets;
import org.apache.hadoop.nfs.nfs3.FileHandle;
import org.apache.hadoop.oncrpc.XDR;
@@ -62,10 +63,10 @@ public String getSymData() {
@Override
public void serialize(XDR xdr) {
handle.serialize(xdr);
- xdr.writeInt(name.getBytes().length);
- xdr.writeFixedOpaque(name.getBytes());
+ xdr.writeInt(name.getBytes(Charsets.UTF_8).length);
+ xdr.writeFixedOpaque(name.getBytes(Charsets.UTF_8));
symAttr.serialize(xdr);
- xdr.writeInt(symData.getBytes().length);
- xdr.writeFixedOpaque(symData.getBytes());
+ xdr.writeInt(symData.getBytes(Charsets.UTF_8).length);
+ xdr.writeFixedOpaque(symData.getBytes(Charsets.UTF_8));
}
}
\ No newline at end of file
diff --git a/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/oncrpc/XDR.java b/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/oncrpc/XDR.java
index 2fdabe2fda789..474559ea92fcb 100644
--- a/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/oncrpc/XDR.java
+++ b/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/oncrpc/XDR.java
@@ -19,6 +19,7 @@
import java.nio.ByteBuffer;
+import org.apache.commons.io.Charsets;
import org.jboss.netty.buffer.ChannelBuffer;
import org.jboss.netty.buffer.ChannelBuffers;
@@ -165,11 +166,11 @@ public void writeVariableOpaque(byte[] src) {
}
public String readString() {
- return new String(readVariableOpaque());
+ return new String(readVariableOpaque(), Charsets.UTF_8);
}
public void writeString(String s) {
- writeVariableOpaque(s.getBytes());
+ writeVariableOpaque(s.getBytes(Charsets.UTF_8));
}
private void writePadding() {
diff --git a/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/oncrpc/security/CredentialsSys.java b/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/oncrpc/security/CredentialsSys.java
index 997ce35c60d09..d13c3615b2c98 100644
--- a/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/oncrpc/security/CredentialsSys.java
+++ b/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/oncrpc/security/CredentialsSys.java
@@ -20,6 +20,7 @@
import java.net.InetAddress;
import java.net.UnknownHostException;
+import org.apache.commons.io.Charsets;
import org.apache.hadoop.oncrpc.XDR;
/** Credential used by AUTH_SYS */
@@ -93,7 +94,7 @@ public void read(XDR xdr) {
@Override
public void write(XDR xdr) {
// mStamp + mHostName.length + mHostName + mUID + mGID + mAuxGIDs.count
- mCredentialsLength = 20 + mHostName.getBytes().length;
+ mCredentialsLength = 20 + mHostName.getBytes(Charsets.UTF_8).length;
// mAuxGIDs
if (mAuxGIDs != null && mAuxGIDs.length > 0) {
mCredentialsLength += mAuxGIDs.length * 4;
diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/client/HttpFSFileSystem.java b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/client/HttpFSFileSystem.java
index 367308d421ce0..5b079e940509c 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/client/HttpFSFileSystem.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/client/HttpFSFileSystem.java
@@ -78,7 +78,7 @@
/**
* HttpFSServer implementation of the FileSystemAccess FileSystem.
- *
+ *
* This implementation allows a user to access HDFS over HTTP via a HttpFSServer server.
*/
@InterfaceAudience.Private
@@ -223,7 +223,7 @@ public String getMethod() {
/**
* Convenience method that creates a HttpURLConnection for the
* HttpFSServer file system operations.
- *
+ *
* This methods performs and injects any needed authentication credentials
* via the {@link #getConnection(URL, String)} method
*
@@ -289,7 +289,7 @@ public HttpURLConnection run() throws Exception {
/**
* Convenience method that creates a HttpURLConnection for the specified URL.
- *
+ *
* This methods performs and injects any needed authentication credentials.
*
* @param url url to connect to.
@@ -371,7 +371,7 @@ protected int getDefaultPort() {
/**
* HttpFSServer subclass of the FSDataInputStream.
- *
+ *
* This implementation does not support the
* PositionReadable and Seekable methods.
*/
@@ -414,8 +414,8 @@ public boolean seekToNewSource(long targetPos) throws IOException {
/**
* Opens an FSDataInputStream at the indicated Path.
- *
- * IMPORTANT: the returned does not support the
+ *
+ * IMPORTANT: the returned FSDataInputStream does not support the
* PositionReadable and Seekable methods.
*
* @param f the file name to open
@@ -434,7 +434,7 @@ public FSDataInputStream open(Path f, int bufferSize) throws IOException {
/**
* HttpFSServer subclass of the FSDataOutputStream.
- *
+ *
* This implementation closes the underlying HTTP connection validating the Http connection status
* at closing time.
*/
@@ -516,7 +516,7 @@ private FSDataOutputStream uploadData(String method, Path f, Map
/**
* Opens an FSDataOutputStream at the indicated Path with write-progress
* reporting.
- *
+ *
* IMPORTANT: The Progressable parameter is not used.
*
* @param f the file name to open.
@@ -549,7 +549,7 @@ public FSDataOutputStream create(Path f, FsPermission permission,
/**
* Append to an existing file (optional operation).
- *
+ *
* IMPORTANT: The Progressable parameter is not used.
*
* @param f the existing file to be appended.
@@ -838,7 +838,7 @@ public boolean setReplication(Path src, short replication)
* Modify the ACL entries for a file.
*
* @param path Path to modify
- * @param aclSpec List describing modifications
+ * @param aclSpec describing modifications
* @throws IOException
*/
@Override
@@ -855,7 +855,7 @@ public void modifyAclEntries(Path path, List aclSpec)
/**
* Remove the specified ACL entries from a file
* @param path Path to modify
- * @param aclSpec List describing entries to remove
+ * @param aclSpec describing entries to remove
* @throws IOException
*/
@Override
@@ -900,7 +900,7 @@ public void removeAcl(Path path) throws IOException {
/**
* Set the ACLs for the given file
* @param path Path to modify
- * @param aclSpec List describing modifications, must include
+ * @param aclSpec describing modifications, must include
* entries for user, group, and others for compatibility
* with permission bits.
* @throws IOException
diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/client/HttpsFSFileSystem.java b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/client/HttpsFSFileSystem.java
index a696cd8c119ea..0a2e08dad0493 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/client/HttpsFSFileSystem.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/client/HttpsFSFileSystem.java
@@ -18,10 +18,10 @@
package org.apache.hadoop.fs.http.client;
/**
- * HttpFSServer implementation of the FileSystemAccess FileSystem for SSL.
- *
- * This implementation allows a user to access HDFS over HTTPS via a
- * HttpFSServer server.
+ * HttpFSServer implementation of the FileSystemAccess FileSystem for SSL.
+ *
+ * This implementation allows a user to access HDFS over HTTPS via a
+ * HttpFSServer server.
*/
public class HttpsFSFileSystem extends HttpFSFileSystem {
diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/CheckUploadContentTypeFilter.java b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/CheckUploadContentTypeFilter.java
index 67df9a8e642e3..836b4ce9ffd87 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/CheckUploadContentTypeFilter.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/CheckUploadContentTypeFilter.java
@@ -50,7 +50,7 @@ public class CheckUploadContentTypeFilter implements Filter {
/**
* Initializes the filter.
- *
+ *
* This implementation is a NOP.
*
* @param config filter configuration.
@@ -103,7 +103,7 @@ public void doFilter(ServletRequest request, ServletResponse response,
/**
* Destroys the filter.
- *
+ *
* This implementation is a NOP.
*/
@Override
diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/HttpFSAuthenticationFilter.java b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/HttpFSAuthenticationFilter.java
index 8b332fc6e9abf..f0fe4c559614b 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/HttpFSAuthenticationFilter.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/HttpFSAuthenticationFilter.java
@@ -44,7 +44,7 @@ public class HttpFSAuthenticationFilter
/**
* Returns the hadoop-auth configuration from HttpFSServer's configuration.
- *
+ *
* It returns all HttpFSServer's configuration properties prefixed with
* httpfs.authentication. The httpfs.authentication
* prefix is removed from the returned property names.
diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/HttpFSExceptionProvider.java b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/HttpFSExceptionProvider.java
index 3a8d9ada4e688..aed634312349b 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/HttpFSExceptionProvider.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/HttpFSExceptionProvider.java
@@ -43,7 +43,6 @@ public class HttpFSExceptionProvider extends ExceptionProvider {
/**
* Maps different exceptions thrown by HttpFSServer to HTTP status codes.
- *
*
* - SecurityException : HTTP UNAUTHORIZED
* - FileNotFoundException : HTTP NOT_FOUND
diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/HttpFSServer.java b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/HttpFSServer.java
index f9eb454d9a5ff..910371834617e 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/HttpFSServer.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/HttpFSServer.java
@@ -85,7 +85,7 @@
/**
* Main class of HttpFSServer server.
- *
+ *
* The HttpFSServer class uses Jersey JAX-RS to binds HTTP requests to the
* different operations.
*/
@@ -117,7 +117,7 @@ private T fsExecute(UserGroupInformation ugi, FileSystemAccess.FileSystemExe
/**
* Returns a filesystem instance. The fileystem instance is wired for release at the completion of
* the current Servlet request via the {@link FileSystemReleaseFilter}.
- *
+ *
* If a do-as user is specified, the current user must be a valid proxyuser, otherwise an
* AccessControlException will be thrown.
*
diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/HttpFSServerWebApp.java b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/HttpFSServerWebApp.java
index b7ae3015e3b3f..66438b5f4ab41 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/HttpFSServerWebApp.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/HttpFSServerWebApp.java
@@ -34,9 +34,9 @@
* HttpFSServer server, it is a javax.servlet.ServletContextListener
* implementation that is wired in HttpFSServer's WAR
* WEB-INF/web.xml.
- *
+ *
* It provides acces to the server context via the singleton {@link #get}.
- *
+ *
* All the configuration is loaded from configuration properties prefixed
* with httpfs..
*/
diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/lib/lang/XException.java b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/lib/lang/XException.java
index f974159c63867..467ca235653bf 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/lib/lang/XException.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/lib/lang/XException.java
@@ -61,7 +61,7 @@ private XException(ERROR error, String message, Throwable cause) {
/**
* Creates an XException using another XException as cause.
- *
+ *
* The error code and error message are extracted from the cause.
*
* @param cause exception cause.
@@ -95,7 +95,7 @@ public ERROR getError() {
/**
* Creates a message using a error message template and arguments.
- *
+ *
* The template must be in JDK MessageFormat syntax
* (using {#} positional parameters).
*
diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/lib/server/BaseService.java b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/lib/server/BaseService.java
index 088f90058dd24..9d9ce7a1115bf 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/lib/server/BaseService.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/lib/server/BaseService.java
@@ -44,11 +44,11 @@ public BaseService(String prefix) {
/**
* Initializes the service.
- *
+ *
* It collects all service properties (properties having the
* #SERVER#.#SERVICE#. prefix). The property names are then
* trimmed from the #SERVER#.#SERVICE#. prefix.
- *
+ *
* After collecting the service properties it delegates to the
* {@link #init()} method.
*
@@ -75,7 +75,7 @@ public final void init(Server server) throws ServiceException {
/**
* Post initializes the service. This method is called by the
* {@link Server} after all services of the server have been initialized.
- *
+ *
* This method does a NOP.
*
* @throws ServiceException thrown if the service could not be
@@ -88,7 +88,7 @@ public void postInit() throws ServiceException {
/**
* Destroy the services. This method is called once, when the
* {@link Server} owning the service is being destroyed.
- *
+ *
* This method does a NOP.
*/
@Override
@@ -98,7 +98,7 @@ public void destroy() {
/**
* Returns the service dependencies of this service. The service will be
* instantiated only if all the service dependencies are already initialized.
- *
+ *
* This method returns an empty array (size 0)
*
* @return an empty array (size 0).
@@ -110,7 +110,7 @@ public Class[] getServiceDependencies() {
/**
* Notification callback when the server changes its status.
- *
+ *
* This method returns an empty array (size 0)
*
* @param oldStatus old server status.
@@ -154,7 +154,7 @@ protected String getPrefixedName(String name) {
/**
* Returns the service configuration properties. Property
* names are trimmed off from its prefix.
- *
+ *
* The sevice configuration properties are all properties
* with names starting with #SERVER#.#SERVICE#.
* in the server configuration.
@@ -169,7 +169,7 @@ protected Configuration getServiceConfig() {
/**
* Initializes the server.
- *
+ *
* This method is called by {@link #init(Server)} after all service properties
* (properties prefixed with
*
diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/lib/server/Server.java b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/lib/server/Server.java
index d083831c58a21..5c1bb4f7278bf 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/lib/server/Server.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/lib/server/Server.java
@@ -42,40 +42,39 @@
/**
* A Server class provides standard configuration, logging and {@link Service}
* lifecyle management.
- *
+ *
* A Server normally has a home directory, a configuration directory, a temp
* directory and logs directory.
- *
+ *
* The Server configuration is loaded from 2 overlapped files,
* #SERVER#-default.xml and #SERVER#-site.xml. The
* default file is loaded from the classpath, the site file is laoded from the
* configuration directory.
- *
+ *
* The Server collects all configuration properties prefixed with
* #SERVER#. The property names are then trimmed from the
* #SERVER# prefix.
- *
+ *
* The Server log configuration is loaded from the
* #SERVICE#-log4j.properties file in the configuration directory.
- *
+ *
* The lifecycle of server is defined in by {@link Server.Status} enum.
* When a server is create, its status is UNDEF, when being initialized it is
* BOOTING, once initialization is complete by default transitions to NORMAL.
* The #SERVER#.startup.status configuration property can be used
* to specify a different startup status (NORMAL, ADMIN or HALTED).
- *
+ *
* Services classes are defined in the #SERVER#.services and
* #SERVER#.services.ext properties. They are loaded in order
* (services first, then services.ext).
- *
+ *
* Before initializing the services, they are traversed and duplicate service
* interface are removed from the service list. The last service using a given
* interface wins (this enables a simple override mechanism).
- *
+ *
* After the services have been resoloved by interface de-duplication they are
* initialized in order. Once all services are initialized they are
* post-initialized (this enables late/conditional service bindings).
- *
*/
@InterfaceAudience.Private
public class Server {
@@ -152,7 +151,7 @@ public boolean isOperational() {
/**
* Creates a server instance.
- *
+ *
* The config, log and temp directories are all under the specified home directory.
*
* @param name server name.
@@ -177,9 +176,9 @@ public Server(String name, String homeDir, String configDir, String logDir, Stri
/**
* Creates a server instance.
- *
+ *
* The config, log and temp directories are all under the specified home directory.
- *
+ *
* It uses the provided configuration instead loading it from the config dir.
*
* @param name server name.
@@ -192,7 +191,7 @@ public Server(String name, String homeDir, Configuration config) {
/**
* Creates a server instance.
- *
+ *
* It uses the provided configuration instead loading it from the config dir.
*
* @param name server name.
@@ -250,9 +249,9 @@ public Status getStatus() {
/**
* Sets a new server status.
- *
+ *
* The status must be settable.
- *
+ *
* All services will be notified o the status change via the
* {@link Service#serverStatusChange(Server.Status, Server.Status)} method. If a service
* throws an exception during the notification, the server will be destroyed.
@@ -299,7 +298,7 @@ protected void ensureOperational() {
/**
* Convenience method that returns a resource as inputstream from the
* classpath.
- *
+ *
* It first attempts to use the Thread's context classloader and if not
* set it uses the ClassUtils classloader.
*
@@ -319,7 +318,7 @@ static InputStream getResource(String name) {
/**
* Initializes the Server.
- *
+ *
* The initialization steps are:
*
* - It verifies the service home and temp directories exist
@@ -335,6 +334,7 @@ static InputStream getResource(String name) {
* - Initializes the services
* - Post-initializes the services
* - Sets the server startup status
+ *
*
* @throws ServerException thrown if the server could not be initialized.
*/
@@ -625,7 +625,7 @@ protected void destroyServices() {
/**
* Destroys the server.
- *
+ *
* All services are destroyed in reverse order of initialization, then the
* Log4j framework is shutdown.
*/
@@ -651,7 +651,7 @@ public String getName() {
/**
* Returns the server prefix for server configuration properties.
- *
+ *
* By default it is the server name.
*
* @return the prefix for server configuration properties.
@@ -733,10 +733,10 @@ public T get(Class serviceKlass) {
/**
* Adds a service programmatically.
- *
+ *
* If a service with the same interface exists, it will be destroyed and
* removed before the given one is initialized and added.
- *
+ *
* If an exception is thrown the server is destroyed.
*
* @param klass service class to add.
diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/lib/servlet/FileSystemReleaseFilter.java b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/lib/servlet/FileSystemReleaseFilter.java
index 827bcff891657..cf73979f6476d 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/lib/servlet/FileSystemReleaseFilter.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/lib/servlet/FileSystemReleaseFilter.java
@@ -33,7 +33,7 @@
/**
* The FileSystemReleaseFilter releases back to the
* {@link FileSystemAccess} service a FileSystem instance.
- *
+ *
* This filter is useful in situations where a servlet request
* is streaming out HDFS data and the corresponding filesystem
* instance have to be closed after the streaming completes.
@@ -44,7 +44,7 @@ public abstract class FileSystemReleaseFilter implements Filter {
/**
* Initializes the filter.
- *
+ *
* This implementation is a NOP.
*
* @param filterConfig filter configuration.
@@ -83,7 +83,7 @@ public void doFilter(ServletRequest servletRequest, ServletResponse servletRespo
/**
* Destroys the filter.
- *
+ *
* This implementation is a NOP.
*/
@Override
diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/lib/servlet/HostnameFilter.java b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/lib/servlet/HostnameFilter.java
index dd395f6749549..64f4926f5af1e 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/lib/servlet/HostnameFilter.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/lib/servlet/HostnameFilter.java
@@ -43,7 +43,7 @@ public class HostnameFilter implements Filter {
/**
* Initializes the filter.
- *
+ *
* This implementation is a NOP.
*
* @param config filter configuration.
@@ -56,7 +56,7 @@ public void init(FilterConfig config) throws ServletException {
/**
* Resolves the requester hostname and delegates the request to the chain.
- *
+ *
* The requester hostname is available via the {@link #get} method.
*
* @param request servlet request.
@@ -101,7 +101,7 @@ public static String get() {
/**
* Destroys the filter.
- *
+ *
* This implementation is a NOP.
*/
@Override
diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/lib/servlet/MDCFilter.java b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/lib/servlet/MDCFilter.java
index 07b552d7ec510..156cf64ab00cc 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/lib/servlet/MDCFilter.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/lib/servlet/MDCFilter.java
@@ -33,7 +33,7 @@
/**
* Filter that sets request contextual information for the slf4j MDC.
- *
+ *
* It sets the following values:
*
* - hostname: if the {@link HostnameFilter} is present and configured
@@ -48,7 +48,7 @@ public class MDCFilter implements Filter {
/**
* Initializes the filter.
- *
+ *
* This implementation is a NOP.
*
* @param config filter configuration.
@@ -93,7 +93,7 @@ public void doFilter(ServletRequest request, ServletResponse response, FilterCha
/**
* Destroys the filter.
- *
+ *
* This implementation is a NOP.
*/
@Override
diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/lib/servlet/ServerWebApp.java b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/lib/servlet/ServerWebApp.java
index 9b0ea2a11e1cc..cd1659383bbef 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/lib/servlet/ServerWebApp.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/lib/servlet/ServerWebApp.java
@@ -75,21 +75,21 @@ protected ServerWebApp(String name, String homeDir, Configuration config) {
/**
* Constructor. Subclasses must have a default constructor specifying
* the server name.
- *
+ *
* The server name is used to resolve the Java System properties that define
* the server home, config, log and temp directories.
- *
+ *
* The home directory is looked in the Java System property
* #SERVER_NAME#.home.dir.
- *
+ *
* The config directory is looked in the Java System property
* #SERVER_NAME#.config.dir, if not defined it resolves to
* the #SERVER_HOME_DIR#/conf directory.
- *
+ *
* The log directory is looked in the Java System property
* #SERVER_NAME#.log.dir, if not defined it resolves to
* the #SERVER_HOME_DIR#/log directory.
- *
+ *
* The temp directory is looked in the Java System property
* #SERVER_NAME#.temp.dir, if not defined it resolves to
* the #SERVER_HOME_DIR#/temp directory.
@@ -105,7 +105,7 @@ public ServerWebApp(String name) {
/**
* Returns the server home directory.
- *
+ *
* It is looked up in the Java System property
* #SERVER_NAME#.home.dir.
*
@@ -159,15 +159,15 @@ public void contextInitialized(ServletContextEvent event) {
}
/**
- * Resolves the host & port InetSocketAddress the web server is listening to.
- *
+ * Resolves the host and port InetSocketAddress the web server is listening to.
+ *
* This implementation looks for the following 2 properties:
*
* - #SERVER_NAME#.http.hostname
* - #SERVER_NAME#.http.port
*
*
- * @return the host & port InetSocketAddress the web server is listening to.
+ * @return the host and port InetSocketAddress the web server is listening to.
* @throws ServerException thrown if any of the above 2 properties is not defined.
*/
protected InetSocketAddress resolveAuthority() throws ServerException {
@@ -217,7 +217,7 @@ public InetSocketAddress getAuthority() throws ServerException {
/**
* Sets an alternate hostname:port InetSocketAddress to use.
- *
+ *
* For testing purposes.
*
* @param authority alterante authority.
diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/lib/util/Check.java b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/lib/util/Check.java
index a398e75845466..31666e83e2711 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/lib/util/Check.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/lib/util/Check.java
@@ -26,7 +26,7 @@
/**
* Utility methods to check preconditions.
- *
+ *
* Commonly used for method arguments preconditions.
*/
@InterfaceAudience.Private
diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/lib/util/ConfigurationUtils.java b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/lib/util/ConfigurationUtils.java
index 660eae0835820..6611dd22fefc9 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/lib/util/ConfigurationUtils.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/lib/util/ConfigurationUtils.java
@@ -90,7 +90,7 @@ public static Configuration resolve(Configuration conf) {
/**
* Create a configuration from an InputStream.
- *
+ *
* ERROR canibalized from Configuration.loadResource().
*
* @param is inputstream to read the configuration from.
diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/lib/wsrs/Parameters.java b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/lib/wsrs/Parameters.java
index 0f16a9b53e504..e0f62002c70d4 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/lib/wsrs/Parameters.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/lib/wsrs/Parameters.java
@@ -26,7 +26,7 @@
/**
* Class that contains all parsed JAX-RS parameters.
- *
+ *
* Instances are created by the {@link ParametersProvider} class.
*/
@InterfaceAudience.Private
@@ -63,7 +63,7 @@ public > V get(String name, Class klass) {
*
* @param name parameter name.
* @param klass class of the parameter, used for value casting.
- * @return List the values of the parameter.
+ * @return the values of the parameter.
*/
@SuppressWarnings("unchecked")
public > List getValues(String name, Class klass) {
diff --git a/hadoop-hdfs-project/hadoop-hdfs-nfs/pom.xml b/hadoop-hdfs-project/hadoop-hdfs-nfs/pom.xml
index 42962a6c268f0..9a9d29ceec3ad 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-nfs/pom.xml
+++ b/hadoop-hdfs-project/hadoop-hdfs-nfs/pom.xml
@@ -179,6 +179,11 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd">
xmlenc
compile
+
+ org.bouncycastle
+ bcprov-jdk16
+ test
+
diff --git a/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/conf/NfsConfigKeys.java b/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/conf/NfsConfigKeys.java
index 178d855cb6da2..9e4aaf538f0ad 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/conf/NfsConfigKeys.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/conf/NfsConfigKeys.java
@@ -60,4 +60,17 @@ public class NfsConfigKeys {
public final static String LARGE_FILE_UPLOAD = "nfs.large.file.upload";
public final static boolean LARGE_FILE_UPLOAD_DEFAULT = true;
+
+ public static final String NFS_HTTP_PORT_KEY = "nfs.http.port";
+ public static final int NFS_HTTP_PORT_DEFAULT = 50079;
+ public static final String NFS_HTTP_ADDRESS_KEY = "nfs.http.address";
+ public static final String NFS_HTTP_ADDRESS_DEFAULT = "0.0.0.0:" + NFS_HTTP_PORT_DEFAULT;
+
+ public static final String NFS_HTTPS_PORT_KEY = "nfs.https.port";
+ public static final int NFS_HTTPS_PORT_DEFAULT = 50579;
+ public static final String NFS_HTTPS_ADDRESS_KEY = "nfs.https.address";
+ public static final String NFS_HTTPS_ADDRESS_DEFAULT = "0.0.0.0:" + NFS_HTTPS_PORT_DEFAULT;
+
+ public static final String NFS_METRICS_PERCENTILES_INTERVALS_KEY = "nfs.metrics.percentiles.intervals";
+ public static final String NFS_METRICS_PERCENTILES_INTERVALS_DEFAULT = "";
}
diff --git a/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/Nfs3.java b/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/Nfs3.java
index 3daf7bb68db9b..ac9abf8b02b00 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/Nfs3.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/Nfs3.java
@@ -42,7 +42,8 @@ public Nfs3(NfsConfiguration conf) throws IOException {
public Nfs3(NfsConfiguration conf, DatagramSocket registrationSocket,
boolean allowInsecurePorts) throws IOException {
- super(new RpcProgramNfs3(conf, registrationSocket, allowInsecurePorts), conf);
+ super(RpcProgramNfs3.createRpcProgramNfs3(conf, registrationSocket,
+ allowInsecurePorts), conf);
mountd = new Mountd(conf, registrationSocket, allowInsecurePorts);
}
diff --git a/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/Nfs3HttpServer.java b/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/Nfs3HttpServer.java
new file mode 100644
index 0000000000000..c37a21e7d83f5
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/Nfs3HttpServer.java
@@ -0,0 +1,111 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.nfs.nfs3;
+
+import java.io.IOException;
+import java.net.InetSocketAddress;
+import java.net.URI;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdfs.DFSUtil;
+import org.apache.hadoop.hdfs.nfs.conf.NfsConfigKeys;
+import org.apache.hadoop.hdfs.nfs.conf.NfsConfiguration;
+import org.apache.hadoop.hdfs.server.common.JspHelper;
+import org.apache.hadoop.http.HttpConfig;
+import org.apache.hadoop.http.HttpServer2;
+import org.apache.hadoop.net.NetUtils;
+
+/**
+ * Encapsulates the HTTP server started by the NFS3 gateway.
+ */
+class Nfs3HttpServer {
+ private int infoPort;
+ private int infoSecurePort;
+
+ private HttpServer2 httpServer;
+
+ private final NfsConfiguration conf;
+
+ Nfs3HttpServer(NfsConfiguration conf) {
+ this.conf = conf;
+ }
+
+ void start() throws IOException {
+ final InetSocketAddress httpAddr = getHttpAddress(conf);
+
+ final String httpsAddrString = conf.get(
+ NfsConfigKeys.NFS_HTTPS_ADDRESS_KEY,
+ NfsConfigKeys.NFS_HTTPS_ADDRESS_DEFAULT);
+ InetSocketAddress httpsAddr = NetUtils.createSocketAddr(httpsAddrString);
+
+ HttpServer2.Builder builder = DFSUtil.httpServerTemplateForNNAndJN(conf,
+ httpAddr, httpsAddr, "nfs3",
+ NfsConfigKeys.DFS_NFS_KERBEROS_PRINCIPAL_KEY,
+ NfsConfigKeys.DFS_NFS_KEYTAB_FILE_KEY);
+
+ this.httpServer = builder.build();
+ this.httpServer.start();
+
+ HttpConfig.Policy policy = DFSUtil.getHttpPolicy(conf);
+ int connIdx = 0;
+ if (policy.isHttpEnabled()) {
+ infoPort = httpServer.getConnectorAddress(connIdx++).getPort();
+ }
+
+ if (policy.isHttpsEnabled()) {
+ infoSecurePort = httpServer.getConnectorAddress(connIdx).getPort();
+ }
+ }
+
+ void stop() throws IOException {
+ if (httpServer != null) {
+ try {
+ httpServer.stop();
+ } catch (Exception e) {
+ throw new IOException(e);
+ }
+ }
+ }
+
+ public int getPort() {
+ return this.infoPort;
+ }
+
+ public int getSecurePort() {
+ return this.infoSecurePort;
+ }
+
+ /**
+ * Return the URI that locates the HTTP server.
+ */
+ public URI getServerURI() {
+ // getHttpClientScheme() only returns https for HTTPS_ONLY policy. This
+ // matches the behavior that the first connector is a HTTPS connector only
+ // for HTTPS_ONLY policy.
+ InetSocketAddress addr = httpServer.getConnectorAddress(0);
+ return URI.create(DFSUtil.getHttpClientScheme(conf) + "://"
+ + NetUtils.getHostPortString(addr));
+ }
+
+ public InetSocketAddress getHttpAddress(Configuration conf) {
+ String addr = conf.get(NfsConfigKeys.NFS_HTTP_ADDRESS_KEY,
+ NfsConfigKeys.NFS_HTTP_ADDRESS_DEFAULT);
+ return NetUtils.createSocketAddr(addr, NfsConfigKeys.NFS_HTTP_PORT_DEFAULT,
+ NfsConfigKeys.NFS_HTTP_ADDRESS_KEY);
+ }
+}
\ No newline at end of file
diff --git a/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/Nfs3Metrics.java b/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/Nfs3Metrics.java
new file mode 100644
index 0000000000000..d36ea732f0f2d
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/Nfs3Metrics.java
@@ -0,0 +1,220 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.nfs.nfs3;
+
+import static org.apache.hadoop.metrics2.impl.MsInfo.SessionId;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdfs.DFSConfigKeys;
+import org.apache.hadoop.hdfs.nfs.conf.NfsConfigKeys;
+import org.apache.hadoop.metrics2.MetricsSystem;
+import org.apache.hadoop.metrics2.annotation.Metric;
+import org.apache.hadoop.metrics2.annotation.Metrics;
+import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;
+import org.apache.hadoop.metrics2.lib.MetricsRegistry;
+import org.apache.hadoop.metrics2.lib.MutableCounterLong;
+import org.apache.hadoop.metrics2.lib.MutableQuantiles;
+import org.apache.hadoop.metrics2.lib.MutableRate;
+import org.apache.hadoop.metrics2.source.JvmMetrics;
+
+/**
+ * This class is for maintaining the various NFS gateway activity statistics and
+ * publishing them through the metrics interfaces.
+ */
+@InterfaceAudience.Private
+@Metrics(about = "Nfs3 metrics", context = "dfs")
+public class Nfs3Metrics {
+ // All mutable rates are in nanoseconds
+ // No metric for nullProcedure;
+ @Metric MutableRate getattr;
+ @Metric MutableRate setattr;
+ @Metric MutableRate lookup;
+ @Metric MutableRate access;
+ @Metric MutableRate readlink;
+ @Metric MutableRate read;
+ final MutableQuantiles[] readNanosQuantiles;
+ @Metric MutableRate write;
+ final MutableQuantiles[] writeNanosQuantiles;
+ @Metric MutableRate create;
+ @Metric MutableRate mkdir;
+ @Metric MutableRate symlink;
+ @Metric MutableRate mknod;
+ @Metric MutableRate remove;
+ @Metric MutableRate rmdir;
+ @Metric MutableRate rename;
+ @Metric MutableRate link;
+ @Metric MutableRate readdir;
+ @Metric MutableRate readdirplus;
+ @Metric MutableRate fsstat;
+ @Metric MutableRate fsinfo;
+ @Metric MutableRate pathconf;
+ @Metric MutableRate commit;
+ final MutableQuantiles[] commitNanosQuantiles;
+
+ @Metric MutableCounterLong bytesWritten;
+ @Metric MutableCounterLong bytesRead;
+
+ final MetricsRegistry registry = new MetricsRegistry("nfs3");
+ final String name;
+ JvmMetrics jvmMetrics = null;
+
+ public Nfs3Metrics(String name, String sessionId, int[] intervals,
+ final JvmMetrics jvmMetrics) {
+ this.name = name;
+ this.jvmMetrics = jvmMetrics;
+ registry.tag(SessionId, sessionId);
+
+ final int len = intervals.length;
+ readNanosQuantiles = new MutableQuantiles[len];
+ writeNanosQuantiles = new MutableQuantiles[len];
+ commitNanosQuantiles = new MutableQuantiles[len];
+
+ for (int i = 0; i < len; i++) {
+ int interval = intervals[i];
+ readNanosQuantiles[i] = registry.newQuantiles("readProcessNanos"
+ + interval + "s", "Read process in ns", "ops", "latency", interval);
+ writeNanosQuantiles[i] = registry.newQuantiles("writeProcessNanos"
+ + interval + "s", " process in ns", "ops", "latency", interval);
+ commitNanosQuantiles[i] = registry.newQuantiles("commitProcessNanos"
+ + interval + "s", "Read process in ns", "ops", "latency", interval);
+ }
+ }
+
+ public static Nfs3Metrics create(Configuration conf, String gatewayName) {
+ String sessionId = conf.get(DFSConfigKeys.DFS_METRICS_SESSION_ID_KEY);
+ MetricsSystem ms = DefaultMetricsSystem.instance();
+ JvmMetrics jm = JvmMetrics.create(gatewayName, sessionId, ms);
+
+ // Percentile measurement is [,,,] by default
+ int[] intervals = conf.getInts(conf.get(
+ NfsConfigKeys.NFS_METRICS_PERCENTILES_INTERVALS_KEY,
+ NfsConfigKeys.NFS_METRICS_PERCENTILES_INTERVALS_DEFAULT));
+ return ms.register(new Nfs3Metrics(gatewayName, sessionId, intervals, jm));
+ }
+
+ public String name() {
+ return name;
+ }
+
+ public JvmMetrics getJvmMetrics() {
+ return jvmMetrics;
+ }
+
+ public void incrBytesWritten(long bytes) {
+ bytesWritten.incr(bytes);
+ }
+
+ public void incrBytesRead(long bytes) {
+ bytesRead.incr(bytes);
+ }
+
+ public void addGetattr(long latencyNanos) {
+ getattr.add(latencyNanos);
+ }
+
+ public void addSetattr(long latencyNanos) {
+ setattr.add(latencyNanos);
+ }
+
+ public void addLookup(long latencyNanos) {
+ lookup.add(latencyNanos);
+ }
+
+ public void addAccess(long latencyNanos) {
+ access.add(latencyNanos);
+ }
+
+ public void addReadlink(long latencyNanos) {
+ readlink.add(latencyNanos);
+ }
+
+ public void addRead(long latencyNanos) {
+ read.add(latencyNanos);
+ for (MutableQuantiles q : readNanosQuantiles) {
+ q.add(latencyNanos);
+ }
+ }
+
+ public void addWrite(long latencyNanos) {
+ write.add(latencyNanos);
+ for (MutableQuantiles q : writeNanosQuantiles) {
+ q.add(latencyNanos);
+ }
+ }
+
+ public void addCreate(long latencyNanos) {
+ create.add(latencyNanos);
+ }
+
+ public void addMkdir(long latencyNanos) {
+ mkdir.add(latencyNanos);
+ }
+
+ public void addSymlink(long latencyNanos) {
+ symlink.add(latencyNanos);
+ }
+
+ public void addMknod(long latencyNanos) {
+ mknod.add(latencyNanos);
+ }
+
+ public void addRemove(long latencyNanos) {
+ remove.add(latencyNanos);
+ }
+
+ public void addRmdir(long latencyNanos) {
+ rmdir.add(latencyNanos);
+ }
+
+ public void addRename(long latencyNanos) {
+ rename.add(latencyNanos);
+ }
+
+ public void addLink(long latencyNanos) {
+ link.add(latencyNanos);
+ }
+
+ public void addReaddir(long latencyNanos) {
+ readdir.add(latencyNanos);
+ }
+
+ public void addReaddirplus(long latencyNanos) {
+ readdirplus.add(latencyNanos);
+ }
+
+ public void addFsstat(long latencyNanos) {
+ fsstat.add(latencyNanos);
+ }
+
+ public void addFsinfo(long latencyNanos) {
+ fsinfo.add(latencyNanos);
+ }
+
+ public void addPathconf(long latencyNanos) {
+ pathconf.add(latencyNanos);
+ }
+
+ public void addCommit(long latencyNanos) {
+ commit.add(latencyNanos);
+ for (MutableQuantiles q : commitNanosQuantiles) {
+ q.add(latencyNanos);
+ }
+ }
+
+}
\ No newline at end of file
diff --git a/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/Nfs3Utils.java b/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/Nfs3Utils.java
index 50e83ed4faeb7..cc17394197a5f 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/Nfs3Utils.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/Nfs3Utils.java
@@ -213,4 +213,8 @@ public static byte[] longToByte(long v) {
data[7] = (byte) (v >>> 0);
return data;
}
+
+ public static long getElapsedTime(long startTimeNano) {
+ return System.nanoTime() - startTimeNano;
+ }
}
diff --git a/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/OpenFileCtx.java b/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/OpenFileCtx.java
index b31baf58f5d03..a06d1c5c02b35 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/OpenFileCtx.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/OpenFileCtx.java
@@ -129,9 +129,8 @@ static class CommitCtx {
private final Channel channel;
private final int xid;
private final Nfs3FileAttributes preOpAttr;
-
- // Remember time for debug purpose
- private final long startTime;
+
+ public final long startTime;
long getOffset() {
return offset;
@@ -159,7 +158,7 @@ long getStartTime() {
this.channel = channel;
this.xid = xid;
this.preOpAttr = preOpAttr;
- this.startTime = Time.monotonicNow();
+ this.startTime = System.nanoTime();
}
@Override
@@ -687,6 +686,8 @@ private void receivedNewWriteInternal(DFSClient dfsClient,
WccData fileWcc = new WccData(preOpAttr, latestAttr);
WRITE3Response response = new WRITE3Response(Nfs3Status.NFS3_OK,
fileWcc, count, stableHow, Nfs3Constant.WRITE_COMMIT_VERF);
+ RpcProgramNfs3.metrics.addWrite(Nfs3Utils
+ .getElapsedTime(writeCtx.startTime));
Nfs3Utils
.writeChannel(channel, response.serialize(new XDR(),
xid, new VerifierNone()), xid);
@@ -1131,14 +1132,16 @@ private void processCommits(long offset) {
COMMIT3Response response = new COMMIT3Response(status, wccData,
Nfs3Constant.WRITE_COMMIT_VERF);
+ RpcProgramNfs3.metrics.addCommit(Nfs3Utils
+ .getElapsedTime(commit.startTime));
Nfs3Utils.writeChannelCommit(commit.getChannel(), response
.serialize(new XDR(), commit.getXid(),
new VerifierNone()), commit.getXid());
if (LOG.isDebugEnabled()) {
LOG.debug("FileId: " + latestAttr.getFileId() + " Service time:"
- + (Time.monotonicNow() - commit.getStartTime())
- + "ms. Sent response for commit:" + commit);
+ + Nfs3Utils.getElapsedTime(commit.startTime)
+ + "ns. Sent response for commit:" + commit);
}
entry = pendingCommits.firstEntry();
}
@@ -1162,6 +1165,7 @@ private void doSingleWrite(final WriteCtx writeCtx) {
// The write is not protected by lock. asyncState is used to make sure
// there is one thread doing write back at any time
writeCtx.writeData(fos);
+ RpcProgramNfs3.metrics.incrBytesWritten(writeCtx.getCount());
long flushedOffset = getFlushedOffset();
if (flushedOffset != (offset + count)) {
@@ -1213,6 +1217,7 @@ private void doSingleWrite(final WriteCtx writeCtx) {
}
WRITE3Response response = new WRITE3Response(Nfs3Status.NFS3_OK,
fileWcc, count, stableHow, Nfs3Constant.WRITE_COMMIT_VERF);
+ RpcProgramNfs3.metrics.addWrite(Nfs3Utils.getElapsedTime(writeCtx.startTime));
Nfs3Utils.writeChannel(channel, response.serialize(
new XDR(), xid, new VerifierNone()), xid);
}
diff --git a/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/PrivilegedNfsGatewayStarter.java b/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/PrivilegedNfsGatewayStarter.java
index 98862eda5e7e8..3934d7c494b72 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/PrivilegedNfsGatewayStarter.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/PrivilegedNfsGatewayStarter.java
@@ -26,7 +26,7 @@
/**
* This class is used to allow the initial registration of the NFS gateway with
- * the system portmap daemon to come from a privileged (< 1024) port. This is
+ * the system portmap daemon to come from a privileged (< 1024) port. This is
* necessary on certain operating systems to work around this bug in rpcbind:
*
* Red Hat: https://bugzilla.redhat.com/show_bug.cgi?id=731542
diff --git a/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/RpcProgramNfs3.java b/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/RpcProgramNfs3.java
index f86dbecd44c58..148d4f7630b91 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/RpcProgramNfs3.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/RpcProgramNfs3.java
@@ -25,6 +25,7 @@
import java.net.InetSocketAddress;
import java.net.SocketAddress;
import java.nio.ByteBuffer;
+import java.nio.charset.Charset;
import java.util.EnumSet;
import org.apache.commons.logging.Log;
@@ -47,6 +48,8 @@
import org.apache.hadoop.hdfs.protocol.HdfsConstants;
import org.apache.hadoop.hdfs.protocol.HdfsFileStatus;
import org.apache.hadoop.ipc.RemoteException;
+import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;
+import org.apache.hadoop.net.DNS;
import org.apache.hadoop.nfs.AccessPrivilege;
import org.apache.hadoop.nfs.NfsExports;
import org.apache.hadoop.nfs.NfsFileType;
@@ -162,6 +165,8 @@ public class RpcProgramNfs3 extends RpcProgram implements Nfs3Interface {
private final RpcCallCache rpcCallCache;
private JvmPauseMonitor pauseMonitor;
+ private Nfs3HttpServer infoServer = null;
+ static Nfs3Metrics metrics;
public RpcProgramNfs3(NfsConfiguration config, DatagramSocket registrationSocket,
boolean allowInsecurePorts) throws IOException {
@@ -204,8 +209,20 @@ public RpcProgramNfs3(NfsConfiguration config, DatagramSocket registrationSocket
}
rpcCallCache = new RpcCallCache("NFS3", 256);
+ infoServer = new Nfs3HttpServer(config);
}
+ public static RpcProgramNfs3 createRpcProgramNfs3(NfsConfiguration config,
+ DatagramSocket registrationSocket, boolean allowInsecurePorts)
+ throws IOException {
+ DefaultMetricsSystem.initialize("Nfs3");
+ String displayName = DNS.getDefaultHost("default", "default")
+ + config.getInt(NfsConfigKeys.DFS_NFS_SERVER_PORT_KEY,
+ NfsConfigKeys.DFS_NFS_SERVER_PORT_DEFAULT);
+ metrics = Nfs3Metrics.create(config, displayName);
+ return new RpcProgramNfs3(config, registrationSocket, allowInsecurePorts);
+ }
+
private void clearDirectory(String writeDumpDir) throws IOException {
File dumpDir = new File(writeDumpDir);
if (dumpDir.exists()) {
@@ -220,14 +237,20 @@ private void clearDirectory(String writeDumpDir) throws IOException {
throw new IOException("Cannot create dump directory " + dumpDir);
}
}
-
+
@Override
public void startDaemons() {
if (pauseMonitor == null) {
pauseMonitor = new JvmPauseMonitor(config);
pauseMonitor.start();
+ metrics.getJvmMetrics().setPauseMonitor(pauseMonitor);
}
writeManager.startAsyncDataSerivce();
+ try {
+ infoServer.start();
+ } catch (IOException e) {
+ LOG.error("failed to start web server", e);
+ }
}
@Override
@@ -238,6 +261,19 @@ public void stopDaemons() {
if (pauseMonitor != null) {
pauseMonitor.stop();
}
+ // Stop the web server
+ if (infoServer != null) {
+ try {
+ infoServer.stop();
+ } catch (Exception e) {
+ LOG.warn("Exception shutting down web server", e);
+ }
+ }
+ }
+
+ @VisibleForTesting
+ Nfs3HttpServer getInfoServer() {
+ return this.infoServer;
}
// Checks the type of IOException and maps it to appropriate Nfs3Status code.
@@ -631,15 +667,16 @@ READLINK3Response readlink(XDR xdr, SecurityHandler securityHandler,
}
int rtmax = config.getInt(NfsConfigKeys.DFS_NFS_MAX_READ_TRANSFER_SIZE_KEY,
NfsConfigKeys.DFS_NFS_MAX_READ_TRANSFER_SIZE_DEFAULT);
- if (rtmax < target.getBytes().length) {
- LOG.error("Link size: " + target.getBytes().length
+ if (rtmax < target.getBytes(Charset.forName("UTF-8")).length) {
+ LOG.error("Link size: "
+ + target.getBytes(Charset.forName("UTF-8")).length
+ " is larger than max transfer size: " + rtmax);
return new READLINK3Response(Nfs3Status.NFS3ERR_IO, postOpAttr,
new byte[0]);
}
return new READLINK3Response(Nfs3Status.NFS3_OK, postOpAttr,
- target.getBytes());
+ target.getBytes(Charset.forName("UTF-8")));
} catch (IOException e) {
LOG.warn("Readlink error: " + e.getClass(), e);
@@ -748,6 +785,7 @@ READ3Response read(XDR xdr, SecurityHandler securityHandler,
try {
readCount = fis.read(offset, readbuffer, 0, count);
+ metrics.incrBytesRead(readCount);
} catch (IOException e) {
// TODO: A cleaner way is to throw a new type of exception
// which requires incompatible changes.
@@ -1442,7 +1480,8 @@ private DirectoryListing listPaths(DFSClient dfsClient, String dirFileIdPath,
throw io;
}
// This happens when startAfter was just deleted
- LOG.info("Cookie couldn't be found: " + new String(startAfter)
+ LOG.info("Cookie couldn't be found: "
+ + new String(startAfter, Charset.forName("UTF-8"))
+ ", do listing from beginning");
dlisting = dfsClient
.listPaths(dirFileIdPath, HdfsFileStatus.EMPTY_NAME);
@@ -1551,7 +1590,7 @@ public READDIR3Response readdir(XDR xdr, SecurityHandler securityHandler,
startAfter = HdfsFileStatus.EMPTY_NAME;
} else {
String inodeIdPath = Nfs3Utils.getFileIdPath(cookie);
- startAfter = inodeIdPath.getBytes();
+ startAfter = inodeIdPath.getBytes(Charset.forName("UTF-8"));
}
dlisting = listPaths(dfsClient, dirFileIdPath, startAfter);
@@ -1713,7 +1752,7 @@ READDIRPLUS3Response readdirplus(XDR xdr, SecurityHandler securityHandler,
startAfter = HdfsFileStatus.EMPTY_NAME;
} else {
String inodeIdPath = Nfs3Utils.getFileIdPath(cookie);
- startAfter = inodeIdPath.getBytes();
+ startAfter = inodeIdPath.getBytes(Charset.forName("UTF-8"));
}
dlisting = listPaths(dfsClient, dirFileIdPath, startAfter);
@@ -2026,8 +2065,8 @@ COMMIT3Response commit(XDR xdr, Channel channel, int xid,
: (request.getOffset() + request.getCount());
// Insert commit as an async request
- writeManager.handleCommit(dfsClient, handle, commitOffset,
- channel, xid, preOpAttr);
+ writeManager.handleCommit(dfsClient, handle, commitOffset, channel, xid,
+ preOpAttr);
return null;
} catch (IOException e) {
LOG.warn("Exception ", e);
@@ -2109,20 +2148,29 @@ public void handleInternal(ChannelHandlerContext ctx, RpcInfo info) {
}
}
}
-
+
+ // Since write and commit could be async, they use their own startTime and
+ // only record success requests.
+ final long startTime = System.nanoTime();
+
NFS3Response response = null;
if (nfsproc3 == NFSPROC3.NULL) {
response = nullProcedure();
} else if (nfsproc3 == NFSPROC3.GETATTR) {
response = getattr(xdr, info);
+ metrics.addGetattr(Nfs3Utils.getElapsedTime(startTime));
} else if (nfsproc3 == NFSPROC3.SETATTR) {
response = setattr(xdr, info);
+ metrics.addSetattr(Nfs3Utils.getElapsedTime(startTime));
} else if (nfsproc3 == NFSPROC3.LOOKUP) {
response = lookup(xdr, info);
+ metrics.addLookup(Nfs3Utils.getElapsedTime(startTime));
} else if (nfsproc3 == NFSPROC3.ACCESS) {
response = access(xdr, info);
+ metrics.addAccess(Nfs3Utils.getElapsedTime(startTime));
} else if (nfsproc3 == NFSPROC3.READLINK) {
response = readlink(xdr, info);
+ metrics.addReadlink(Nfs3Utils.getElapsedTime(startTime));
} else if (nfsproc3 == NFSPROC3.READ) {
if (LOG.isDebugEnabled()) {
LOG.debug(Nfs3Utils.READ_RPC_START + xid);
@@ -2131,6 +2179,7 @@ public void handleInternal(ChannelHandlerContext ctx, RpcInfo info) {
if (LOG.isDebugEnabled() && (nfsproc3 == NFSPROC3.READ)) {
LOG.debug(Nfs3Utils.READ_RPC_END + xid);
}
+ metrics.addRead(Nfs3Utils.getElapsedTime(startTime));
} else if (nfsproc3 == NFSPROC3.WRITE) {
if (LOG.isDebugEnabled()) {
LOG.debug(Nfs3Utils.WRITE_RPC_START + xid);
@@ -2139,30 +2188,43 @@ public void handleInternal(ChannelHandlerContext ctx, RpcInfo info) {
// Write end debug trace is in Nfs3Utils.writeChannel
} else if (nfsproc3 == NFSPROC3.CREATE) {
response = create(xdr, info);
+ metrics.addCreate(Nfs3Utils.getElapsedTime(startTime));
} else if (nfsproc3 == NFSPROC3.MKDIR) {
response = mkdir(xdr, info);
+ metrics.addMkdir(Nfs3Utils.getElapsedTime(startTime));
} else if (nfsproc3 == NFSPROC3.SYMLINK) {
response = symlink(xdr, info);
+ metrics.addSymlink(Nfs3Utils.getElapsedTime(startTime));
} else if (nfsproc3 == NFSPROC3.MKNOD) {
response = mknod(xdr, info);
+ metrics.addMknod(Nfs3Utils.getElapsedTime(startTime));
} else if (nfsproc3 == NFSPROC3.REMOVE) {
response = remove(xdr, info);
+ metrics.addRemove(Nfs3Utils.getElapsedTime(startTime));
} else if (nfsproc3 == NFSPROC3.RMDIR) {
response = rmdir(xdr, info);
+ metrics.addRmdir(Nfs3Utils.getElapsedTime(startTime));
} else if (nfsproc3 == NFSPROC3.RENAME) {
response = rename(xdr, info);
+ metrics.addRename(Nfs3Utils.getElapsedTime(startTime));
} else if (nfsproc3 == NFSPROC3.LINK) {
response = link(xdr, info);
+ metrics.addLink(Nfs3Utils.getElapsedTime(startTime));
} else if (nfsproc3 == NFSPROC3.READDIR) {
response = readdir(xdr, info);
+ metrics.addReaddir(Nfs3Utils.getElapsedTime(startTime));
} else if (nfsproc3 == NFSPROC3.READDIRPLUS) {
response = readdirplus(xdr, info);
+ metrics.addReaddirplus(Nfs3Utils.getElapsedTime(startTime));
} else if (nfsproc3 == NFSPROC3.FSSTAT) {
response = fsstat(xdr, info);
+ metrics.addFsstat(Nfs3Utils.getElapsedTime(startTime));
} else if (nfsproc3 == NFSPROC3.FSINFO) {
response = fsinfo(xdr, info);
+ metrics.addFsinfo(Nfs3Utils.getElapsedTime(startTime));
} else if (nfsproc3 == NFSPROC3.PATHCONF) {
- response = pathconf(xdr,info);
+ response = pathconf(xdr, info);
+ metrics.addPathconf(Nfs3Utils.getElapsedTime(startTime));
} else if (nfsproc3 == NFSPROC3.COMMIT) {
response = commit(xdr, info);
} else {
diff --git a/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/WriteCtx.java b/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/WriteCtx.java
index 758fd3998b868..82c826fda1eef 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/WriteCtx.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/WriteCtx.java
@@ -84,7 +84,8 @@ public int getOriginalCount() {
private long dumpFileOffset;
private volatile DataState dataState;
-
+ public final long startTime;
+
public DataState getDataState() {
return dataState;
}
@@ -235,6 +236,7 @@ void setReplied(boolean replied) {
this.replied = replied;
this.dataState = dataState;
raf = null;
+ this.startTime = System.nanoTime();
}
@Override
diff --git a/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/WriteManager.java b/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/WriteManager.java
index e71eaa51488d4..df02e04fb6bd4 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/WriteManager.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/WriteManager.java
@@ -224,6 +224,7 @@ int commitBeforeRead(DFSClient dfsClient, FileHandle fileHandle,
status = Nfs3Status.NFS3_OK;
} else {
+ // commit request triggered by read won't create pending comment obj
COMMIT_STATUS ret = openFileCtx.checkCommit(dfsClient, commitOffset,
null, 0, null, true);
switch (ret) {
@@ -260,6 +261,7 @@ int commitBeforeRead(DFSClient dfsClient, FileHandle fileHandle,
void handleCommit(DFSClient dfsClient, FileHandle fileHandle,
long commitOffset, Channel channel, int xid, Nfs3FileAttributes preOpAttr) {
+ long startTime = System.nanoTime();
int status;
OpenFileCtx openFileCtx = fileContextCache.get(fileHandle);
@@ -306,9 +308,9 @@ void handleCommit(DFSClient dfsClient, FileHandle fileHandle,
WccData fileWcc = new WccData(Nfs3Utils.getWccAttr(preOpAttr), postOpAttr);
COMMIT3Response response = new COMMIT3Response(status, fileWcc,
Nfs3Constant.WRITE_COMMIT_VERF);
+ RpcProgramNfs3.metrics.addCommit(Nfs3Utils.getElapsedTime(startTime));
Nfs3Utils.writeChannelCommit(channel,
- response.serialize(new XDR(), xid, new VerifierNone()),
- xid);
+ response.serialize(new XDR(), xid, new VerifierNone()), xid);
}
/**
diff --git a/hadoop-hdfs-project/hadoop-hdfs-nfs/src/test/java/org/apache/hadoop/hdfs/nfs/nfs3/TestNfs3HttpServer.java b/hadoop-hdfs-project/hadoop-hdfs-nfs/src/test/java/org/apache/hadoop/hdfs/nfs/nfs3/TestNfs3HttpServer.java
new file mode 100644
index 0000000000000..46dbd42f4c97b
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-nfs/src/test/java/org/apache/hadoop/hdfs/nfs/nfs3/TestNfs3HttpServer.java
@@ -0,0 +1,93 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.nfs.nfs3;
+
+import static org.junit.Assert.assertTrue;
+
+import java.io.File;
+import java.net.URL;
+
+import org.apache.hadoop.fs.FileUtil;
+import org.apache.hadoop.hdfs.DFSConfigKeys;
+import org.apache.hadoop.hdfs.DFSTestUtil;
+import org.apache.hadoop.hdfs.MiniDFSCluster;
+import org.apache.hadoop.hdfs.nfs.conf.NfsConfigKeys;
+import org.apache.hadoop.hdfs.nfs.conf.NfsConfiguration;
+import org.apache.hadoop.http.HttpConfig;
+import org.apache.hadoop.security.ssl.KeyStoreTestUtil;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+public class TestNfs3HttpServer {
+ private static final String BASEDIR = System.getProperty("test.build.dir",
+ "target/test-dir") + "/" + TestNfs3HttpServer.class.getSimpleName();
+ private static NfsConfiguration conf = new NfsConfiguration();
+ private static MiniDFSCluster cluster;
+ private static String keystoresDir;
+ private static String sslConfDir;
+
+ @BeforeClass
+ public static void setUp() throws Exception {
+ conf.set(DFSConfigKeys.DFS_HTTP_POLICY_KEY,
+ HttpConfig.Policy.HTTP_AND_HTTPS.name());
+ conf.set(NfsConfigKeys.NFS_HTTP_ADDRESS_KEY, "localhost:0");
+ conf.set(NfsConfigKeys.NFS_HTTPS_ADDRESS_KEY, "localhost:0");
+ // Use emphral port in case tests are running in parallel
+ conf.setInt(NfsConfigKeys.DFS_NFS_SERVER_PORT_KEY, 0);
+ conf.setInt(NfsConfigKeys.DFS_NFS_MOUNTD_PORT_KEY, 0);
+
+ File base = new File(BASEDIR);
+ FileUtil.fullyDelete(base);
+ base.mkdirs();
+ keystoresDir = new File(BASEDIR).getAbsolutePath();
+ sslConfDir = KeyStoreTestUtil.getClasspathDir(TestNfs3HttpServer.class);
+ KeyStoreTestUtil.setupSSLConfig(keystoresDir, sslConfDir, conf, false);
+
+ cluster = new MiniDFSCluster.Builder(conf).numDataNodes(1).build();
+ cluster.waitActive();
+ }
+
+ @AfterClass
+ public static void tearDown() throws Exception {
+ FileUtil.fullyDelete(new File(BASEDIR));
+ if (cluster != null) {
+ cluster.shutdown();
+ }
+ KeyStoreTestUtil.cleanupSSLConfig(keystoresDir, sslConfDir);
+ }
+
+ @Test
+ public void testHttpServer() throws Exception {
+ Nfs3 nfs = new Nfs3(conf);
+ nfs.startServiceInternal(false);
+ RpcProgramNfs3 nfsd = (RpcProgramNfs3) nfs.getRpcProgram();
+ Nfs3HttpServer infoServer = nfsd.getInfoServer();
+
+ String urlRoot = infoServer.getServerURI().toString();
+
+ // Check default servlets.
+ String pageContents = DFSTestUtil.urlGet(new URL(urlRoot + "/jmx"));
+ assertTrue("Bad contents: " + pageContents,
+ pageContents.contains("java.lang:type="));
+ System.out.println("pc:" + pageContents);
+
+ int port = infoServer.getSecurePort();
+ assertTrue("Can't get https port", port > 0);
+ }
+}
diff --git a/hadoop-hdfs-project/hadoop-hdfs-nfs/src/test/resources/core-site.xml b/hadoop-hdfs-project/hadoop-hdfs-nfs/src/test/resources/core-site.xml
index f90ca03c3b6e8..f400bf2ff4317 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-nfs/src/test/resources/core-site.xml
+++ b/hadoop-hdfs-project/hadoop-hdfs-nfs/src/test/resources/core-site.xml
@@ -20,10 +20,24 @@
nfs.server.port
2079
+
+ Specify the port number used by Hadoop NFS.
+ Notice that the default value here is different than the default Hadoop nfs
+ port 2049 specified in hdfs-default.xml. 2049 is also the default port for
+ Linux nfs. The setting here allows starting Hadoop nfs for testing even if
+ nfs server (linux or Hadoop) is aready running on he same host.
+
nfs.mountd.port
4272
+
+ Specify the port number used by Hadoop mount daemon.
+ Notice that the default value here is different than 4242 specified in
+ hdfs-default.xml. This setting allows starting Hadoop nfs mountd for
+ testing even if the Linux or Hadoop mountd is already running on the
+ same host.
+
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 3cd8e15d1e5b0..c38b92ad14a1c 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -272,6 +272,10 @@ Release 2.7.0 - UNRELEASED
HDFS-6982. nntop: topÂ-like tool for name node users.
(Maysam Yabandeh via wang)
+ HDFS-7424. Add web UI for NFS gateway (brandonli)
+
+ HDFS-7449. Add metrics to NFS gateway (brandonli)
+
IMPROVEMENTS
HDFS-7055. Add tracing to DFSInputStream (cmccabe)
@@ -395,8 +399,88 @@ Release 2.7.0 - UNRELEASED
HDFS-7436. Consolidate implementation of concat(). (wheat9)
+ HDFS-7440. Consolidate snapshot related operations in a single class.
+ (wheat9)
+
+ HDFS-6803 Document DFSClient#DFSInputStream expectations reading and preading
+ in concurrent context. (stack via stevel)
+
+ HDFS-7310. Mover can give first priority to local DN if it has target storage type
+ available in local DN. (Vinayakumar B via umamahesh)
+
+ HDFS-7210. Avoid two separate RPC's namenode.append() and namenode.getFileInfo()
+ for an append call from DFSClient. (Vinayakumar B via umamahesh)
+
+ HDFS-7450. Consolidate the implementation of GetFileInfo(), GetListings() and
+ GetContentSummary() into a single class. (wheat9)
+
+ HDFS-7438. Consolidate the implementation of rename() into a single class.
+ (wheat9)
+
+ HDFS-7462. Consolidate implementation of mkdirs() into a single class.
+ (wheat9)
+
+ HDFS-7446. HDFS inotify should have the ability to determine what txid it
+ has read up to (cmccabe)
+
+ HDFS-6735. A minor optimization to avoid pread() be blocked by read()
+ inside the same DFSInputStream (Lars Hofhansl via stack)
+
+ HDFS-7458. Add description to the nfs ports in core-site.xml used by nfs
+ test to avoid confusion (Yongjun Zhang via brandonli)
+
+ HDFS-7468. Moving verify* functions to corresponding classes.
+ (Li Lu via wheat9)
+
+ HDFS-7478. Move org.apache.hadoop.hdfs.server.namenode.NNConf to
+ FSNamesystem. (Li Lu via wheat9)
+
+ HDFS-7474. Avoid resolving path in FSPermissionChecker. (jing9)
+
+ HDFS-7459. Consolidate cache-related implementation in FSNamesystem into
+ a single class. (wheat9)
+
+ HDFS-7476. Consolidate ACL-related operations to a single class.
+ (wheat9 via cnauroth)
+
+ HDFS-7384. 'getfacl' command and 'getAclStatus' output should be in sync.
+ (Vinayakumar B via cnauroth)
+
+ HDFS-7486. Consolidate XAttr-related implementation into a single class.
+ (wheat9)
+
+ HDFS-7498. Simplify the logic in INodesInPath. (jing9)
+
+ HDFS-7463. Simplify FSNamesystem#getBlockLocationsUpdateTimes. (wheat9)
+
+ HDFS-7509. Avoid resolving path multiple times. (jing9)
+
+ HDFS-7426. Change nntop JMX format to be a JSON blob. (wang)
+
+ HDFS-7513. HDFS inotify: add defaultBlockSize to CreateEvent (cmccabe)
+
+ HDFS-7536. Remove unused CryptoCodec in org.apache.hadoop.fs.Hdfs.
+ (Yi Liu via wheat9)
+
+ HDFS-7528. Consolidate symlink-related implementation into a single class.
+ (wheat9)
+
+ HDFS-7531. Improve the concurrent access on FsVolumeList (Lei Xu via Colin
+ P. McCabe)
+
+ HDFS-7373. Clean up temporary files after fsimage transfer failures.
+ (kihwal)
+
+ HDFS-7543. Avoid path resolution when getting FileStatus for audit logs.
+ (wheat9)
+
+ HDFS-7530. Allow renaming of encryption zone roots. (Charles Lamb via wang)
+
OPTIMIZATIONS
+ HDFS-7454. Reduce memory footprint for AclEntries in NameNode.
+ (Vinayakumar B via wheat9)
+
BUG FIXES
HDFS-6741. Improve permission denied message when
@@ -486,6 +570,63 @@ Release 2.7.0 - UNRELEASED
HDFS-7303. NN UI fails to distinguish datanodes on the same host.
(Benoy Antony via wheat9)
+ HDFS-7097. Allow block reports to be processed during checkpointing on
+ standby name node. (kihwal via wang)
+
+ HDFS-7444. convertToBlockUnderConstruction should preserve BlockCollection.
+ (wheat9)
+
+ HDFS-7448 TestBookKeeperHACheckpoints fails in trunk build
+ (Akira Ajisaka via stevel)
+
+ HDFS-7472. Fix typo in message of ReplicaNotFoundException.
+ (Masatake Iwasaki via wheat9)
+
+ HDFS-7473. Document setting dfs.namenode.fs-limits.max-directory-items to 0
+ is invalid. (Akira AJISAKA via cnauroth)
+
+ HDFS-7481. Add ACL indicator to the "Permission Denied" exception.
+ (vinayakumarb)
+
+ HDFS-7502. Fix findbugs warning in hdfs-nfs project.
+ (Brandon Li via wheat9)
+
+ HDFS-5578. [JDK8] Fix Javadoc errors caused by incorrect or illegal tags
+ in doc comments. (Andrew Purtell via wheat9)
+
+ HDFS-7475. Make TestLazyPersistFiles#testLazyPersistBlocksAreSaved
+ deterministic. (Xiaoyu Yao via Arpit Agarwal)
+
+ HDFS-7515. Fix new findbugs warnings in hadoop-hdfs. (wheat9)
+
+ HDFS-7497. Inconsistent report of decommissioning DataNodes between
+ dfsadmin and NameNode webui. (Yongjun Zhang via wang)
+
+ HDFS-7517. Remove redundant non-null checks in FSNamesystem#
+ getBlockLocations. (wheat9)
+
+ HDFS-7514. TestTextCommand fails on Windows. (Arpit Agarwal)
+
+ HDFS-7506. Consolidate implementation of setting inode attributes into a
+ single class. (wheat9)
+
+ HDFS-7516. Fix findbugs warnings in hdfs-nfs project. (brandonli)
+
+ HDFS-6425. Large postponedMisreplicatedBlocks has impact on blockReport
+ latency. (Ming Ma via kihwal)
+
+ HDFS-7494. Checking of closed in DFSInputStream#pread() should be protected
+ by synchronization (Ted Yu via Colin P. McCabe)
+
+ HDFS-7431. log message for InvalidMagicNumberException may be incorrect.
+ (Yi Liu via cnauroth)
+
+ HDFS-7552. Change FsVolumeList toString() to fix
+ TestDataNodeVolumeFailureToleration (Liang Xie via Colin P. McCabe)
+
+ HDFS-7557. Fix spacing for a few keys in DFSConfigKeys.java
+ (Colin P.McCabe)
+
Release 2.6.1 - UNRELEASED
INCOMPATIBLE CHANGES
@@ -504,6 +645,15 @@ Release 2.6.1 - UNRELEASED
HDFS-4882. Prevent the Namenode's LeaseManager from looping forever in
checkLeases (Ravi Prakash via Colin P. McCabe)
+ HDFS-7489. Incorrect locking in FsVolumeList#checkDirs can hang datanodes
+ (Noah Lorang via Colin P. McCabe)
+
+ HDFS-7503. Namenode restart after large deletions can cause slow
+ processReport. (Arpit Agarwal)
+
+ HDFS-7443. Datanode upgrade to BLOCKID_BASED_LAYOUT fails if duplicate
+ block files are present in the same volume (cmccabe)
+
Release 2.6.0 - 2014-11-18
INCOMPATIBLE CHANGES
diff --git a/hadoop-hdfs-project/hadoop-hdfs/dev-support/findbugsExcludeFile.xml b/hadoop-hdfs-project/hadoop-hdfs/dev-support/findbugsExcludeFile.xml
index 2ddc4cc501ad4..dedeeced0ccb9 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/dev-support/findbugsExcludeFile.xml
+++ b/hadoop-hdfs-project/hadoop-hdfs/dev-support/findbugsExcludeFile.xml
@@ -207,4 +207,13 @@
+
+
+
+
+
+
diff --git a/hadoop-hdfs-project/hadoop-hdfs/pom.xml b/hadoop-hdfs-project/hadoop-hdfs/pom.xml
index 2aab073b47809..bad1792546716 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/pom.xml
+++ b/hadoop-hdfs-project/hadoop-hdfs/pom.xml
@@ -252,6 +252,9 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd">
+
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/test/java/org/apache/hadoop/contrib/bkjournal/TestBookKeeperHACheckpoints.java b/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/test/java/org/apache/hadoop/contrib/bkjournal/TestBookKeeperHACheckpoints.java
index 91cab553bcd3a..b74cd7fbd3485 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/test/java/org/apache/hadoop/contrib/bkjournal/TestBookKeeperHACheckpoints.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/test/java/org/apache/hadoop/contrib/bkjournal/TestBookKeeperHACheckpoints.java
@@ -52,7 +52,7 @@ public void setupCluster() throws Exception {
cluster = new MiniDFSCluster.Builder(conf)
.nnTopology(topology)
- .numDataNodes(0)
+ .numDataNodes(1)
.manageNameDfsSharedDirs(false)
.build();
cluster.waitActive();
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs b/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs
index 8ff0ee61f620a..98a89b7ed4881 100755
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs
@@ -180,6 +180,7 @@ case ${COMMAND} in
CLASS='org.apache.hadoop.hdfs.server.namenode.NameNode'
hadoop_debug "Appending HADOOP_NAMENODE_OPTS onto HADOOP_OPTS"
HADOOP_OPTS="${HADOOP_OPTS} ${HADOOP_NAMENODE_OPTS}"
+ hadoop_add_param HADOOP_OPTS hdfs.audit.logger "-Dhdfs.audit.logger=${HDFS_AUDIT_LOGGER}"
;;
nfs3)
supportdaemonization="true"
@@ -221,6 +222,7 @@ case ${COMMAND} in
CLASS='org.apache.hadoop.hdfs.server.namenode.SecondaryNameNode'
hadoop_debug "Appending HADOOP_SECONDARYNAMENODE_OPTS onto HADOOP_OPTS"
HADOOP_OPTS="${HADOOP_OPTS} ${HADOOP_SECONDARYNAMENODE_OPTS}"
+ hadoop_add_param HADOOP_OPTS hdfs.audit.logger "-Dhdfs.audit.logger=${HDFS_AUDIT_LOGGER}"
;;
snapshotDiff)
CLASS=org.apache.hadoop.hdfs.tools.snapshot.SnapshotDiff
@@ -245,6 +247,8 @@ case ${COMMAND} in
;;
esac
+hadoop_verify_user "${COMMAND}"
+
if [[ -n "${secure_service}" ]]; then
HADOOP_SECURE_USER="${secure_user}"
hadoop_verify_secure_prereq
@@ -271,7 +275,6 @@ if [[ "${HADOOP_DAEMON_MODE}" != "default" ]]; then
fi
fi
-hadoop_add_param HADOOP_OPTS Xmx "${JAVA_HEAP_MAX}"
hadoop_finalize
if [[ -n "${supportdaemonization}" ]]; then
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs-config.sh b/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs-config.sh
index bb17aa1db68dd..ed7b0420ae6cb 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs-config.sh
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs-config.sh
@@ -56,11 +56,12 @@ function hadoop_subproject_init
HADOOP_IDENT_STRING="${HADOOP_HDFS_IDENT_STRING:-$HADOOP_IDENT_STRING}"
HADOOP_HDFS_IDENT_STRING="${HADOOP_IDENT_STRING}"
+
# turn on the defaults
-
- export HADOOP_NAMENODE_OPTS=${HADOOP_NAMENODE_OPTS:-"-Dhadoop.security.logger=INFO,RFAS -Dhdfs.audit.logger=INFO,NullAppender"}
- export HADOOP_SECONDARYNAMENODE_OPTS=${HADOOP_SECONDARYNAMENODE_OPTS:-"-Dhadoop.security.logger=INFO,RFAS -Dhdfs.audit.logger=INFO,NullAppender"}
+ export HDFS_AUDIT_LOGGER=${HDFS_AUDIT_LOGGER:-INFO,NullAppender}
+ export HADOOP_NAMENODE_OPTS=${HADOOP_NAMENODE_OPTS:-"-Dhadoop.security.logger=INFO,RFAS"}
+ export HADOOP_SECONDARYNAMENODE_OPTS=${HADOOP_SECONDARYNAMENODE_OPTS:-"-Dhadoop.security.logger=INFO,RFAS"}
export HADOOP_DATANODE_OPTS=${HADOOP_DATANODE_OPTS:-"-Dhadoop.security.logger=ERROR,RFAS"}
export HADOOP_DN_SECURE_EXTRA_OPTS=${HADOOP_DN_SECURE_EXTRA_OPTS:-"-jvm server"}
export HADOOP_NFS3_SECURE_EXTRA_OPTS=${HADOOP_NFS3_SECURE_EXTRA_OPTS:-"-jvm server"}
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/fs/Hdfs.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/fs/Hdfs.java
index 111630c5f59dd..1b9f515deb8f6 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/fs/Hdfs.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/fs/Hdfs.java
@@ -30,7 +30,6 @@
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.crypto.CryptoCodec;
import org.apache.hadoop.fs.permission.AclEntry;
import org.apache.hadoop.fs.permission.AclStatus;
import org.apache.hadoop.fs.permission.FsAction;
@@ -61,7 +60,6 @@
public class Hdfs extends AbstractFileSystem {
DFSClient dfs;
- final CryptoCodec factory;
private boolean verifyChecksum = true;
static {
@@ -88,7 +86,6 @@ public class Hdfs extends AbstractFileSystem {
}
this.dfs = new DFSClient(theUri, conf, getStatistics());
- this.factory = CryptoCodec.getInstance(conf);
}
@Override
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/BlockReaderFactory.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/BlockReaderFactory.java
index 13e0a5226854a..7e40917fff009 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/BlockReaderFactory.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/BlockReaderFactory.java
@@ -668,7 +668,6 @@ private BlockReader getRemoteBlockReaderFromTcp() throws IOException {
Peer peer = null;
try {
curPeer = nextTcpPeer();
- if (curPeer == null) break;
if (curPeer.fromCache) remainingCacheTries--;
peer = curPeer.peer;
blockReader = getRemoteBlockReader(peer);
@@ -699,7 +698,6 @@ private BlockReader getRemoteBlockReaderFromTcp() throws IOException {
}
}
}
- return null;
}
public static class BlockReaderPeer {
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
index 056a1b3beeca7..62db1fac8ccc8 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
@@ -163,6 +163,7 @@
import org.apache.hadoop.hdfs.protocol.HdfsConstants.RollingUpgradeAction;
import org.apache.hadoop.hdfs.protocol.HdfsConstants.SafeModeAction;
import org.apache.hadoop.hdfs.protocol.HdfsFileStatus;
+import org.apache.hadoop.hdfs.protocol.LastBlockWithStatus;
import org.apache.hadoop.hdfs.protocol.LocatedBlock;
import org.apache.hadoop.hdfs.protocol.LocatedBlocks;
import org.apache.hadoop.hdfs.protocol.NSQuotaExceededException;
@@ -1770,9 +1771,9 @@ public String getLinkTarget(String path) throws IOException {
/** Method to get stream returned by append call */
private DFSOutputStream callAppend(String src,
int buffersize, Progressable progress) throws IOException {
- LocatedBlock lastBlock = null;
+ LastBlockWithStatus lastBlockWithStatus = null;
try {
- lastBlock = namenode.append(src, clientName);
+ lastBlockWithStatus = namenode.append(src, clientName);
} catch(RemoteException re) {
throw re.unwrapRemoteException(AccessControlException.class,
FileNotFoundException.class,
@@ -1782,9 +1783,10 @@ private DFSOutputStream callAppend(String src,
UnresolvedPathException.class,
SnapshotAccessControlException.class);
}
- HdfsFileStatus newStat = getFileInfo(src);
+ HdfsFileStatus newStat = lastBlockWithStatus.getFileStatus();
return DFSOutputStream.newStreamForAppend(this, src, buffersize, progress,
- lastBlock, newStat, dfsClientConf.createChecksum());
+ lastBlockWithStatus.getLastBlock(), newStat,
+ dfsClientConf.createChecksum());
}
/**
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
index 78cae9cb258ba..fb958f1008e55 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
@@ -325,6 +325,10 @@ public class DFSConfigKeys extends CommonConfigurationKeys {
public static final String DFS_NAMENODE_USE_STALE_DATANODE_FOR_WRITE_RATIO_KEY = "dfs.namenode.write.stale.datanode.ratio";
public static final float DFS_NAMENODE_USE_STALE_DATANODE_FOR_WRITE_RATIO_DEFAULT = 0.5f;
+ // Number of blocks to rescan for each iteration of postponedMisreplicatedBlocks.
+ public static final String DFS_NAMENODE_BLOCKS_PER_POSTPONEDBLOCKS_RESCAN_KEY = "dfs.namenode.blocks.per.postponedblocks.rescan";
+ public static final long DFS_NAMENODE_BLOCKS_PER_POSTPONEDBLOCKS_RESCAN_KEY_DEFAULT = 10000;
+
// Replication monitoring related keys
public static final String DFS_NAMENODE_INVALIDATE_WORK_PCT_PER_ITERATION =
"dfs.namenode.invalidate.work.pct.per.iteration";
@@ -723,17 +727,17 @@ public class DFSConfigKeys extends CommonConfigurationKeys {
"dfs.client.hedged.read.threadpool.size";
public static final int DEFAULT_DFSCLIENT_HEDGED_READ_THREADPOOL_SIZE = 0;
- // Slow io warning log threshold settings for dfsclient and datanode.
- public static final String DFS_CLIENT_SLOW_IO_WARNING_THRESHOLD_KEY =
- "dfs.client.slow.io.warning.threshold.ms";
- public static final long DFS_CLIENT_SLOW_IO_WARNING_THRESHOLD_DEFAULT = 30000;
- public static final String DFS_DATANODE_SLOW_IO_WARNING_THRESHOLD_KEY =
- "dfs.datanode.slow.io.warning.threshold.ms";
- public static final long DFS_DATANODE_SLOW_IO_WARNING_THRESHOLD_DEFAULT = 300;
-
- public static final String DFS_DATANODE_BLOCK_ID_LAYOUT_UPGRADE_THREADS_KEY =
- "dfs.datanode.block.id.layout.upgrade.threads";
- public static final int DFS_DATANODE_BLOCK_ID_LAYOUT_UPGRADE_THREADS = 12;
+ // Slow io warning log threshold settings for dfsclient and datanode.
+ public static final String DFS_CLIENT_SLOW_IO_WARNING_THRESHOLD_KEY =
+ "dfs.client.slow.io.warning.threshold.ms";
+ public static final long DFS_CLIENT_SLOW_IO_WARNING_THRESHOLD_DEFAULT = 30000;
+ public static final String DFS_DATANODE_SLOW_IO_WARNING_THRESHOLD_KEY =
+ "dfs.datanode.slow.io.warning.threshold.ms";
+ public static final long DFS_DATANODE_SLOW_IO_WARNING_THRESHOLD_DEFAULT = 300;
+
+ public static final String DFS_DATANODE_BLOCK_ID_LAYOUT_UPGRADE_THREADS_KEY =
+ "dfs.datanode.block.id.layout.upgrade.threads";
+ public static final int DFS_DATANODE_BLOCK_ID_LAYOUT_UPGRADE_THREADS = 12;
public static final String DFS_NAMENODE_INOTIFY_MAX_EVENTS_PER_RPC_KEY =
"dfs.namenode.inotify.max.events.per.rpc";
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSInotifyEventInputStream.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSInotifyEventInputStream.java
index 73c5f55a43b07..83b92b95387f5 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSInotifyEventInputStream.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSInotifyEventInputStream.java
@@ -19,11 +19,10 @@
package org.apache.hadoop.hdfs;
import com.google.common.collect.Iterators;
-import com.google.common.util.concurrent.UncheckedExecutionException;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
-import org.apache.hadoop.hdfs.inotify.Event;
-import org.apache.hadoop.hdfs.inotify.EventsList;
+import org.apache.hadoop.hdfs.inotify.EventBatch;
+import org.apache.hadoop.hdfs.inotify.EventBatchList;
import org.apache.hadoop.hdfs.inotify.MissingEventsException;
import org.apache.hadoop.hdfs.protocol.ClientProtocol;
import org.apache.hadoop.util.Time;
@@ -33,13 +32,7 @@
import java.io.IOException;
import java.util.Iterator;
import java.util.Random;
-import java.util.concurrent.Callable;
-import java.util.concurrent.ExecutionException;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
-import java.util.concurrent.Future;
import java.util.concurrent.TimeUnit;
-import java.util.concurrent.TimeoutException;
/**
* Stream for reading inotify events. DFSInotifyEventInputStreams should not
@@ -52,7 +45,7 @@ public class DFSInotifyEventInputStream {
.class);
private final ClientProtocol namenode;
- private Iterator it;
+ private Iterator it;
private long lastReadTxid;
/**
* The most recent txid the NameNode told us it has sync'ed -- helps us
@@ -78,22 +71,22 @@ public class DFSInotifyEventInputStream {
}
/**
- * Returns the next event in the stream or null if no new events are currently
- * available.
+ * Returns the next batch of events in the stream or null if no new
+ * batches are currently available.
*
* @throws IOException because of network error or edit log
* corruption. Also possible if JournalNodes are unresponsive in the
* QJM setting (even one unresponsive JournalNode is enough in rare cases),
* so catching this exception and retrying at least a few times is
* recommended.
- * @throws MissingEventsException if we cannot return the next event in the
- * stream because the data for the event (and possibly some subsequent events)
- * has been deleted (generally because this stream is a very large number of
- * events behind the current state of the NameNode). It is safe to continue
- * reading from the stream after this exception is thrown -- the next
- * available event will be returned.
+ * @throws MissingEventsException if we cannot return the next batch in the
+ * stream because the data for the events (and possibly some subsequent
+ * events) has been deleted (generally because this stream is a very large
+ * number of transactions behind the current state of the NameNode). It is
+ * safe to continue reading from the stream after this exception is thrown
+ * The next available batch of events will be returned.
*/
- public Event poll() throws IOException, MissingEventsException {
+ public EventBatch poll() throws IOException, MissingEventsException {
// need to keep retrying until the NN sends us the latest committed txid
if (lastReadTxid == -1) {
LOG.debug("poll(): lastReadTxid is -1, reading current txid from NN");
@@ -101,14 +94,14 @@ public Event poll() throws IOException, MissingEventsException {
return null;
}
if (!it.hasNext()) {
- EventsList el = namenode.getEditsFromTxid(lastReadTxid + 1);
+ EventBatchList el = namenode.getEditsFromTxid(lastReadTxid + 1);
if (el.getLastTxid() != -1) {
// we only want to set syncTxid when we were actually able to read some
// edits on the NN -- otherwise it will seem like edits are being
// generated faster than we can read them when the problem is really
// that we are temporarily unable to read edits
syncTxid = el.getSyncTxid();
- it = el.getEvents().iterator();
+ it = el.getBatches().iterator();
long formerLastReadTxid = lastReadTxid;
lastReadTxid = el.getLastTxid();
if (el.getFirstTxid() != formerLastReadTxid + 1) {
@@ -131,18 +124,18 @@ public Event poll() throws IOException, MissingEventsException {
}
/**
- * Return a estimate of how many events behind the NameNode's current state
- * this stream is. Clients should periodically call this method and check if
- * its result is steadily increasing, which indicates that they are falling
- * behind (i.e. events are being generated faster than the client is reading
- * them). If a client falls too far behind events may be deleted before the
- * client can read them.
+ * Return a estimate of how many transaction IDs behind the NameNode's
+ * current state this stream is. Clients should periodically call this method
+ * and check if its result is steadily increasing, which indicates that they
+ * are falling behind (i.e. transaction are being generated faster than the
+ * client is reading them). If a client falls too far behind events may be
+ * deleted before the client can read them.
*
* A return value of -1 indicates that an estimate could not be produced, and
* should be ignored. The value returned by this method is really only useful
* when compared to previous or subsequent returned values.
*/
- public long getEventsBehindEstimate() {
+ public long getTxidsBehindEstimate() {
if (syncTxid == 0) {
return -1;
} else {
@@ -155,8 +148,8 @@ public long getEventsBehindEstimate() {
}
/**
- * Returns the next event in the stream, waiting up to the specified amount of
- * time for a new event. Returns null if a new event is not available at the
+ * Returns the next event batch in the stream, waiting up to the specified
+ * amount of time for a new batch. Returns null if one is not available at the
* end of the specified amount of time. The time before the method returns may
* exceed the specified amount of time by up to the time required for an RPC
* to the NameNode.
@@ -168,12 +161,12 @@ public long getEventsBehindEstimate() {
* see {@link DFSInotifyEventInputStream#poll()}
* @throws InterruptedException if the calling thread is interrupted
*/
- public Event poll(long time, TimeUnit tu) throws IOException,
+ public EventBatch poll(long time, TimeUnit tu) throws IOException,
InterruptedException, MissingEventsException {
long initialTime = Time.monotonicNow();
long totalWait = TimeUnit.MILLISECONDS.convert(time, tu);
long nextWait = INITIAL_WAIT_MS;
- Event next = null;
+ EventBatch next = null;
while ((next = poll()) == null) {
long timeLeft = totalWait - (Time.monotonicNow() - initialTime);
if (timeLeft <= 0) {
@@ -193,17 +186,17 @@ public Event poll(long time, TimeUnit tu) throws IOException,
}
/**
- * Returns the next event in the stream, waiting indefinitely if a new event
- * is not immediately available.
+ * Returns the next batch of events in the stream, waiting indefinitely if
+ * a new batch is not immediately available.
*
* @throws IOException see {@link DFSInotifyEventInputStream#poll()}
* @throws MissingEventsException see
* {@link DFSInotifyEventInputStream#poll()}
* @throws InterruptedException if the calling thread is interrupted
*/
- public Event take() throws IOException, InterruptedException,
+ public EventBatch take() throws IOException, InterruptedException,
MissingEventsException {
- Event next = null;
+ EventBatch next = null;
int nextWaitMin = INITIAL_WAIT_MS;
while ((next = poll()) == null) {
// sleep for a random period between nextWaitMin and nextWaitMin * 2
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java
index 9794eeca3c038..ed46b16d3404e 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java
@@ -41,6 +41,7 @@
import java.util.concurrent.ExecutorCompletionService;
import java.util.concurrent.Future;
import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
import org.apache.commons.io.IOUtils;
import org.apache.hadoop.classification.InterfaceAudience;
@@ -90,19 +91,34 @@ public class DFSInputStream extends FSInputStream
public static boolean tcpReadsDisabledForTesting = false;
private long hedgedReadOpsLoopNumForTesting = 0;
private final DFSClient dfsClient;
- private boolean closed = false;
+ private AtomicBoolean closed = new AtomicBoolean(false);
private final String src;
- private BlockReader blockReader = null;
private final boolean verifyChecksum;
- private LocatedBlocks locatedBlocks = null;
- private long lastBlockBeingWrittenLength = 0;
- private FileEncryptionInfo fileEncryptionInfo = null;
+
+ // state by stateful read only:
+ // (protected by lock on this)
+ /////
private DatanodeInfo currentNode = null;
private LocatedBlock currentLocatedBlock = null;
private long pos = 0;
private long blockEnd = -1;
+ private BlockReader blockReader = null;
+ ////
+
+ // state shared by stateful and positional read:
+ // (protected by lock on infoLock)
+ ////
+ private LocatedBlocks locatedBlocks = null;
+ private long lastBlockBeingWrittenLength = 0;
+ private FileEncryptionInfo fileEncryptionInfo = null;
private CachingStrategy cachingStrategy;
+ ////
+
private final ReadStatistics readStatistics = new ReadStatistics();
+ // lock for state shared between read and pread
+ // Note: Never acquire a lock on with this lock held to avoid deadlocks
+ // (it's OK to acquire this lock when the lock on is held)
+ private final Object infoLock = new Object();
/**
* Track the ByteBuffers that we have handed out to readers.
@@ -226,35 +242,38 @@ void addToDeadNodes(DatanodeInfo dnInfo) {
this.dfsClient = dfsClient;
this.verifyChecksum = verifyChecksum;
this.src = src;
- this.cachingStrategy =
- dfsClient.getDefaultReadCachingStrategy();
+ synchronized (infoLock) {
+ this.cachingStrategy = dfsClient.getDefaultReadCachingStrategy();
+ }
openInfo();
}
/**
* Grab the open-file info from namenode
*/
- synchronized void openInfo() throws IOException, UnresolvedLinkException {
- lastBlockBeingWrittenLength = fetchLocatedBlocksAndGetLastBlockLength();
- int retriesForLastBlockLength = dfsClient.getConf().retryTimesForGetLastBlockLength;
- while (retriesForLastBlockLength > 0) {
- // Getting last block length as -1 is a special case. When cluster
- // restarts, DNs may not report immediately. At this time partial block
- // locations will not be available with NN for getting the length. Lets
- // retry for 3 times to get the length.
- if (lastBlockBeingWrittenLength == -1) {
- DFSClient.LOG.warn("Last block locations not available. "
- + "Datanodes might not have reported blocks completely."
- + " Will retry for " + retriesForLastBlockLength + " times");
- waitFor(dfsClient.getConf().retryIntervalForGetLastBlockLength);
- lastBlockBeingWrittenLength = fetchLocatedBlocksAndGetLastBlockLength();
- } else {
- break;
+ void openInfo() throws IOException, UnresolvedLinkException {
+ synchronized(infoLock) {
+ lastBlockBeingWrittenLength = fetchLocatedBlocksAndGetLastBlockLength();
+ int retriesForLastBlockLength = dfsClient.getConf().retryTimesForGetLastBlockLength;
+ while (retriesForLastBlockLength > 0) {
+ // Getting last block length as -1 is a special case. When cluster
+ // restarts, DNs may not report immediately. At this time partial block
+ // locations will not be available with NN for getting the length. Lets
+ // retry for 3 times to get the length.
+ if (lastBlockBeingWrittenLength == -1) {
+ DFSClient.LOG.warn("Last block locations not available. "
+ + "Datanodes might not have reported blocks completely."
+ + " Will retry for " + retriesForLastBlockLength + " times");
+ waitFor(dfsClient.getConf().retryIntervalForGetLastBlockLength);
+ lastBlockBeingWrittenLength = fetchLocatedBlocksAndGetLastBlockLength();
+ } else {
+ break;
+ }
+ retriesForLastBlockLength--;
+ }
+ if (retriesForLastBlockLength == 0) {
+ throw new IOException("Could not obtain the last block locations.");
}
- retriesForLastBlockLength--;
- }
- if (retriesForLastBlockLength == 0) {
- throw new IOException("Could not obtain the last block locations.");
}
}
@@ -306,7 +325,6 @@ private long fetchLocatedBlocksAndGetLastBlockLength() throws IOException {
fileEncryptionInfo = locatedBlocks.getFileEncryptionInfo();
- currentNode = null;
return lastBlockBeingWrittenLength;
}
@@ -359,21 +377,25 @@ private long readBlockLength(LocatedBlock locatedblock) throws IOException {
throw new IOException("Cannot obtain block length for " + locatedblock);
}
- public synchronized long getFileLength() {
- return locatedBlocks == null? 0:
- locatedBlocks.getFileLength() + lastBlockBeingWrittenLength;
+ public long getFileLength() {
+ synchronized(infoLock) {
+ return locatedBlocks == null? 0:
+ locatedBlocks.getFileLength() + lastBlockBeingWrittenLength;
+ }
}
// Short circuit local reads are forbidden for files that are
// under construction. See HDFS-2757.
- synchronized boolean shortCircuitForbidden() {
- return locatedBlocks.isUnderConstruction();
+ boolean shortCircuitForbidden() {
+ synchronized(infoLock) {
+ return locatedBlocks.isUnderConstruction();
+ }
}
/**
* Returns the datanode from which the stream is currently reading.
*/
- public DatanodeInfo getCurrentDatanode() {
+ public synchronized DatanodeInfo getCurrentDatanode() {
return currentNode;
}
@@ -403,59 +425,67 @@ public synchronized List getAllBlocks() throws IOException {
* @return located block
* @throws IOException
*/
- private synchronized LocatedBlock getBlockAt(long offset,
+ private LocatedBlock getBlockAt(long offset,
boolean updatePosition) throws IOException {
- assert (locatedBlocks != null) : "locatedBlocks is null";
+ synchronized(infoLock) {
+ assert (locatedBlocks != null) : "locatedBlocks is null";
- final LocatedBlock blk;
+ final LocatedBlock blk;
- //check offset
- if (offset < 0 || offset >= getFileLength()) {
- throw new IOException("offset < 0 || offset >= getFileLength(), offset="
- + offset
- + ", updatePosition=" + updatePosition
- + ", locatedBlocks=" + locatedBlocks);
- }
- else if (offset >= locatedBlocks.getFileLength()) {
- // offset to the portion of the last block,
- // which is not known to the name-node yet;
- // getting the last block
- blk = locatedBlocks.getLastLocatedBlock();
- }
- else {
- // search cached blocks first
- int targetBlockIdx = locatedBlocks.findBlock(offset);
- if (targetBlockIdx < 0) { // block is not cached
- targetBlockIdx = LocatedBlocks.getInsertIndex(targetBlockIdx);
- // fetch more blocks
- final LocatedBlocks newBlocks = dfsClient.getLocatedBlocks(src, offset);
- assert (newBlocks != null) : "Could not find target position " + offset;
- locatedBlocks.insertRange(targetBlockIdx, newBlocks.getLocatedBlocks());
+ //check offset
+ if (offset < 0 || offset >= getFileLength()) {
+ throw new IOException("offset < 0 || offset >= getFileLength(), offset="
+ + offset
+ + ", updatePosition=" + updatePosition
+ + ", locatedBlocks=" + locatedBlocks);
+ }
+ else if (offset >= locatedBlocks.getFileLength()) {
+ // offset to the portion of the last block,
+ // which is not known to the name-node yet;
+ // getting the last block
+ blk = locatedBlocks.getLastLocatedBlock();
+ }
+ else {
+ // search cached blocks first
+ int targetBlockIdx = locatedBlocks.findBlock(offset);
+ if (targetBlockIdx < 0) { // block is not cached
+ targetBlockIdx = LocatedBlocks.getInsertIndex(targetBlockIdx);
+ // fetch more blocks
+ final LocatedBlocks newBlocks = dfsClient.getLocatedBlocks(src, offset);
+ assert (newBlocks != null) : "Could not find target position " + offset;
+ locatedBlocks.insertRange(targetBlockIdx, newBlocks.getLocatedBlocks());
+ }
+ blk = locatedBlocks.get(targetBlockIdx);
}
- blk = locatedBlocks.get(targetBlockIdx);
- }
- // update current position
- if (updatePosition) {
- pos = offset;
- blockEnd = blk.getStartOffset() + blk.getBlockSize() - 1;
- currentLocatedBlock = blk;
+ // update current position
+ if (updatePosition) {
+ // synchronized not strictly needed, since we only get here
+ // from synchronized caller methods
+ synchronized(this) {
+ pos = offset;
+ blockEnd = blk.getStartOffset() + blk.getBlockSize() - 1;
+ currentLocatedBlock = blk;
+ }
+ }
+ return blk;
}
- return blk;
}
/** Fetch a block from namenode and cache it */
- private synchronized void fetchBlockAt(long offset) throws IOException {
- int targetBlockIdx = locatedBlocks.findBlock(offset);
- if (targetBlockIdx < 0) { // block is not cached
- targetBlockIdx = LocatedBlocks.getInsertIndex(targetBlockIdx);
- }
- // fetch blocks
- final LocatedBlocks newBlocks = dfsClient.getLocatedBlocks(src, offset);
- if (newBlocks == null) {
- throw new IOException("Could not find target position " + offset);
+ private void fetchBlockAt(long offset) throws IOException {
+ synchronized(infoLock) {
+ int targetBlockIdx = locatedBlocks.findBlock(offset);
+ if (targetBlockIdx < 0) { // block is not cached
+ targetBlockIdx = LocatedBlocks.getInsertIndex(targetBlockIdx);
+ }
+ // fetch blocks
+ final LocatedBlocks newBlocks = dfsClient.getLocatedBlocks(src, offset);
+ if (newBlocks == null) {
+ throw new IOException("Could not find target position " + offset);
+ }
+ locatedBlocks.insertRange(targetBlockIdx, newBlocks.getLocatedBlocks());
}
- locatedBlocks.insertRange(targetBlockIdx, newBlocks.getLocatedBlocks());
}
/**
@@ -467,7 +497,7 @@ private synchronized void fetchBlockAt(long offset) throws IOException {
* @return consequent segment of located blocks
* @throws IOException
*/
- private synchronized List getBlockRange(long offset,
+ private List getBlockRange(long offset,
long length) throws IOException {
// getFileLength(): returns total file length
// locatedBlocks.getFileLength(): returns length of completed blocks
@@ -475,26 +505,27 @@ private synchronized List getBlockRange(long offset,
throw new IOException("Offset: " + offset +
" exceeds file length: " + getFileLength());
}
+ synchronized(infoLock) {
+ final List blocks;
+ final long lengthOfCompleteBlk = locatedBlocks.getFileLength();
+ final boolean readOffsetWithinCompleteBlk = offset < lengthOfCompleteBlk;
+ final boolean readLengthPastCompleteBlk = offset + length > lengthOfCompleteBlk;
- final List blocks;
- final long lengthOfCompleteBlk = locatedBlocks.getFileLength();
- final boolean readOffsetWithinCompleteBlk = offset < lengthOfCompleteBlk;
- final boolean readLengthPastCompleteBlk = offset + length > lengthOfCompleteBlk;
+ if (readOffsetWithinCompleteBlk) {
+ //get the blocks of finalized (completed) block range
+ blocks = getFinalizedBlockRange(offset,
+ Math.min(length, lengthOfCompleteBlk - offset));
+ } else {
+ blocks = new ArrayList(1);
+ }
- if (readOffsetWithinCompleteBlk) {
- //get the blocks of finalized (completed) block range
- blocks = getFinalizedBlockRange(offset,
- Math.min(length, lengthOfCompleteBlk - offset));
- } else {
- blocks = new ArrayList(1);
- }
+ // get the blocks from incomplete block range
+ if (readLengthPastCompleteBlk) {
+ blocks.add(locatedBlocks.getLastLocatedBlock());
+ }
- // get the blocks from incomplete block range
- if (readLengthPastCompleteBlk) {
- blocks.add(locatedBlocks.getLastLocatedBlock());
+ return blocks;
}
-
- return blocks;
}
/**
@@ -502,35 +533,37 @@ private synchronized List getBlockRange(long offset,
* Includes only the complete blocks.
* Fetch them from the namenode if not cached.
*/
- private synchronized List getFinalizedBlockRange(
+ private List getFinalizedBlockRange(
long offset, long length) throws IOException {
- assert (locatedBlocks != null) : "locatedBlocks is null";
- List blockRange = new ArrayList();
- // search cached blocks first
- int blockIdx = locatedBlocks.findBlock(offset);
- if (blockIdx < 0) { // block is not cached
- blockIdx = LocatedBlocks.getInsertIndex(blockIdx);
- }
- long remaining = length;
- long curOff = offset;
- while(remaining > 0) {
- LocatedBlock blk = null;
- if(blockIdx < locatedBlocks.locatedBlockCount())
- blk = locatedBlocks.get(blockIdx);
- if (blk == null || curOff < blk.getStartOffset()) {
- LocatedBlocks newBlocks;
- newBlocks = dfsClient.getLocatedBlocks(src, curOff, remaining);
- locatedBlocks.insertRange(blockIdx, newBlocks.getLocatedBlocks());
- continue;
+ synchronized(infoLock) {
+ assert (locatedBlocks != null) : "locatedBlocks is null";
+ List blockRange = new ArrayList();
+ // search cached blocks first
+ int blockIdx = locatedBlocks.findBlock(offset);
+ if (blockIdx < 0) { // block is not cached
+ blockIdx = LocatedBlocks.getInsertIndex(blockIdx);
+ }
+ long remaining = length;
+ long curOff = offset;
+ while(remaining > 0) {
+ LocatedBlock blk = null;
+ if(blockIdx < locatedBlocks.locatedBlockCount())
+ blk = locatedBlocks.get(blockIdx);
+ if (blk == null || curOff < blk.getStartOffset()) {
+ LocatedBlocks newBlocks;
+ newBlocks = dfsClient.getLocatedBlocks(src, curOff, remaining);
+ locatedBlocks.insertRange(blockIdx, newBlocks.getLocatedBlocks());
+ continue;
+ }
+ assert curOff >= blk.getStartOffset() : "Block not found";
+ blockRange.add(blk);
+ long bytesRead = blk.getStartOffset() + blk.getBlockSize() - curOff;
+ remaining -= bytesRead;
+ curOff += bytesRead;
+ blockIdx++;
}
- assert curOff >= blk.getStartOffset() : "Block not found";
- blockRange.add(blk);
- long bytesRead = blk.getStartOffset() + blk.getBlockSize() - curOff;
- remaining -= bytesRead;
- curOff += bytesRead;
- blockIdx++;
- }
- return blockRange;
+ return blockRange;
+ }
}
/**
@@ -573,6 +606,12 @@ private synchronized DatanodeInfo blockSeekTo(long target) throws IOException {
try {
ExtendedBlock blk = targetBlock.getBlock();
Token accessToken = targetBlock.getBlockToken();
+ CachingStrategy curCachingStrategy;
+ boolean shortCircuitForbidden;
+ synchronized(infoLock) {
+ curCachingStrategy = cachingStrategy;
+ shortCircuitForbidden = shortCircuitForbidden();
+ }
blockReader = new BlockReaderFactory(dfsClient.getConf()).
setInetSocketAddress(targetAddr).
setRemotePeerFactory(dfsClient).
@@ -585,8 +624,8 @@ private synchronized DatanodeInfo blockSeekTo(long target) throws IOException {
setVerifyChecksum(verifyChecksum).
setClientName(dfsClient.clientName).
setLength(blk.getNumBytes() - offsetIntoBlock).
- setCachingStrategy(cachingStrategy).
- setAllowShortCircuitLocalReads(!shortCircuitForbidden()).
+ setCachingStrategy(curCachingStrategy).
+ setAllowShortCircuitLocalReads(!shortCircuitForbidden).
setClientCacheContext(dfsClient.getClientContext()).
setUserGroupInformation(dfsClient.ugi).
setConfiguration(dfsClient.getConfiguration()).
@@ -623,7 +662,8 @@ private synchronized DatanodeInfo blockSeekTo(long target) throws IOException {
*/
@Override
public synchronized void close() throws IOException {
- if (closed) {
+ if (!closed.compareAndSet(false, true)) {
+ DFSClient.LOG.warn("DFSInputStream has been closed already");
return;
}
dfsClient.checkOpen();
@@ -647,7 +687,6 @@ public void accept(ByteBuffer k, Object v) {
blockReader = null;
}
super.close();
- closed = true;
}
@Override
@@ -782,9 +821,9 @@ private synchronized int readBuffer(ReaderStrategy reader, int off, int len,
}
}
- private int readWithStrategy(ReaderStrategy strategy, int off, int len) throws IOException {
+ private synchronized int readWithStrategy(ReaderStrategy strategy, int off, int len) throws IOException {
dfsClient.checkOpen();
- if (closed) {
+ if (closed.get()) {
throw new IOException("Stream closed");
}
Map> corruptedBlockMap
@@ -800,9 +839,11 @@ private int readWithStrategy(ReaderStrategy strategy, int off, int len) throws I
currentNode = blockSeekTo(pos);
}
int realLen = (int) Math.min(len, (blockEnd - pos + 1L));
- if (locatedBlocks.isLastBlockComplete()) {
- realLen = (int) Math.min(realLen,
- locatedBlocks.getFileLength() - pos);
+ synchronized(infoLock) {
+ if (locatedBlocks.isLastBlockComplete()) {
+ realLen = (int) Math.min(realLen,
+ locatedBlocks.getFileLength() - pos);
+ }
}
int result = readBuffer(strategy, off, realLen, corruptedBlockMap);
@@ -1055,8 +1096,8 @@ private void actualGetFromOneDataNode(final DNAddrPair datanode,
// start of the loop.
CachingStrategy curCachingStrategy;
boolean allowShortCircuitLocalReads;
- synchronized (this) {
- block = getBlockAt(block.getStartOffset(), false);
+ block = getBlockAt(block.getStartOffset(), false);
+ synchronized(infoLock) {
curCachingStrategy = cachingStrategy;
allowShortCircuitLocalReads = !shortCircuitForbidden();
}
@@ -1335,7 +1376,7 @@ private int pread(long position, byte[] buffer, int offset, int length)
throws IOException {
// sanity checks
dfsClient.checkOpen();
- if (closed) {
+ if (closed.get()) {
throw new IOException("Stream closed");
}
failures = 0;
@@ -1444,7 +1485,7 @@ public synchronized void seek(long targetPos) throws IOException {
if (targetPos < 0) {
throw new EOFException("Cannot seek to negative offset");
}
- if (closed) {
+ if (closed.get()) {
throw new IOException("Stream is closed!");
}
boolean done = false;
@@ -1488,7 +1529,7 @@ public synchronized void seek(long targetPos) throws IOException {
* Same as {@link #seekToNewSource(long)} except that it does not exclude
* the current datanode and might connect to the same node.
*/
- private synchronized boolean seekToBlockSource(long targetPos)
+ private boolean seekToBlockSource(long targetPos)
throws IOException {
currentNode = blockSeekTo(targetPos);
return true;
@@ -1531,7 +1572,7 @@ public synchronized long getPos() throws IOException {
*/
@Override
public synchronized int available() throws IOException {
- if (closed) {
+ if (closed.get()) {
throw new IOException("Stream closed");
}
@@ -1575,11 +1616,13 @@ public synchronized ReadStatistics getReadStatistics() {
return new ReadStatistics(readStatistics);
}
- public synchronized FileEncryptionInfo getFileEncryptionInfo() {
- return fileEncryptionInfo;
+ public FileEncryptionInfo getFileEncryptionInfo() {
+ synchronized(infoLock) {
+ return fileEncryptionInfo;
+ }
}
- private synchronized void closeCurrentBlockReader() {
+ private void closeCurrentBlockReader() {
if (blockReader == null) return;
// Close the current block reader so that the new caching settings can
// take effect immediately.
@@ -1594,18 +1637,20 @@ private synchronized void closeCurrentBlockReader() {
@Override
public synchronized void setReadahead(Long readahead)
throws IOException {
- this.cachingStrategy =
- new CachingStrategy.Builder(this.cachingStrategy).
- setReadahead(readahead).build();
+ synchronized (infoLock) {
+ this.cachingStrategy =
+ new CachingStrategy.Builder(this.cachingStrategy).setReadahead(readahead).build();
+ }
closeCurrentBlockReader();
}
@Override
public synchronized void setDropBehind(Boolean dropBehind)
throws IOException {
- this.cachingStrategy =
- new CachingStrategy.Builder(this.cachingStrategy).
- setDropBehind(dropBehind).build();
+ synchronized (infoLock) {
+ this.cachingStrategy =
+ new CachingStrategy.Builder(this.cachingStrategy).setDropBehind(dropBehind).build();
+ }
closeCurrentBlockReader();
}
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java
index e574d1d080d8f..67d314302641f 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java
@@ -39,6 +39,7 @@
import java.util.List;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicReference;
import org.apache.hadoop.HadoopIllegalArgumentException;
@@ -241,8 +242,6 @@ private static class Packet {
/**
* Create a new packet.
*
- * @param pktSize maximum size of the packet,
- * including checksum data and actual data.
* @param chunksPerPkt maximum number of chunks per packet.
* @param offsetInBlock offset in bytes into the HDFS block.
*/
@@ -405,7 +404,8 @@ public DatanodeInfo load(DatanodeInfo key) throws Exception {
private String[] favoredNodes;
volatile boolean hasError = false;
volatile int errorIndex = -1;
- volatile int restartingNodeIndex = -1; // Restarting node index
+ // Restarting node index
+ AtomicInteger restartingNodeIndex = new AtomicInteger(-1);
private long restartDeadline = 0; // Deadline of DN restart
private BlockConstructionStage stage; // block construction stage
private long bytesSent = 0; // number of bytes that've been sent
@@ -556,7 +556,7 @@ public void run() {
try {
// process datanode IO errors if any
boolean doSleep = false;
- if (hasError && (errorIndex >= 0 || restartingNodeIndex >= 0)) {
+ if (hasError && (errorIndex >= 0 || restartingNodeIndex.get() >= 0)) {
doSleep = processDatanodeError();
}
@@ -699,7 +699,7 @@ public void run() {
}
} catch (Throwable e) {
// Log warning if there was a real error.
- if (restartingNodeIndex == -1) {
+ if (restartingNodeIndex.get() == -1) {
DFSClient.LOG.warn("DataStreamer Exception", e);
}
if (e instanceof IOException) {
@@ -708,7 +708,7 @@ public void run() {
setLastException(new IOException("DataStreamer Exception: ",e));
}
hasError = true;
- if (errorIndex == -1 && restartingNodeIndex == -1) {
+ if (errorIndex == -1 && restartingNodeIndex.get() == -1) {
// Not a datanode issue
streamerClosed = true;
}
@@ -806,7 +806,7 @@ synchronized void setErrorIndex(int idx) {
/** Set the restarting node index. Called by responder */
synchronized void setRestartingNodeIndex(int idx) {
- restartingNodeIndex = idx;
+ restartingNodeIndex.set(idx);
// If the data streamer has already set the primary node
// bad, clear it. It is likely that the write failed due to
// the DN shutdown. Even if it was a real failure, the pipeline
@@ -821,7 +821,7 @@ synchronized void setRestartingNodeIndex(int idx) {
*/
synchronized void tryMarkPrimaryDatanodeFailed() {
// There should be no existing error and no ongoing restart.
- if ((errorIndex == -1) && (restartingNodeIndex == -1)) {
+ if ((errorIndex == -1) && (restartingNodeIndex.get() == -1)) {
errorIndex = 0;
}
}
@@ -962,7 +962,7 @@ public void run() {
synchronized (dataQueue) {
dataQueue.notifyAll();
}
- if (restartingNodeIndex == -1) {
+ if (restartingNodeIndex.get() == -1) {
DFSClient.LOG.warn("DFSOutputStream ResponseProcessor exception "
+ " for block " + block, e);
}
@@ -1186,7 +1186,7 @@ private boolean setupPipelineForAppendOrRecovery() throws IOException {
// Sleep before reconnect if a dn is restarting.
// This process will be repeated until the deadline or the datanode
// starts back up.
- if (restartingNodeIndex >= 0) {
+ if (restartingNodeIndex.get() >= 0) {
// 4 seconds or the configured deadline period, whichever is shorter.
// This is the retry interval and recovery will be retried in this
// interval until timeout or success.
@@ -1196,7 +1196,7 @@ private boolean setupPipelineForAppendOrRecovery() throws IOException {
Thread.sleep(delay);
} catch (InterruptedException ie) {
lastException.set(new IOException("Interrupted while waiting for " +
- "datanode to restart. " + nodes[restartingNodeIndex]));
+ "datanode to restart. " + nodes[restartingNodeIndex.get()]));
streamerClosed = true;
return false;
}
@@ -1237,21 +1237,21 @@ private boolean setupPipelineForAppendOrRecovery() throws IOException {
setPipeline(newnodes, newStorageTypes, newStorageIDs);
// Just took care of a node error while waiting for a node restart
- if (restartingNodeIndex >= 0) {
+ if (restartingNodeIndex.get() >= 0) {
// If the error came from a node further away than the restarting
// node, the restart must have been complete.
- if (errorIndex > restartingNodeIndex) {
- restartingNodeIndex = -1;
- } else if (errorIndex < restartingNodeIndex) {
+ if (errorIndex > restartingNodeIndex.get()) {
+ restartingNodeIndex.set(-1);
+ } else if (errorIndex < restartingNodeIndex.get()) {
// the node index has shifted.
- restartingNodeIndex--;
+ restartingNodeIndex.decrementAndGet();
} else {
// this shouldn't happen...
assert false;
}
}
- if (restartingNodeIndex == -1) {
+ if (restartingNodeIndex.get() == -1) {
hasError = false;
}
lastException.set(null);
@@ -1293,10 +1293,10 @@ private boolean setupPipelineForAppendOrRecovery() throws IOException {
success = createBlockOutputStream(nodes, storageTypes, newGS, isRecovery);
}
- if (restartingNodeIndex >= 0) {
+ if (restartingNodeIndex.get() >= 0) {
assert hasError == true;
// check errorIndex set above
- if (errorIndex == restartingNodeIndex) {
+ if (errorIndex == restartingNodeIndex.get()) {
// ignore, if came from the restarting node
errorIndex = -1;
}
@@ -1306,8 +1306,8 @@ private boolean setupPipelineForAppendOrRecovery() throws IOException {
}
// expired. declare the restarting node dead
restartDeadline = 0;
- int expiredNodeIndex = restartingNodeIndex;
- restartingNodeIndex = -1;
+ int expiredNodeIndex = restartingNodeIndex.get();
+ restartingNodeIndex.set(-1);
DFSClient.LOG.warn("Datanode did not restart in time: " +
nodes[expiredNodeIndex]);
// Mark the restarting node as failed. If there is any other failed
@@ -1459,7 +1459,7 @@ private boolean createBlockOutputStream(DatanodeInfo[] nodes,
// from the local datanode. Thus it is safe to treat this as a
// regular node error.
if (PipelineAck.isRestartOOBStatus(pipelineStatus) &&
- restartingNodeIndex == -1) {
+ restartingNodeIndex.get() == -1) {
checkRestart = true;
throw new IOException("A datanode is restarting.");
}
@@ -1476,10 +1476,10 @@ private boolean createBlockOutputStream(DatanodeInfo[] nodes,
assert null == blockStream : "Previous blockStream unclosed";
blockStream = out;
result = true; // success
- restartingNodeIndex = -1;
+ restartingNodeIndex.set(-1);
hasError = false;
} catch (IOException ie) {
- if (restartingNodeIndex == -1) {
+ if (restartingNodeIndex.get() == -1) {
DFSClient.LOG.info("Exception in createBlockOutputStream", ie);
}
if (ie instanceof InvalidEncryptionKeyException && refetchEncryptionKey > 0) {
@@ -1511,10 +1511,10 @@ private boolean createBlockOutputStream(DatanodeInfo[] nodes,
if (checkRestart && shouldWaitForRestart(errorIndex)) {
restartDeadline = dfsClient.getConf().datanodeRestartTimeout +
Time.now();
- restartingNodeIndex = errorIndex;
+ restartingNodeIndex.set(errorIndex);
errorIndex = -1;
DFSClient.LOG.info("Waiting for the datanode to be restarted: " +
- nodes[restartingNodeIndex]);
+ nodes[restartingNodeIndex.get()]);
}
hasError = true;
setLastException(ie);
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java
index f1bfcb4fafa75..8b3f5121bf037 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java
@@ -341,15 +341,20 @@ public static byte[] string2Bytes(String str) {
/**
* Given a list of path components returns a path as a UTF8 String
*/
- public static String byteArray2PathString(byte[][] pathComponents) {
+ public static String byteArray2PathString(byte[][] pathComponents,
+ int offset, int length) {
if (pathComponents.length == 0) {
return "";
- } else if (pathComponents.length == 1
+ }
+ Preconditions.checkArgument(offset >= 0 && offset < pathComponents.length);
+ Preconditions.checkArgument(length >= 0 && offset + length <=
+ pathComponents.length);
+ if (pathComponents.length == 1
&& (pathComponents[0] == null || pathComponents[0].length == 0)) {
return Path.SEPARATOR;
}
StringBuilder result = new StringBuilder();
- for (int i = 0; i < pathComponents.length; i++) {
+ for (int i = offset; i < offset + length; i++) {
result.append(new String(pathComponents[i], Charsets.UTF_8));
if (i < pathComponents.length - 1) {
result.append(Path.SEPARATOR_CHAR);
@@ -358,6 +363,10 @@ public static String byteArray2PathString(byte[][] pathComponents) {
return result.toString();
}
+ public static String byteArray2PathString(byte[][] pathComponents) {
+ return byteArray2PathString(pathComponents, 0, pathComponents.length);
+ }
+
/**
* Converts a list of path components into a path using Path.SEPARATOR.
*
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/inotify/Event.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/inotify/Event.java
index e8a34e7c21d0c..5ceff1b6a17df 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/inotify/Event.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/inotify/Event.java
@@ -101,6 +101,7 @@ public static enum INodeType {
private FsPermission perms;
private String symlinkTarget;
private boolean overwrite;
+ private long defaultBlockSize;
public static class Builder {
private INodeType iNodeType;
@@ -112,6 +113,7 @@ public static class Builder {
private FsPermission perms;
private String symlinkTarget;
private boolean overwrite;
+ private long defaultBlockSize = 0;
public Builder iNodeType(INodeType type) {
this.iNodeType = type;
@@ -158,6 +160,11 @@ public Builder overwrite(boolean overwrite) {
return this;
}
+ public Builder defaultBlockSize(long defaultBlockSize) {
+ this.defaultBlockSize = defaultBlockSize;
+ return this;
+ }
+
public CreateEvent build() {
return new CreateEvent(this);
}
@@ -174,6 +181,7 @@ private CreateEvent(Builder b) {
this.perms = b.perms;
this.symlinkTarget = b.symlinkTarget;
this.overwrite = b.overwrite;
+ this.defaultBlockSize = b.defaultBlockSize;
}
public INodeType getiNodeType() {
@@ -220,6 +228,10 @@ public String getSymlinkTarget() {
public boolean getOverwrite() {
return overwrite;
}
+
+ public long getDefaultBlockSize() {
+ return defaultBlockSize;
+ }
}
/**
@@ -398,11 +410,36 @@ public static class RenameEvent extends Event {
private String dstPath;
private long timestamp;
- public RenameEvent(String srcPath, String dstPath, long timestamp) {
+ public static class Builder {
+ private String srcPath;
+ private String dstPath;
+ private long timestamp;
+
+ public Builder srcPath(String srcPath) {
+ this.srcPath = srcPath;
+ return this;
+ }
+
+ public Builder dstPath(String dstPath) {
+ this.dstPath = dstPath;
+ return this;
+ }
+
+ public Builder timestamp(long timestamp) {
+ this.timestamp = timestamp;
+ return this;
+ }
+
+ public RenameEvent build() {
+ return new RenameEvent(this);
+ }
+ }
+
+ private RenameEvent(Builder builder) {
super(EventType.RENAME);
- this.srcPath = srcPath;
- this.dstPath = dstPath;
- this.timestamp = timestamp;
+ this.srcPath = builder.srcPath;
+ this.dstPath = builder.dstPath;
+ this.timestamp = builder.timestamp;
}
public String getSrcPath() {
@@ -427,9 +464,22 @@ public long getTimestamp() {
public static class AppendEvent extends Event {
private String path;
- public AppendEvent(String path) {
+ public static class Builder {
+ private String path;
+
+ public Builder path(String path) {
+ this.path = path;
+ return this;
+ }
+
+ public AppendEvent build() {
+ return new AppendEvent(this);
+ }
+ }
+
+ private AppendEvent(Builder b) {
super(EventType.APPEND);
- this.path = path;
+ this.path = b.path;
}
public String getPath() {
@@ -444,10 +494,29 @@ public static class UnlinkEvent extends Event {
private String path;
private long timestamp;
- public UnlinkEvent(String path, long timestamp) {
+ public static class Builder {
+ private String path;
+ private long timestamp;
+
+ public Builder path(String path) {
+ this.path = path;
+ return this;
+ }
+
+ public Builder timestamp(long timestamp) {
+ this.timestamp = timestamp;
+ return this;
+ }
+
+ public UnlinkEvent build() {
+ return new UnlinkEvent(this);
+ }
+ }
+
+ private UnlinkEvent(Builder builder) {
super(EventType.UNLINK);
- this.path = path;
- this.timestamp = timestamp;
+ this.path = builder.path;
+ this.timestamp = builder.timestamp;
}
public String getPath() {
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/inotify/EventBatch.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/inotify/EventBatch.java
new file mode 100644
index 0000000000000..0ad1070c7e867
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/inotify/EventBatch.java
@@ -0,0 +1,41 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hdfs.inotify;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+
+/**
+ * A batch of events that all happened on the same transaction ID.
+ */
+@InterfaceAudience.Public
+public class EventBatch {
+ private final long txid;
+ private final Event[] events;
+
+ public EventBatch(long txid, Event[] events) {
+ this.txid = txid;
+ this.events = events;
+ }
+
+ public long getTxid() {
+ return txid;
+ }
+
+ public Event[] getEvents() { return events; }
+}
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/inotify/EventsList.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/inotify/EventBatchList.java
similarity index 65%
rename from hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/inotify/EventsList.java
rename to hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/inotify/EventBatchList.java
index 6d02d3c298017..9c9703869b101 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/inotify/EventsList.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/inotify/EventBatchList.java
@@ -23,30 +23,30 @@
import java.util.List;
/**
- * Contains a set of events, the transaction ID in the edit log up to which we
- * read to produce these events, and the first txid we observed when producing
- * these events (the last of which is for the purpose of determining whether we
- * have missed events due to edit deletion). Also contains the most recent txid
- * that the NameNode has sync'ed, so the client can determine how far behind in
- * the edit log it is.
+ * Contains a list of event batches, the transaction ID in the edit log up to
+ * which we read to produce these events, and the first txid we observed when
+ * producing these events (the last of which is for the purpose of determining
+ * whether we have missed events due to edit deletion). Also contains the most
+ * recent txid that the NameNode has sync'ed, so the client can determine how
+ * far behind in the edit log it is.
*/
@InterfaceAudience.Private
-public class EventsList {
- private List events;
+public class EventBatchList {
+ private List batches;
private long firstTxid;
private long lastTxid;
private long syncTxid;
- public EventsList(List events, long firstTxid, long lastTxid,
- long syncTxid) {
- this.events = events;
+ public EventBatchList(List batches, long firstTxid,
+ long lastTxid, long syncTxid) {
+ this.batches = batches;
this.firstTxid = firstTxid;
this.lastTxid = lastTxid;
this.syncTxid = syncTxid;
}
- public List getEvents() {
- return events;
+ public List getBatches() {
+ return batches;
}
public long getFirstTxid() {
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java
index 6506bef52edfa..2301575f0e3fd 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java
@@ -43,7 +43,7 @@
import org.apache.hadoop.fs.permission.FsAction;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.hdfs.DFSConfigKeys;
-import org.apache.hadoop.hdfs.inotify.EventsList;
+import org.apache.hadoop.hdfs.inotify.EventBatchList;
import org.apache.hadoop.hdfs.protocol.HdfsConstants.RollingUpgradeAction;
import org.apache.hadoop.hdfs.security.token.block.DataEncryptionKey;
import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenIdentifier;
@@ -203,7 +203,8 @@ public HdfsFileStatus create(String src, FsPermission masked,
* Append to the end of the file.
* @param src path of the file being created.
* @param clientName name of the current client.
- * @return information about the last partial block if any.
+ * @return wrapper with information about the last partial block and file
+ * status if any
* @throws AccessControlException if permission to append file is
* denied by the system. As usually on the client side the exception will
* be wrapped into {@link org.apache.hadoop.ipc.RemoteException}.
@@ -224,7 +225,7 @@ public HdfsFileStatus create(String src, FsPermission masked,
* @throws UnsupportedOperationException if append is not supported
*/
@AtMostOnce
- public LocatedBlock append(String src, String clientName)
+ public LastBlockWithStatus append(String src, String clientName)
throws AccessControlException, DSQuotaExceededException,
FileNotFoundException, SafeModeException, UnresolvedLinkException,
SnapshotAccessControlException, IOException;
@@ -1407,9 +1408,9 @@ public List listXAttrs(String src)
public long getCurrentEditLogTxid() throws IOException;
/**
- * Get an ordered list of events corresponding to the edit log transactions
- * from txid onwards.
+ * Get an ordered list of batches of events corresponding to the edit log
+ * transactions for txids equal to or greater than txid.
*/
@Idempotent
- public EventsList getEditsFromTxid(long txid) throws IOException;
+ public EventBatchList getEditsFromTxid(long txid) throws IOException;
}
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/LastBlockWithStatus.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/LastBlockWithStatus.java
new file mode 100644
index 0000000000000..1cd80f93a2c26
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/LastBlockWithStatus.java
@@ -0,0 +1,46 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.protocol;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+
+/**
+ * Class to contain Lastblock and HdfsFileStatus for the Append operation
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Evolving
+public class LastBlockWithStatus {
+
+ private final LocatedBlock lastBlock;
+
+ private final HdfsFileStatus fileStatus;
+
+ public LastBlockWithStatus(LocatedBlock lastBlock, HdfsFileStatus fileStatus) {
+ this.lastBlock = lastBlock;
+ this.fileStatus = fileStatus;
+ }
+
+ public LocatedBlock getLastBlock() {
+ return lastBlock;
+ }
+
+ public HdfsFileStatus getFileStatus() {
+ return fileStatus;
+ }
+}
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/LocatedBlocks.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/LocatedBlocks.java
index 436fa14e1c57a..fc739cf711e33 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/LocatedBlocks.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/LocatedBlocks.java
@@ -34,14 +34,17 @@ public class LocatedBlocks {
private final long fileLength;
private final List blocks; // array of blocks with prioritized locations
private final boolean underConstruction;
- private LocatedBlock lastLocatedBlock = null;
- private boolean isLastBlockComplete = false;
- private FileEncryptionInfo fileEncryptionInfo = null;
+ private final LocatedBlock lastLocatedBlock;
+ private final boolean isLastBlockComplete;
+ private final FileEncryptionInfo fileEncryptionInfo;
public LocatedBlocks() {
fileLength = 0;
blocks = null;
underConstruction = false;
+ lastLocatedBlock = null;
+ isLastBlockComplete = false;
+ fileEncryptionInfo = null;
}
public LocatedBlocks(long flength, boolean isUnderConstuction,
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/datatransfer/sasl/InvalidMagicNumberException.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/datatransfer/sasl/InvalidMagicNumberException.java
index eb494a87e171e..cc9b0c69807c9 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/datatransfer/sasl/InvalidMagicNumberException.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/datatransfer/sasl/InvalidMagicNumberException.java
@@ -31,14 +31,26 @@
public class InvalidMagicNumberException extends IOException {
private static final long serialVersionUID = 1L;
+ private final boolean handshake4Encryption;
/**
* Creates a new InvalidMagicNumberException.
*
* @param magicNumber expected value
*/
- public InvalidMagicNumberException(int magicNumber) {
+ public InvalidMagicNumberException(final int magicNumber,
+ final boolean handshake4Encryption) {
super(String.format("Received %x instead of %x from client.",
magicNumber, SASL_TRANSFER_MAGIC_NUMBER));
+ this.handshake4Encryption = handshake4Encryption;
+ }
+
+ /**
+ * Return true if it's handshake for encryption
+ *
+ * @return boolean true if it's handshake for encryption
+ */
+ public boolean isHandshake4Encryption() {
+ return handshake4Encryption;
}
}
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/datatransfer/sasl/SaslDataTransferServer.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/datatransfer/sasl/SaslDataTransferServer.java
index 1d2b30b88bba8..f060beb3475c1 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/datatransfer/sasl/SaslDataTransferServer.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/datatransfer/sasl/SaslDataTransferServer.java
@@ -357,7 +357,8 @@ private IOStreamPair doSaslHandshake(OutputStream underlyingOut,
int magicNumber = in.readInt();
if (magicNumber != SASL_TRANSFER_MAGIC_NUMBER) {
- throw new InvalidMagicNumberException(magicNumber);
+ throw new InvalidMagicNumberException(magicNumber,
+ dnConf.getEncryptDataTransfer());
}
try {
// step 1
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolServerSideTranslatorPB.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolServerSideTranslatorPB.java
index a92d4555e6eff..5b6609bfbbee8 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolServerSideTranslatorPB.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolServerSideTranslatorPB.java
@@ -35,6 +35,7 @@
import org.apache.hadoop.hdfs.protocol.DirectoryListing;
import org.apache.hadoop.hdfs.protocol.EncryptionZone;
import org.apache.hadoop.hdfs.protocol.HdfsFileStatus;
+import org.apache.hadoop.hdfs.protocol.LastBlockWithStatus;
import org.apache.hadoop.hdfs.protocol.LocatedBlock;
import org.apache.hadoop.hdfs.protocol.LocatedBlocks;
import org.apache.hadoop.hdfs.protocol.RollingUpgradeInfo;
@@ -247,9 +248,6 @@ public class ClientNamenodeProtocolServerSideTranslatorPB implements
private static final CreateResponseProto VOID_CREATE_RESPONSE =
CreateResponseProto.newBuilder().build();
- private static final AppendResponseProto VOID_APPEND_RESPONSE =
- AppendResponseProto.newBuilder().build();
-
private static final SetPermissionResponseProto VOID_SET_PERM_RESPONSE =
SetPermissionResponseProto.newBuilder().build();
@@ -407,17 +405,21 @@ public CreateResponseProto create(RpcController controller,
throw new ServiceException(e);
}
}
-
+
@Override
public AppendResponseProto append(RpcController controller,
AppendRequestProto req) throws ServiceException {
try {
- LocatedBlock result = server.append(req.getSrc(), req.getClientName());
- if (result != null) {
- return AppendResponseProto.newBuilder()
- .setBlock(PBHelper.convert(result)).build();
+ LastBlockWithStatus result = server.append(req.getSrc(),
+ req.getClientName());
+ AppendResponseProto.Builder builder = AppendResponseProto.newBuilder();
+ if (result.getLastBlock() != null) {
+ builder.setBlock(PBHelper.convert(result.getLastBlock()));
+ }
+ if (result.getFileStatus() != null) {
+ builder.setStat(PBHelper.convert(result.getFileStatus()));
}
- return VOID_APPEND_RESPONSE;
+ return builder.build();
} catch (IOException e) {
throw new ServiceException(e);
}
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolTranslatorPB.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolTranslatorPB.java
index 077a3e9489cc3..58049204fc4ab 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolTranslatorPB.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolTranslatorPB.java
@@ -25,6 +25,7 @@
import java.util.List;
import com.google.common.collect.Lists;
+
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.crypto.CipherSuite;
@@ -44,7 +45,7 @@
import org.apache.hadoop.fs.permission.AclStatus;
import org.apache.hadoop.fs.permission.FsAction;
import org.apache.hadoop.fs.permission.FsPermission;
-import org.apache.hadoop.hdfs.inotify.EventsList;
+import org.apache.hadoop.hdfs.inotify.EventBatchList;
import org.apache.hadoop.hdfs.protocol.AlreadyBeingCreatedException;
import org.apache.hadoop.hdfs.protocol.BlockStoragePolicy;
import org.apache.hadoop.hdfs.protocol.CacheDirectiveEntry;
@@ -63,6 +64,7 @@
import org.apache.hadoop.hdfs.protocol.HdfsConstants.RollingUpgradeAction;
import org.apache.hadoop.hdfs.protocol.HdfsConstants.SafeModeAction;
import org.apache.hadoop.hdfs.protocol.HdfsFileStatus;
+import org.apache.hadoop.hdfs.protocol.LastBlockWithStatus;
import org.apache.hadoop.hdfs.protocol.LocatedBlock;
import org.apache.hadoop.hdfs.protocol.LocatedBlocks;
import org.apache.hadoop.hdfs.protocol.NSQuotaExceededException;
@@ -188,7 +190,6 @@
import com.google.protobuf.ByteString;
import com.google.protobuf.ServiceException;
-
import static org.apache.hadoop.fs.BatchedRemoteIterator.BatchedListEntries;
import static org.apache.hadoop.hdfs.protocol.proto.EncryptionZonesProtos
.EncryptionZoneProto;
@@ -301,7 +302,7 @@ public HdfsFileStatus create(String src, FsPermission masked,
}
@Override
- public LocatedBlock append(String src, String clientName)
+ public LastBlockWithStatus append(String src, String clientName)
throws AccessControlException, DSQuotaExceededException,
FileNotFoundException, SafeModeException, UnresolvedLinkException,
IOException {
@@ -311,7 +312,11 @@ public LocatedBlock append(String src, String clientName)
.build();
try {
AppendResponseProto res = rpcProxy.append(null, req);
- return res.hasBlock() ? PBHelper.convert(res.getBlock()) : null;
+ LocatedBlock lastBlock = res.hasBlock() ? PBHelper
+ .convert(res.getBlock()) : null;
+ HdfsFileStatus stat = (res.hasStat()) ? PBHelper.convert(res.getStat())
+ : null;
+ return new LastBlockWithStatus(lastBlock, stat);
} catch (ServiceException e) {
throw ProtobufHelper.getRemoteException(e);
}
@@ -1480,7 +1485,7 @@ public long getCurrentEditLogTxid() throws IOException {
}
@Override
- public EventsList getEditsFromTxid(long txid) throws IOException {
+ public EventBatchList getEditsFromTxid(long txid) throws IOException {
GetEditsFromTxidRequestProto req = GetEditsFromTxidRequestProto.newBuilder()
.setTxid(txid).build();
try {
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java
index 53ea6cf823d27..ee6d58cbfae99 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java
@@ -46,11 +46,12 @@
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.ha.HAServiceProtocol.HAServiceState;
import org.apache.hadoop.ha.proto.HAServiceProtocolProtos;
+import org.apache.hadoop.hdfs.inotify.EventBatch;
import org.apache.hadoop.hdfs.protocol.BlockStoragePolicy;
import org.apache.hadoop.hdfs.DFSUtil;
import org.apache.hadoop.hdfs.StorageType;
import org.apache.hadoop.hdfs.inotify.Event;
-import org.apache.hadoop.hdfs.inotify.EventsList;
+import org.apache.hadoop.hdfs.inotify.EventBatchList;
import org.apache.hadoop.hdfs.protocol.Block;
import org.apache.hadoop.hdfs.protocol.CacheDirectiveEntry;
import org.apache.hadoop.hdfs.protocol.CacheDirectiveInfo;
@@ -2277,15 +2278,24 @@ public static List convertAclEntry(List aclSpec) {
public static AclStatus convert(GetAclStatusResponseProto e) {
AclStatusProto r = e.getResult();
- return new AclStatus.Builder().owner(r.getOwner()).group(r.getGroup())
- .stickyBit(r.getSticky())
- .addEntries(convertAclEntry(r.getEntriesList())).build();
+ AclStatus.Builder builder = new AclStatus.Builder();
+ builder.owner(r.getOwner()).group(r.getGroup()).stickyBit(r.getSticky())
+ .addEntries(convertAclEntry(r.getEntriesList()));
+ if (r.hasPermission()) {
+ builder.setPermission(convert(r.getPermission()));
+ }
+ return builder.build();
}
public static GetAclStatusResponseProto convert(AclStatus e) {
- AclStatusProto r = AclStatusProto.newBuilder().setOwner(e.getOwner())
+ AclStatusProto.Builder builder = AclStatusProto.newBuilder();
+ builder.setOwner(e.getOwner())
.setGroup(e.getGroup()).setSticky(e.isStickyBit())
- .addAllEntries(convertAclEntryProto(e.getEntries())).build();
+ .addAllEntries(convertAclEntryProto(e.getEntries()));
+ if (e.getPermission() != null) {
+ builder.setPermission(convert(e.getPermission()));
+ }
+ AclStatusProto r = builder.build();
return GetAclStatusResponseProto.newBuilder().setResult(r).build();
}
@@ -2517,173 +2527,206 @@ private static InotifyProtos.INodeType createTypeConvert(Event.CreateEvent.INode
}
}
- public static EventsList convert(GetEditsFromTxidResponseProto resp) throws
+ public static EventBatchList convert(GetEditsFromTxidResponseProto resp) throws
IOException {
- List events = Lists.newArrayList();
- for (InotifyProtos.EventProto p : resp.getEventsList().getEventsList()) {
- switch(p.getType()) {
- case EVENT_CLOSE:
- InotifyProtos.CloseEventProto close =
- InotifyProtos.CloseEventProto.parseFrom(p.getContents());
- events.add(new Event.CloseEvent(close.getPath(), close.getFileSize(),
- close.getTimestamp()));
- break;
- case EVENT_CREATE:
- InotifyProtos.CreateEventProto create =
- InotifyProtos.CreateEventProto.parseFrom(p.getContents());
- events.add(new Event.CreateEvent.Builder()
- .iNodeType(createTypeConvert(create.getType()))
- .path(create.getPath())
- .ctime(create.getCtime())
- .ownerName(create.getOwnerName())
- .groupName(create.getGroupName())
- .perms(convert(create.getPerms()))
- .replication(create.getReplication())
- .symlinkTarget(create.getSymlinkTarget().isEmpty() ? null :
- create.getSymlinkTarget())
- .overwrite(create.getOverwrite()).build());
- break;
- case EVENT_METADATA:
- InotifyProtos.MetadataUpdateEventProto meta =
- InotifyProtos.MetadataUpdateEventProto.parseFrom(p.getContents());
- events.add(new Event.MetadataUpdateEvent.Builder()
- .path(meta.getPath())
- .metadataType(metadataUpdateTypeConvert(meta.getType()))
- .mtime(meta.getMtime())
- .atime(meta.getAtime())
- .replication(meta.getReplication())
- .ownerName(
- meta.getOwnerName().isEmpty() ? null : meta.getOwnerName())
- .groupName(
- meta.getGroupName().isEmpty() ? null : meta.getGroupName())
- .perms(meta.hasPerms() ? convert(meta.getPerms()) : null)
- .acls(meta.getAclsList().isEmpty() ? null : convertAclEntry(
- meta.getAclsList()))
- .xAttrs(meta.getXAttrsList().isEmpty() ? null : convertXAttrs(
- meta.getXAttrsList()))
- .xAttrsRemoved(meta.getXAttrsRemoved())
- .build());
- break;
- case EVENT_RENAME:
- InotifyProtos.RenameEventProto rename =
- InotifyProtos.RenameEventProto.parseFrom(p.getContents());
- events.add(new Event.RenameEvent(rename.getSrcPath(), rename.getDestPath(),
- rename.getTimestamp()));
- break;
- case EVENT_APPEND:
- InotifyProtos.AppendEventProto reopen =
- InotifyProtos.AppendEventProto.parseFrom(p.getContents());
- events.add(new Event.AppendEvent(reopen.getPath()));
- break;
- case EVENT_UNLINK:
- InotifyProtos.UnlinkEventProto unlink =
- InotifyProtos.UnlinkEventProto.parseFrom(p.getContents());
- events.add(new Event.UnlinkEvent(unlink.getPath(), unlink.getTimestamp()));
- break;
- default:
- throw new RuntimeException("Unexpected inotify event type: " +
- p.getType());
+ final InotifyProtos.EventsListProto list = resp.getEventsList();
+ final long firstTxid = list.getFirstTxid();
+ final long lastTxid = list.getLastTxid();
+
+ List batches = Lists.newArrayList();
+ if (list.getEventsList().size() > 0) {
+ throw new IOException("Can't handle old inotify server response.");
+ }
+ for (InotifyProtos.EventBatchProto bp : list.getBatchList()) {
+ long txid = bp.getTxid();
+ if ((txid != -1) && ((txid < firstTxid) || (txid > lastTxid))) {
+ throw new IOException("Error converting TxidResponseProto: got a " +
+ "transaction id " + txid + " that was outside the range of [" +
+ firstTxid + ", " + lastTxid + "].");
}
+ List events = Lists.newArrayList();
+ for (InotifyProtos.EventProto p : bp.getEventsList()) {
+ switch (p.getType()) {
+ case EVENT_CLOSE:
+ InotifyProtos.CloseEventProto close =
+ InotifyProtos.CloseEventProto.parseFrom(p.getContents());
+ events.add(new Event.CloseEvent(close.getPath(),
+ close.getFileSize(), close.getTimestamp()));
+ break;
+ case EVENT_CREATE:
+ InotifyProtos.CreateEventProto create =
+ InotifyProtos.CreateEventProto.parseFrom(p.getContents());
+ events.add(new Event.CreateEvent.Builder()
+ .iNodeType(createTypeConvert(create.getType()))
+ .path(create.getPath())
+ .ctime(create.getCtime())
+ .ownerName(create.getOwnerName())
+ .groupName(create.getGroupName())
+ .perms(convert(create.getPerms()))
+ .replication(create.getReplication())
+ .symlinkTarget(create.getSymlinkTarget().isEmpty() ? null :
+ create.getSymlinkTarget())
+ .defaultBlockSize(create.getDefaultBlockSize())
+ .overwrite(create.getOverwrite()).build());
+ break;
+ case EVENT_METADATA:
+ InotifyProtos.MetadataUpdateEventProto meta =
+ InotifyProtos.MetadataUpdateEventProto.parseFrom(p.getContents());
+ events.add(new Event.MetadataUpdateEvent.Builder()
+ .path(meta.getPath())
+ .metadataType(metadataUpdateTypeConvert(meta.getType()))
+ .mtime(meta.getMtime())
+ .atime(meta.getAtime())
+ .replication(meta.getReplication())
+ .ownerName(
+ meta.getOwnerName().isEmpty() ? null : meta.getOwnerName())
+ .groupName(
+ meta.getGroupName().isEmpty() ? null : meta.getGroupName())
+ .perms(meta.hasPerms() ? convert(meta.getPerms()) : null)
+ .acls(meta.getAclsList().isEmpty() ? null : convertAclEntry(
+ meta.getAclsList()))
+ .xAttrs(meta.getXAttrsList().isEmpty() ? null : convertXAttrs(
+ meta.getXAttrsList()))
+ .xAttrsRemoved(meta.getXAttrsRemoved())
+ .build());
+ break;
+ case EVENT_RENAME:
+ InotifyProtos.RenameEventProto rename =
+ InotifyProtos.RenameEventProto.parseFrom(p.getContents());
+ events.add(new Event.RenameEvent.Builder()
+ .srcPath(rename.getSrcPath())
+ .dstPath(rename.getDestPath())
+ .timestamp(rename.getTimestamp())
+ .build());
+ break;
+ case EVENT_APPEND:
+ InotifyProtos.AppendEventProto reopen =
+ InotifyProtos.AppendEventProto.parseFrom(p.getContents());
+ events.add(new Event.AppendEvent.Builder()
+ .path(reopen.getPath())
+ .build());
+ break;
+ case EVENT_UNLINK:
+ InotifyProtos.UnlinkEventProto unlink =
+ InotifyProtos.UnlinkEventProto.parseFrom(p.getContents());
+ events.add(new Event.UnlinkEvent.Builder()
+ .path(unlink.getPath())
+ .timestamp(unlink.getTimestamp())
+ .build());
+ break;
+ default:
+ throw new RuntimeException("Unexpected inotify event type: " +
+ p.getType());
+ }
+ }
+ batches.add(new EventBatch(txid, events.toArray(new Event[0])));
}
- return new EventsList(events, resp.getEventsList().getFirstTxid(),
+ return new EventBatchList(batches, resp.getEventsList().getFirstTxid(),
resp.getEventsList().getLastTxid(), resp.getEventsList().getSyncTxid());
}
- public static GetEditsFromTxidResponseProto convertEditsResponse(EventsList el) {
+ public static GetEditsFromTxidResponseProto convertEditsResponse(EventBatchList el) {
InotifyProtos.EventsListProto.Builder builder =
InotifyProtos.EventsListProto.newBuilder();
- for (Event e : el.getEvents()) {
- switch(e.getEventType()) {
- case CLOSE:
- Event.CloseEvent ce = (Event.CloseEvent) e;
- builder.addEvents(InotifyProtos.EventProto.newBuilder()
- .setType(InotifyProtos.EventType.EVENT_CLOSE)
- .setContents(
- InotifyProtos.CloseEventProto.newBuilder()
- .setPath(ce.getPath())
- .setFileSize(ce.getFileSize())
- .setTimestamp(ce.getTimestamp()).build().toByteString()
- ).build());
- break;
- case CREATE:
- Event.CreateEvent ce2 = (Event.CreateEvent) e;
- builder.addEvents(InotifyProtos.EventProto.newBuilder()
- .setType(InotifyProtos.EventType.EVENT_CREATE)
- .setContents(
- InotifyProtos.CreateEventProto.newBuilder()
- .setType(createTypeConvert(ce2.getiNodeType()))
- .setPath(ce2.getPath())
- .setCtime(ce2.getCtime())
- .setOwnerName(ce2.getOwnerName())
- .setGroupName(ce2.getGroupName())
- .setPerms(convert(ce2.getPerms()))
- .setReplication(ce2.getReplication())
- .setSymlinkTarget(ce2.getSymlinkTarget() == null ?
- "" : ce2.getSymlinkTarget())
- .setOverwrite(ce2.getOverwrite()).build().toByteString()
- ).build());
- break;
- case METADATA:
- Event.MetadataUpdateEvent me = (Event.MetadataUpdateEvent) e;
- InotifyProtos.MetadataUpdateEventProto.Builder metaB =
- InotifyProtos.MetadataUpdateEventProto.newBuilder()
- .setPath(me.getPath())
- .setType(metadataUpdateTypeConvert(me.getMetadataType()))
- .setMtime(me.getMtime())
- .setAtime(me.getAtime())
- .setReplication(me.getReplication())
- .setOwnerName(me.getOwnerName() == null ? "" :
- me.getOwnerName())
- .setGroupName(me.getGroupName() == null ? "" :
- me.getGroupName())
- .addAllAcls(me.getAcls() == null ?
- Lists.newArrayList() :
- convertAclEntryProto(me.getAcls()))
- .addAllXAttrs(me.getxAttrs() == null ?
- Lists.newArrayList() :
- convertXAttrProto(me.getxAttrs()))
- .setXAttrsRemoved(me.isxAttrsRemoved());
- if (me.getPerms() != null) {
- metaB.setPerms(convert(me.getPerms()));
+ for (EventBatch b : el.getBatches()) {
+ List events = Lists.newArrayList();
+ for (Event e : b.getEvents()) {
+ switch (e.getEventType()) {
+ case CLOSE:
+ Event.CloseEvent ce = (Event.CloseEvent) e;
+ events.add(InotifyProtos.EventProto.newBuilder()
+ .setType(InotifyProtos.EventType.EVENT_CLOSE)
+ .setContents(
+ InotifyProtos.CloseEventProto.newBuilder()
+ .setPath(ce.getPath())
+ .setFileSize(ce.getFileSize())
+ .setTimestamp(ce.getTimestamp()).build().toByteString()
+ ).build());
+ break;
+ case CREATE:
+ Event.CreateEvent ce2 = (Event.CreateEvent) e;
+ events.add(InotifyProtos.EventProto.newBuilder()
+ .setType(InotifyProtos.EventType.EVENT_CREATE)
+ .setContents(
+ InotifyProtos.CreateEventProto.newBuilder()
+ .setType(createTypeConvert(ce2.getiNodeType()))
+ .setPath(ce2.getPath())
+ .setCtime(ce2.getCtime())
+ .setOwnerName(ce2.getOwnerName())
+ .setGroupName(ce2.getGroupName())
+ .setPerms(convert(ce2.getPerms()))
+ .setReplication(ce2.getReplication())
+ .setSymlinkTarget(ce2.getSymlinkTarget() == null ?
+ "" : ce2.getSymlinkTarget())
+ .setDefaultBlockSize(ce2.getDefaultBlockSize())
+ .setOverwrite(ce2.getOverwrite()).build().toByteString()
+ ).build());
+ break;
+ case METADATA:
+ Event.MetadataUpdateEvent me = (Event.MetadataUpdateEvent) e;
+ InotifyProtos.MetadataUpdateEventProto.Builder metaB =
+ InotifyProtos.MetadataUpdateEventProto.newBuilder()
+ .setPath(me.getPath())
+ .setType(metadataUpdateTypeConvert(me.getMetadataType()))
+ .setMtime(me.getMtime())
+ .setAtime(me.getAtime())
+ .setReplication(me.getReplication())
+ .setOwnerName(me.getOwnerName() == null ? "" :
+ me.getOwnerName())
+ .setGroupName(me.getGroupName() == null ? "" :
+ me.getGroupName())
+ .addAllAcls(me.getAcls() == null ?
+ Lists.newArrayList() :
+ convertAclEntryProto(me.getAcls()))
+ .addAllXAttrs(me.getxAttrs() == null ?
+ Lists.newArrayList() :
+ convertXAttrProto(me.getxAttrs()))
+ .setXAttrsRemoved(me.isxAttrsRemoved());
+ if (me.getPerms() != null) {
+ metaB.setPerms(convert(me.getPerms()));
+ }
+ events.add(InotifyProtos.EventProto.newBuilder()
+ .setType(InotifyProtos.EventType.EVENT_METADATA)
+ .setContents(metaB.build().toByteString())
+ .build());
+ break;
+ case RENAME:
+ Event.RenameEvent re = (Event.RenameEvent) e;
+ events.add(InotifyProtos.EventProto.newBuilder()
+ .setType(InotifyProtos.EventType.EVENT_RENAME)
+ .setContents(
+ InotifyProtos.RenameEventProto.newBuilder()
+ .setSrcPath(re.getSrcPath())
+ .setDestPath(re.getDstPath())
+ .setTimestamp(re.getTimestamp()).build().toByteString()
+ ).build());
+ break;
+ case APPEND:
+ Event.AppendEvent re2 = (Event.AppendEvent) e;
+ events.add(InotifyProtos.EventProto.newBuilder()
+ .setType(InotifyProtos.EventType.EVENT_APPEND)
+ .setContents(
+ InotifyProtos.AppendEventProto.newBuilder()
+ .setPath(re2.getPath()).build().toByteString()
+ ).build());
+ break;
+ case UNLINK:
+ Event.UnlinkEvent ue = (Event.UnlinkEvent) e;
+ events.add(InotifyProtos.EventProto.newBuilder()
+ .setType(InotifyProtos.EventType.EVENT_UNLINK)
+ .setContents(
+ InotifyProtos.UnlinkEventProto.newBuilder()
+ .setPath(ue.getPath())
+ .setTimestamp(ue.getTimestamp()).build().toByteString()
+ ).build());
+ break;
+ default:
+ throw new RuntimeException("Unexpected inotify event: " + e);
}
- builder.addEvents(InotifyProtos.EventProto.newBuilder()
- .setType(InotifyProtos.EventType.EVENT_METADATA)
- .setContents(metaB.build().toByteString())
- .build());
- break;
- case RENAME:
- Event.RenameEvent re = (Event.RenameEvent) e;
- builder.addEvents(InotifyProtos.EventProto.newBuilder()
- .setType(InotifyProtos.EventType.EVENT_RENAME)
- .setContents(
- InotifyProtos.RenameEventProto.newBuilder()
- .setSrcPath(re.getSrcPath())
- .setDestPath(re.getDstPath())
- .setTimestamp(re.getTimestamp()).build().toByteString()
- ).build());
- break;
- case APPEND:
- Event.AppendEvent re2 = (Event.AppendEvent) e;
- builder.addEvents(InotifyProtos.EventProto.newBuilder()
- .setType(InotifyProtos.EventType.EVENT_APPEND)
- .setContents(
- InotifyProtos.AppendEventProto.newBuilder()
- .setPath(re2.getPath()).build().toByteString()
- ).build());
- break;
- case UNLINK:
- Event.UnlinkEvent ue = (Event.UnlinkEvent) e;
- builder.addEvents(InotifyProtos.EventProto.newBuilder()
- .setType(InotifyProtos.EventType.EVENT_UNLINK)
- .setContents(
- InotifyProtos.UnlinkEventProto.newBuilder()
- .setPath(ue.getPath())
- .setTimestamp(ue.getTimestamp()).build().toByteString()
- ).build());
- break;
- default:
- throw new RuntimeException("Unexpected inotify event: " + e);
}
+ builder.addBatch(InotifyProtos.EventBatchProto.newBuilder().
+ setTxid(b.getTxid()).
+ addAllEvents(events));
}
builder.setFirstTxid(el.getFirstTxid());
builder.setLastTxid(el.getLastTxid());
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/JournalNode.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/JournalNode.java
index 588bc580ce54c..a5a40f1158fea 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/JournalNode.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/JournalNode.java
@@ -233,6 +233,7 @@ private File getLogDir(String jid) {
Preconditions.checkArgument(jid != null &&
!jid.isEmpty(),
"bad journal identifier: %s", jid);
+ assert jid != null;
return new File(new File(dir), jid);
}
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Dispatcher.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Dispatcher.java
index 6ede40a58a530..63e151ca97aec 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Dispatcher.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Dispatcher.java
@@ -243,6 +243,10 @@ private boolean markMovedIfGoodBlock(DBlock block, StorageType targetStorageType
*/
private boolean chooseProxySource() {
final DatanodeInfo targetDN = target.getDatanodeInfo();
+ // if source and target are same nodes then no need of proxy
+ if (source.getDatanodeInfo().equals(targetDN) && addTo(source)) {
+ return true;
+ }
// if node group is supported, first try add nodes in the same node group
if (cluster.isNodeGroupAware()) {
for (StorageGroup loc : block.getLocations()) {
@@ -375,19 +379,6 @@ public static class DBlock extends MovedBlocks.Locations {
public DBlock(Block block) {
super(block);
}
-
- @Override
- public synchronized boolean isLocatedOn(StorageGroup loc) {
- // currently we only check if replicas are located on the same DataNodes
- // since we do not have the capability to store two replicas in the same
- // DataNode even though they are on two different storage types
- for (StorageGroup existing : locations) {
- if (existing.getDatanodeInfo().equals(loc.getDatanodeInfo())) {
- return true;
- }
- }
- return false;
- }
}
/** The class represents a desired move. */
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfo.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfo.java
index f547b1a630db6..5d0b473f40d46 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfo.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfo.java
@@ -366,13 +366,16 @@ public boolean isComplete() {
public BlockInfoUnderConstruction convertToBlockUnderConstruction(
BlockUCState s, DatanodeStorageInfo[] targets) {
if(isComplete()) {
- return new BlockInfoUnderConstruction(this,
+ BlockInfoUnderConstruction ucBlock = new BlockInfoUnderConstruction(this,
getBlockCollection().getBlockReplication(), s, targets);
+ ucBlock.setBlockCollection(getBlockCollection());
+ return ucBlock;
}
// the block is already under construction
BlockInfoUnderConstruction ucBlock = (BlockInfoUnderConstruction)this;
ucBlock.setBlockUCState(s);
ucBlock.setExpectedLocations(targets);
+ ucBlock.setBlockCollection(getBlockCollection());
return ucBlock;
}
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
index 254643cb34ce0..7e54c0f9c362d 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
@@ -54,7 +54,6 @@
import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
import org.apache.hadoop.fs.FileEncryptionInfo;
-
import org.apache.hadoop.hdfs.protocol.LocatedBlock;
import org.apache.hadoop.hdfs.protocol.LocatedBlocks;
import org.apache.hadoop.hdfs.protocol.UnregisteredNodeException;
@@ -63,6 +62,7 @@
import org.apache.hadoop.hdfs.security.token.block.DataEncryptionKey;
import org.apache.hadoop.hdfs.security.token.block.ExportedBlockKeys;
import org.apache.hadoop.hdfs.server.blockmanagement.CorruptReplicasMap.Reason;
+import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeStorageInfo.AddBlockResult;
import org.apache.hadoop.hdfs.server.blockmanagement.PendingDataNodeMessages.ReportedBlockInfo;
import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.BlockUCState;
import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.ReplicaState;
@@ -1050,21 +1050,6 @@ void removeBlocksAssociatedTo(final DatanodeDescriptor node) {
node.resetBlocks();
invalidateBlocks.remove(node);
-
- // If the DN hasn't block-reported since the most recent
- // failover, then we may have been holding up on processing
- // over-replicated blocks because of it. But we can now
- // process those blocks.
- boolean stale = false;
- for(DatanodeStorageInfo storage : node.getStorageInfos()) {
- if (storage.areBlockContentsStale()) {
- stale = true;
- break;
- }
- }
- if (stale) {
- rescanPostponedMisreplicatedBlocks();
- }
}
/** Remove the blocks associated to the given DatanodeStorageInfo. */
@@ -1785,6 +1770,8 @@ public boolean processReport(final DatanodeID nodeID,
final long startTime = Time.now(); //after acquiring write lock
final long endTime;
DatanodeDescriptor node;
+ Collection invalidatedBlocks = null;
+
try {
node = datanodeManager.getDatanode(nodeID);
if (node == null || !node.isAlive) {
@@ -1813,25 +1800,23 @@ public boolean processReport(final DatanodeID nodeID,
// ordinary block reports. This shortens restart times.
processFirstBlockReport(storageInfo, newReport);
} else {
- processReport(storageInfo, newReport);
+ invalidatedBlocks = processReport(storageInfo, newReport);
}
- // Now that we have an up-to-date block report, we know that any
- // deletions from a previous NN iteration have been accounted for.
- boolean staleBefore = storageInfo.areBlockContentsStale();
storageInfo.receivedBlockReport();
- if (staleBefore && !storageInfo.areBlockContentsStale()) {
- LOG.info("BLOCK* processReport: Received first block report from "
- + storage + " after starting up or becoming active. Its block "
- + "contents are no longer considered stale");
- rescanPostponedMisreplicatedBlocks();
- }
-
} finally {
endTime = Time.now();
namesystem.writeUnlock();
}
+ if (invalidatedBlocks != null) {
+ for (Block b : invalidatedBlocks) {
+ blockLog.info("BLOCK* processReport: " + b + " on " + node
+ + " size " + b.getNumBytes()
+ + " does not belong to any file");
+ }
+ }
+
// Log the block report processing stats from Namenode perspective
final NameNodeMetrics metrics = NameNode.getNameNodeMetrics();
if (metrics != null) {
@@ -1847,36 +1832,80 @@ public boolean processReport(final DatanodeID nodeID,
/**
* Rescan the list of blocks which were previously postponed.
*/
- private void rescanPostponedMisreplicatedBlocks() {
- for (Iterator it = postponedMisreplicatedBlocks.iterator();
- it.hasNext();) {
- Block b = it.next();
-
- BlockInfo bi = blocksMap.getStoredBlock(b);
- if (bi == null) {
- if (LOG.isDebugEnabled()) {
- LOG.debug("BLOCK* rescanPostponedMisreplicatedBlocks: " +
- "Postponed mis-replicated block " + b + " no longer found " +
- "in block map.");
+ void rescanPostponedMisreplicatedBlocks() {
+ if (getPostponedMisreplicatedBlocksCount() == 0) {
+ return;
+ }
+ long startTimeRescanPostponedMisReplicatedBlocks = Time.now();
+ long startPostponedMisReplicatedBlocksCount =
+ getPostponedMisreplicatedBlocksCount();
+ namesystem.writeLock();
+ try {
+ // blocksPerRescan is the configured number of blocks per rescan.
+ // Randomly select blocksPerRescan consecutive blocks from the HashSet
+ // when the number of blocks remaining is larger than blocksPerRescan.
+ // The reason we don't always pick the first blocksPerRescan blocks is to
+ // handle the case if for some reason some datanodes remain in
+ // content stale state for a long time and only impact the first
+ // blocksPerRescan blocks.
+ int i = 0;
+ long startIndex = 0;
+ long blocksPerRescan =
+ datanodeManager.getBlocksPerPostponedMisreplicatedBlocksRescan();
+ long base = getPostponedMisreplicatedBlocksCount() - blocksPerRescan;
+ if (base > 0) {
+ startIndex = DFSUtil.getRandom().nextLong() % (base+1);
+ if (startIndex < 0) {
+ startIndex += (base+1);
}
- it.remove();
- postponedMisreplicatedBlocksCount.decrementAndGet();
- continue;
}
- MisReplicationResult res = processMisReplicatedBlock(bi);
- if (LOG.isDebugEnabled()) {
- LOG.debug("BLOCK* rescanPostponedMisreplicatedBlocks: " +
- "Re-scanned block " + b + ", result is " + res);
+ Iterator it = postponedMisreplicatedBlocks.iterator();
+ for (int tmp = 0; tmp < startIndex; tmp++) {
+ it.next();
}
- if (res != MisReplicationResult.POSTPONE) {
- it.remove();
- postponedMisreplicatedBlocksCount.decrementAndGet();
+
+ for (;it.hasNext(); i++) {
+ Block b = it.next();
+ if (i >= blocksPerRescan) {
+ break;
+ }
+
+ BlockInfo bi = blocksMap.getStoredBlock(b);
+ if (bi == null) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("BLOCK* rescanPostponedMisreplicatedBlocks: " +
+ "Postponed mis-replicated block " + b + " no longer found " +
+ "in block map.");
+ }
+ it.remove();
+ postponedMisreplicatedBlocksCount.decrementAndGet();
+ continue;
+ }
+ MisReplicationResult res = processMisReplicatedBlock(bi);
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("BLOCK* rescanPostponedMisreplicatedBlocks: " +
+ "Re-scanned block " + b + ", result is " + res);
+ }
+ if (res != MisReplicationResult.POSTPONE) {
+ it.remove();
+ postponedMisreplicatedBlocksCount.decrementAndGet();
+ }
}
+ } finally {
+ namesystem.writeUnlock();
+ long endPostponedMisReplicatedBlocksCount =
+ getPostponedMisreplicatedBlocksCount();
+ LOG.info("Rescan of postponedMisreplicatedBlocks completed in " +
+ (Time.now() - startTimeRescanPostponedMisReplicatedBlocks) +
+ " msecs. " + endPostponedMisReplicatedBlocksCount +
+ " blocks are left. " + (startPostponedMisReplicatedBlocksCount -
+ endPostponedMisReplicatedBlocksCount) + " blocks are removed.");
}
}
- private void processReport(final DatanodeStorageInfo storageInfo,
- final BlockListAsLongs report) throws IOException {
+ private Collection processReport(
+ final DatanodeStorageInfo storageInfo,
+ final BlockListAsLongs report) throws IOException {
// Normal case:
// Modify the (block-->datanode) map, according to the difference
// between the old and new block report.
@@ -1907,14 +1936,13 @@ private void processReport(final DatanodeStorageInfo storageInfo,
+ " of " + numBlocksLogged + " reported.");
}
for (Block b : toInvalidate) {
- blockLog.info("BLOCK* processReport: "
- + b + " on " + node + " size " + b.getNumBytes()
- + " does not belong to any file");
addToInvalidates(b, node);
}
for (BlockToMarkCorrupt b : toCorrupt) {
markBlockAsCorrupt(b, storageInfo, node);
}
+
+ return toInvalidate;
}
/**
@@ -2000,8 +2028,9 @@ private void reportDiff(DatanodeStorageInfo storageInfo,
// place a delimiter in the list which separates blocks
// that have been reported from those that have not
BlockInfo delimiter = new BlockInfo(new Block(), (short) 1);
- boolean added = storageInfo.addBlock(delimiter);
- assert added : "Delimiting block cannot be present in the node";
+ AddBlockResult result = storageInfo.addBlock(delimiter);
+ assert result == AddBlockResult.ADDED
+ : "Delimiting block cannot be present in the node";
int headIndex = 0; //currently the delimiter is in the head of the list
int curIndex;
@@ -2394,14 +2423,19 @@ private Block addStoredBlock(final BlockInfo block,
assert bc != null : "Block must belong to a file";
// add block to the datanode
- boolean added = storageInfo.addBlock(storedBlock);
+ AddBlockResult result = storageInfo.addBlock(storedBlock);
int curReplicaDelta;
- if (added) {
+ if (result == AddBlockResult.ADDED) {
curReplicaDelta = 1;
if (logEveryBlock) {
logAddStoredBlock(storedBlock, node);
}
+ } else if (result == AddBlockResult.REPLACED) {
+ curReplicaDelta = 0;
+ blockLog.warn("BLOCK* addStoredBlock: " + "block " + storedBlock
+ + " moved to storageType " + storageInfo.getStorageType()
+ + " on node " + node);
} else {
// if the same block is added again and the replica was corrupt
// previously because of a wrong gen stamp, remove it from the
@@ -2423,7 +2457,7 @@ private Block addStoredBlock(final BlockInfo block,
if(storedBlock.getBlockUCState() == BlockUCState.COMMITTED &&
numLiveReplicas >= minReplication) {
storedBlock = completeBlock(bc, storedBlock, false);
- } else if (storedBlock.isComplete() && added) {
+ } else if (storedBlock.isComplete() && result == AddBlockResult.ADDED) {
// check whether safe replication is reached for the block
// only complete blocks are counted towards that
// Is no-op if not in safe mode.
@@ -3564,6 +3598,7 @@ public void run() {
if (namesystem.isPopulatingReplQueues()) {
computeDatanodeWork();
processPendingReplications();
+ rescanPostponedMisreplicatedBlocks();
}
Thread.sleep(replicationRecheckInterval);
} catch (Throwable t) {
@@ -3632,6 +3667,8 @@ public void clearQueues() {
excessReplicateMap.clear();
invalidateBlocks.clear();
datanodeManager.clearPendingQueues();
+ postponedMisreplicatedBlocks.clear();
+ postponedMisreplicatedBlocksCount.set(0);
};
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlocksMap.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlocksMap.java
index a675635d5d40d..6664034abe37a 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlocksMap.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlocksMap.java
@@ -20,6 +20,7 @@
import java.util.Iterator;
import org.apache.hadoop.hdfs.protocol.Block;
+import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeStorageInfo.AddBlockResult;
import org.apache.hadoop.hdfs.server.protocol.DatanodeStorage;
import org.apache.hadoop.util.GSet;
import org.apache.hadoop.util.LightWeightGSet;
@@ -223,8 +224,9 @@ BlockInfo replaceBlock(BlockInfo newBlock) {
final boolean removed = storage.removeBlock(currentBlock);
Preconditions.checkState(removed, "currentBlock not found.");
- final boolean added = storage.addBlock(newBlock);
- Preconditions.checkState(added, "newBlock already exists.");
+ final AddBlockResult result = storage.addBlock(newBlock);
+ Preconditions.checkState(result == AddBlockResult.ADDED,
+ "newBlock already exists.");
}
// replace block in the map itself
blocks.put(newBlock);
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java
index 356a4a3cf0dac..41d03633feac8 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java
@@ -133,13 +133,18 @@ public class DatanodeManager {
* writing to stale datanodes, i.e., continue using stale nodes for writing.
*/
private final float ratioUseStaleDataNodesForWrite;
-
+
/** The number of stale DataNodes */
private volatile int numStaleNodes;
/** The number of stale storages */
private volatile int numStaleStorages;
+ /**
+ * Number of blocks to check for each postponedMisreplicatedBlocks iteration
+ */
+ private final long blocksPerPostponedMisreplicatedBlocksRescan;
+
/**
* Whether or not this cluster has ever consisted of more than 1 rack,
* according to the NetworkTopology.
@@ -259,6 +264,9 @@ public class DatanodeManager {
this.timeBetweenResendingCachingDirectivesMs = conf.getLong(
DFSConfigKeys.DFS_NAMENODE_PATH_BASED_CACHE_RETRY_INTERVAL_MS,
DFSConfigKeys.DFS_NAMENODE_PATH_BASED_CACHE_RETRY_INTERVAL_MS_DEFAULT);
+ this.blocksPerPostponedMisreplicatedBlocksRescan = conf.getLong(
+ DFSConfigKeys.DFS_NAMENODE_BLOCKS_PER_POSTPONEDBLOCKS_RESCAN_KEY,
+ DFSConfigKeys.DFS_NAMENODE_BLOCKS_PER_POSTPONEDBLOCKS_RESCAN_KEY_DEFAULT);
}
private static long getStaleIntervalFromConf(Configuration conf,
@@ -1112,16 +1120,8 @@ public int getNumDeadDataNodes() {
public List getDecommissioningNodes() {
// There is no need to take namesystem reader lock as
// getDatanodeListForReport will synchronize on datanodeMap
- final List decommissioningNodes
- = new ArrayList();
- final List results = getDatanodeListForReport(
- DatanodeReportType.LIVE);
- for(DatanodeDescriptor node : results) {
- if (node.isDecommissionInProgress()) {
- decommissioningNodes.add(node);
- }
- }
- return decommissioningNodes;
+ // A decommissioning DN may be "alive" or "dead".
+ return getDatanodeListForReport(DatanodeReportType.DECOMMISSIONING);
}
/* Getter and Setter for stale DataNodes related attributes */
@@ -1141,6 +1141,10 @@ public boolean shouldAvoidStaleDataNodesForWrite() {
* ratioUseStaleDataNodesForWrite);
}
+ public long getBlocksPerPostponedMisreplicatedBlocksRescan() {
+ return blocksPerPostponedMisreplicatedBlocksRescan;
+ }
+
/**
* @return The time interval used to mark DataNodes as stale.
*/
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeStorageInfo.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeStorageInfo.java
index 8c44b3043fbe4..a3198e20ef35a 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeStorageInfo.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeStorageInfo.java
@@ -215,10 +215,10 @@ long getBlockPoolUsed() {
return blockPoolUsed;
}
- public boolean addBlock(BlockInfo b) {
+ public AddBlockResult addBlock(BlockInfo b) {
// First check whether the block belongs to a different storage
// on the same DN.
- boolean replaced = false;
+ AddBlockResult result = AddBlockResult.ADDED;
DatanodeStorageInfo otherStorage =
b.findStorageInfo(getDatanodeDescriptor());
@@ -226,10 +226,10 @@ public boolean addBlock(BlockInfo b) {
if (otherStorage != this) {
// The block belongs to a different storage. Remove it first.
otherStorage.removeBlock(b);
- replaced = true;
+ result = AddBlockResult.REPLACED;
} else {
// The block is already associated with this storage.
- return false;
+ return AddBlockResult.ALREADY_EXIST;
}
}
@@ -237,7 +237,7 @@ public boolean addBlock(BlockInfo b) {
b.addStorage(this);
blockList = b.listInsert(blockList, this);
numBlocks++;
- return !replaced;
+ return result;
}
boolean removeBlock(BlockInfo b) {
@@ -358,4 +358,8 @@ static DatanodeStorageInfo getDatanodeStorageInfo(
}
return null;
}
+
+ static enum AddBlockResult {
+ ADDED, REPLACED, ALREADY_EXIST;
+ }
}
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/Storage.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/Storage.java
index 31fdb84385683..e6bd5b289332d 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/Storage.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/Storage.java
@@ -727,7 +727,7 @@ FileLock tryLock() throws IOException {
file.close();
throw e;
}
- if (res != null && !deletionHookAdded) {
+ if (!deletionHookAdded) {
// If the file existed prior to our startup, we didn't
// call deleteOnExit above. But since we successfully locked
// the dir, we can take care of cleaning it up.
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockReceiver.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockReceiver.java
index 2e388f9b9c6e2..08c96be86e6b9 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockReceiver.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockReceiver.java
@@ -29,6 +29,8 @@
import java.io.FileWriter;
import java.io.IOException;
import java.io.OutputStream;
+import java.io.OutputStreamWriter;
+import java.io.Writer;
import java.nio.ByteBuffer;
import java.util.Arrays;
import java.util.LinkedList;
@@ -836,9 +838,8 @@ void receiveBlock(
LOG.warn("Failed to delete restart meta file: " +
restartMeta.getPath());
}
- FileWriter out = null;
- try {
- out = new FileWriter(restartMeta);
+ try (Writer out = new OutputStreamWriter(
+ new FileOutputStream(restartMeta), "UTF-8")) {
// write out the current time.
out.write(Long.toString(Time.now() + restartBudget));
out.flush();
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
index 13c32d530adf2..899a7294e86b6 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
@@ -580,7 +580,8 @@ public IOException call() {
try {
IOException ioe = ioExceptionFuture.get();
if (ioe != null) {
- errorMessageBuilder.append(String.format("FAILED TO ADD: %s: %s\n",
+ errorMessageBuilder.append(
+ String.format("FAILED TO ADD: %s: %s%n",
volume, ioe.getMessage()));
LOG.error("Failed to add volume: " + volume, ioe);
} else {
@@ -588,8 +589,9 @@ public IOException call() {
LOG.info("Successfully added volume: " + volume);
}
} catch (Exception e) {
- errorMessageBuilder.append(String.format("FAILED to ADD: %s: %s\n",
- volume, e.getMessage()));
+ errorMessageBuilder.append(
+ String.format("FAILED to ADD: %s: %s%n", volume,
+ e.toString()));
}
}
}
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataStorage.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataStorage.java
index c90ef954d923f..3ea8ce36dfd01 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataStorage.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataStorage.java
@@ -19,6 +19,7 @@
package org.apache.hadoop.hdfs.server.datanode;
import com.google.common.annotations.VisibleForTesting;
+import com.google.common.collect.ComparisonChain;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import com.google.common.util.concurrent.Futures;
@@ -57,13 +58,16 @@
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
+import java.util.Comparator;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
+import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Properties;
import java.util.Set;
+import java.util.TreeMap;
import java.util.concurrent.Callable;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ExecutorService;
@@ -425,7 +429,7 @@ synchronized void removeVolumes(Collection locations)
LOG.warn(String.format(
"I/O error attempting to unlock storage directory %s.",
sd.getRoot()), e);
- errorMsgBuilder.append(String.format("Failed to remove %s: %s\n",
+ errorMsgBuilder.append(String.format("Failed to remove %s: %s%n",
sd.getRoot(), e.getMessage()));
}
}
@@ -979,10 +983,10 @@ private void linkAllBlocks(DataNode datanode, File fromDir, File fromBbwDir,
}
private static class LinkArgs {
- public File src;
- public File dst;
+ File src;
+ File dst;
- public LinkArgs(File src, File dst) {
+ LinkArgs(File src, File dst) {
this.src = src;
this.dst = dst;
}
@@ -999,9 +1003,19 @@ static void linkBlocks(DataNode datanode, File from, File to, int oldLV,
upgradeToIdBasedLayout = true;
}
- final List idBasedLayoutSingleLinks = Lists.newArrayList();
+ final ArrayList idBasedLayoutSingleLinks = Lists.newArrayList();
linkBlocksHelper(from, to, oldLV, hl, upgradeToIdBasedLayout, to,
idBasedLayoutSingleLinks);
+
+ // Detect and remove duplicate entries.
+ final ArrayList duplicates =
+ findDuplicateEntries(idBasedLayoutSingleLinks);
+ if (!duplicates.isEmpty()) {
+ LOG.error("There are " + duplicates.size() + " duplicate block " +
+ "entries within the same volume.");
+ removeDuplicateEntries(idBasedLayoutSingleLinks, duplicates);
+ }
+
int numLinkWorkers = datanode.getConf().getInt(
DFSConfigKeys.DFS_DATANODE_BLOCK_ID_LAYOUT_UPGRADE_THREADS_KEY,
DFSConfigKeys.DFS_DATANODE_BLOCK_ID_LAYOUT_UPGRADE_THREADS);
@@ -1028,7 +1042,162 @@ public Void call() throws IOException {
Futures.get(f, IOException.class);
}
}
-
+
+ /**
+ * Find duplicate entries with an array of LinkArgs.
+ * Duplicate entries are entries with the same last path component.
+ */
+ static ArrayList findDuplicateEntries(ArrayList all) {
+ // Find duplicates by sorting the list by the final path component.
+ Collections.sort(all, new Comparator() {
+ /**
+ * Compare two LinkArgs objects, such that objects with the same
+ * terminal source path components are grouped together.
+ */
+ @Override
+ public int compare(LinkArgs a, LinkArgs b) {
+ return ComparisonChain.start().
+ compare(a.src.getName(), b.src.getName()).
+ compare(a.src, b.src).
+ compare(a.dst, b.dst).
+ result();
+ }
+ });
+ final ArrayList duplicates = Lists.newArrayList();
+ Long prevBlockId = null;
+ boolean prevWasMeta = false;
+ boolean addedPrev = false;
+ for (int i = 0; i < all.size(); i++) {
+ LinkArgs args = all.get(i);
+ long blockId = Block.getBlockId(args.src.getName());
+ boolean isMeta = Block.isMetaFilename(args.src.getName());
+ if ((prevBlockId == null) ||
+ (prevBlockId.longValue() != blockId)) {
+ prevBlockId = blockId;
+ addedPrev = false;
+ } else if (isMeta == prevWasMeta) {
+ // If we saw another file for the same block ID previously,
+ // and it had the same meta-ness as this file, we have a
+ // duplicate.
+ duplicates.add(args);
+ if (!addedPrev) {
+ duplicates.add(all.get(i - 1));
+ }
+ addedPrev = true;
+ } else {
+ addedPrev = false;
+ }
+ prevWasMeta = isMeta;
+ }
+ return duplicates;
+ }
+
+ /**
+ * Remove duplicate entries from the list.
+ * We do this by choosing:
+ * 1. the entries with the highest genstamp (this takes priority),
+ * 2. the entries with the longest block files,
+ * 3. arbitrarily, if neither #1 nor #2 gives a clear winner.
+ *
+ * Block and metadata files form a pair-- if you take a metadata file from
+ * one subdirectory, you must also take the block file from that
+ * subdirectory.
+ */
+ private static void removeDuplicateEntries(ArrayList all,
+ ArrayList duplicates) {
+ // Maps blockId -> metadata file with highest genstamp
+ TreeMap> highestGenstamps =
+ new TreeMap>();
+ for (LinkArgs duplicate : duplicates) {
+ if (!Block.isMetaFilename(duplicate.src.getName())) {
+ continue;
+ }
+ long blockId = Block.getBlockId(duplicate.src.getName());
+ List prevHighest = highestGenstamps.get(blockId);
+ if (prevHighest == null) {
+ List highest = new LinkedList();
+ highest.add(duplicate);
+ highestGenstamps.put(blockId, highest);
+ continue;
+ }
+ long prevGenstamp =
+ Block.getGenerationStamp(prevHighest.get(0).src.getName());
+ long genstamp = Block.getGenerationStamp(duplicate.src.getName());
+ if (genstamp < prevGenstamp) {
+ continue;
+ }
+ if (genstamp > prevGenstamp) {
+ prevHighest.clear();
+ }
+ prevHighest.add(duplicate);
+ }
+
+ // Remove data / metadata entries that don't have the highest genstamp
+ // from the duplicates list.
+ for (Iterator iter = duplicates.iterator(); iter.hasNext(); ) {
+ LinkArgs duplicate = iter.next();
+ long blockId = Block.getBlockId(duplicate.src.getName());
+ List highest = highestGenstamps.get(blockId);
+ if (highest != null) {
+ boolean found = false;
+ for (LinkArgs high : highest) {
+ if (high.src.getParent().equals(duplicate.src.getParent())) {
+ found = true;
+ break;
+ }
+ }
+ if (!found) {
+ LOG.warn("Unexpectedly low genstamp on " +
+ duplicate.src.getAbsolutePath() + ".");
+ iter.remove();
+ }
+ }
+ }
+
+ // Find the longest block files
+ // We let the "last guy win" here, since we're only interested in
+ // preserving one block file / metadata file pair.
+ TreeMap longestBlockFiles = new TreeMap();
+ for (LinkArgs duplicate : duplicates) {
+ if (Block.isMetaFilename(duplicate.src.getName())) {
+ continue;
+ }
+ long blockId = Block.getBlockId(duplicate.src.getName());
+ LinkArgs prevLongest = longestBlockFiles.get(blockId);
+ if (prevLongest == null) {
+ longestBlockFiles.put(blockId, duplicate);
+ continue;
+ }
+ long blockLength = duplicate.src.length();
+ long prevBlockLength = prevLongest.src.length();
+ if (blockLength < prevBlockLength) {
+ LOG.warn("Unexpectedly short length on " +
+ duplicate.src.getAbsolutePath() + ".");
+ continue;
+ }
+ if (blockLength > prevBlockLength) {
+ LOG.warn("Unexpectedly short length on " +
+ prevLongest.src.getAbsolutePath() + ".");
+ }
+ longestBlockFiles.put(blockId, duplicate);
+ }
+
+ // Remove data / metadata entries that aren't the longest, or weren't
+ // arbitrarily selected by us.
+ for (Iterator iter = all.iterator(); iter.hasNext(); ) {
+ LinkArgs args = iter.next();
+ long blockId = Block.getBlockId(args.src.getName());
+ LinkArgs bestDuplicate = longestBlockFiles.get(blockId);
+ if (bestDuplicate == null) {
+ continue; // file has no duplicates
+ }
+ if (!bestDuplicate.src.getParent().equals(args.src.getParent())) {
+ LOG.warn("Discarding " + args.src.getAbsolutePath() + ".");
+ iter.remove();
+ }
+ }
+ }
+
static void linkBlocksHelper(File from, File to, int oldLV, HardLink hl,
boolean upgradeToIdBasedLayout, File blockRoot,
List idBasedLayoutSingleLinks) throws IOException {
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataXceiver.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataXceiver.java
index 61b9c675efea9..3a2723f16bd4c 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataXceiver.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataXceiver.java
@@ -192,10 +192,17 @@ public void run() {
HdfsConstants.SMALL_BUFFER_SIZE);
socketOut = saslStreams.out;
} catch (InvalidMagicNumberException imne) {
- LOG.info("Failed to read expected encryption handshake from client " +
- "at " + peer.getRemoteAddressString() + ". Perhaps the client " +
- "is running an older version of Hadoop which does not support " +
- "encryption");
+ if (imne.isHandshake4Encryption()) {
+ LOG.info("Failed to read expected encryption handshake from client " +
+ "at " + peer.getRemoteAddressString() + ". Perhaps the client " +
+ "is running an older version of Hadoop which does not support " +
+ "encryption");
+ } else {
+ LOG.info("Failed to read expected SASL data transfer protection " +
+ "handshake from client at " + peer.getRemoteAddressString() +
+ ". Perhaps the client is running an older version of Hadoop " +
+ "which does not support SASL data transfer protection");
+ }
return;
}
@@ -1007,7 +1014,6 @@ public void replaceBlock(final ExtendedBlock block,
updateCurrentThreadName("Replacing block " + block + " from " + delHint);
/* read header */
- block.setNumBytes(dataXceiverServer.estimateBlockSize);
if (datanode.isBlockTokenEnabled) {
try {
datanode.blockPoolTokenSecretManager.checkAccess(blockToken, null, block,
@@ -1039,73 +1045,83 @@ public void replaceBlock(final ExtendedBlock block,
DataOutputStream replyOut = new DataOutputStream(getOutputStream());
boolean IoeDuringCopyBlockOperation = false;
try {
- // get the output stream to the proxy
- final String dnAddr = proxySource.getXferAddr(connectToDnViaHostname);
- if (LOG.isDebugEnabled()) {
- LOG.debug("Connecting to datanode " + dnAddr);
- }
- InetSocketAddress proxyAddr = NetUtils.createSocketAddr(dnAddr);
- proxySock = datanode.newSocket();
- NetUtils.connect(proxySock, proxyAddr, dnConf.socketTimeout);
- proxySock.setSoTimeout(dnConf.socketTimeout);
-
- OutputStream unbufProxyOut = NetUtils.getOutputStream(proxySock,
- dnConf.socketWriteTimeout);
- InputStream unbufProxyIn = NetUtils.getInputStream(proxySock);
- DataEncryptionKeyFactory keyFactory =
- datanode.getDataEncryptionKeyFactoryForBlock(block);
- IOStreamPair saslStreams = datanode.saslClient.socketSend(proxySock,
- unbufProxyOut, unbufProxyIn, keyFactory, blockToken, proxySource);
- unbufProxyOut = saslStreams.out;
- unbufProxyIn = saslStreams.in;
-
- proxyOut = new DataOutputStream(new BufferedOutputStream(unbufProxyOut,
- HdfsConstants.SMALL_BUFFER_SIZE));
- proxyReply = new DataInputStream(new BufferedInputStream(unbufProxyIn,
- HdfsConstants.IO_FILE_BUFFER_SIZE));
-
- /* send request to the proxy */
- IoeDuringCopyBlockOperation = true;
- new Sender(proxyOut).copyBlock(block, blockToken);
- IoeDuringCopyBlockOperation = false;
-
- // receive the response from the proxy
-
- BlockOpResponseProto copyResponse = BlockOpResponseProto.parseFrom(
- PBHelper.vintPrefixed(proxyReply));
-
- if (copyResponse.getStatus() != SUCCESS) {
- if (copyResponse.getStatus() == ERROR_ACCESS_TOKEN) {
+ // Move the block to different storage in the same datanode
+ if (proxySource.equals(datanode.getDatanodeId())) {
+ ReplicaInfo oldReplica = datanode.data.moveBlockAcrossStorage(block,
+ storageType);
+ if (oldReplica != null) {
+ LOG.info("Moved " + block + " from StorageType "
+ + oldReplica.getVolume().getStorageType() + " to " + storageType);
+ }
+ } else {
+ block.setNumBytes(dataXceiverServer.estimateBlockSize);
+ // get the output stream to the proxy
+ final String dnAddr = proxySource.getXferAddr(connectToDnViaHostname);
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Connecting to datanode " + dnAddr);
+ }
+ InetSocketAddress proxyAddr = NetUtils.createSocketAddr(dnAddr);
+ proxySock = datanode.newSocket();
+ NetUtils.connect(proxySock, proxyAddr, dnConf.socketTimeout);
+ proxySock.setSoTimeout(dnConf.socketTimeout);
+
+ OutputStream unbufProxyOut = NetUtils.getOutputStream(proxySock,
+ dnConf.socketWriteTimeout);
+ InputStream unbufProxyIn = NetUtils.getInputStream(proxySock);
+ DataEncryptionKeyFactory keyFactory =
+ datanode.getDataEncryptionKeyFactoryForBlock(block);
+ IOStreamPair saslStreams = datanode.saslClient.socketSend(proxySock,
+ unbufProxyOut, unbufProxyIn, keyFactory, blockToken, proxySource);
+ unbufProxyOut = saslStreams.out;
+ unbufProxyIn = saslStreams.in;
+
+ proxyOut = new DataOutputStream(new BufferedOutputStream(unbufProxyOut,
+ HdfsConstants.SMALL_BUFFER_SIZE));
+ proxyReply = new DataInputStream(new BufferedInputStream(unbufProxyIn,
+ HdfsConstants.IO_FILE_BUFFER_SIZE));
+
+ /* send request to the proxy */
+ IoeDuringCopyBlockOperation = true;
+ new Sender(proxyOut).copyBlock(block, blockToken);
+ IoeDuringCopyBlockOperation = false;
+
+ // receive the response from the proxy
+
+ BlockOpResponseProto copyResponse = BlockOpResponseProto.parseFrom(
+ PBHelper.vintPrefixed(proxyReply));
+
+ if (copyResponse.getStatus() != SUCCESS) {
+ if (copyResponse.getStatus() == ERROR_ACCESS_TOKEN) {
+ throw new IOException("Copy block " + block + " from "
+ + proxySock.getRemoteSocketAddress()
+ + " failed due to access token error");
+ }
throw new IOException("Copy block " + block + " from "
- + proxySock.getRemoteSocketAddress()
- + " failed due to access token error");
+ + proxySock.getRemoteSocketAddress() + " failed");
}
- throw new IOException("Copy block " + block + " from "
- + proxySock.getRemoteSocketAddress() + " failed");
+
+ // get checksum info about the block we're copying
+ ReadOpChecksumInfoProto checksumInfo = copyResponse.getReadOpChecksumInfo();
+ DataChecksum remoteChecksum = DataTransferProtoUtil.fromProto(
+ checksumInfo.getChecksum());
+ // open a block receiver and check if the block does not exist
+ blockReceiver = new BlockReceiver(block, storageType,
+ proxyReply, proxySock.getRemoteSocketAddress().toString(),
+ proxySock.getLocalSocketAddress().toString(),
+ null, 0, 0, 0, "", null, datanode, remoteChecksum,
+ CachingStrategy.newDropBehind(), false);
+
+ // receive a block
+ blockReceiver.receiveBlock(null, null, replyOut, null,
+ dataXceiverServer.balanceThrottler, null, true);
+
+ // notify name node
+ datanode.notifyNamenodeReceivedBlock(
+ block, delHint, blockReceiver.getStorageUuid());
+
+ LOG.info("Moved " + block + " from " + peer.getRemoteAddressString()
+ + ", delHint=" + delHint);
}
-
- // get checksum info about the block we're copying
- ReadOpChecksumInfoProto checksumInfo = copyResponse.getReadOpChecksumInfo();
- DataChecksum remoteChecksum = DataTransferProtoUtil.fromProto(
- checksumInfo.getChecksum());
- // open a block receiver and check if the block does not exist
- blockReceiver = new BlockReceiver(block, storageType,
- proxyReply, proxySock.getRemoteSocketAddress().toString(),
- proxySock.getLocalSocketAddress().toString(),
- null, 0, 0, 0, "", null, datanode, remoteChecksum,
- CachingStrategy.newDropBehind(), false);
-
- // receive a block
- blockReceiver.receiveBlock(null, null, replyOut, null,
- dataXceiverServer.balanceThrottler, null, true);
-
- // notify name node
- datanode.notifyNamenodeReceivedBlock(
- block, delHint, blockReceiver.getStorageUuid());
-
- LOG.info("Moved " + block + " from " + peer.getRemoteAddressString()
- + ", delHint=" + delHint);
-
} catch (IOException ioe) {
opStatus = ERROR;
errMsg = "opReplaceBlock " + block + " received exception " + ioe;
@@ -1117,7 +1133,7 @@ public void replaceBlock(final ExtendedBlock block,
throw ioe;
} finally {
// receive the last byte that indicates the proxy released its thread resource
- if (opStatus == SUCCESS) {
+ if (opStatus == SUCCESS && proxyReply != null) {
try {
proxyReply.readChar();
} catch (IOException ignored) {
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/ReplicaNotFoundException.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/ReplicaNotFoundException.java
index 124574b842f59..b159d3a1b5c55 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/ReplicaNotFoundException.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/ReplicaNotFoundException.java
@@ -37,7 +37,7 @@ public class ReplicaNotFoundException extends IOException {
public final static String NON_EXISTENT_REPLICA =
"Cannot append to a non-existent replica ";
public final static String UNEXPECTED_GS_REPLICA =
- "Cannot append to a replica with unexpeted generation stamp ";
+ "Cannot append to a replica with unexpected generation stamp ";
public ReplicaNotFoundException() {
super();
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/FsDatasetSpi.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/FsDatasetSpi.java
index a02ee0a434d82..462ad31c03c65 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/FsDatasetSpi.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/FsDatasetSpi.java
@@ -43,6 +43,7 @@
import org.apache.hadoop.hdfs.server.datanode.FinalizedReplica;
import org.apache.hadoop.hdfs.server.datanode.Replica;
import org.apache.hadoop.hdfs.server.datanode.ReplicaInPipelineInterface;
+import org.apache.hadoop.hdfs.server.datanode.ReplicaInfo;
import org.apache.hadoop.hdfs.server.datanode.ReplicaNotFoundException;
import org.apache.hadoop.hdfs.server.datanode.StorageLocation;
import org.apache.hadoop.hdfs.server.datanode.UnexpectedReplicaStateException;
@@ -508,4 +509,10 @@ public void onCompleteLazyPersist(String bpId, long blockId,
* Callback from RamDiskAsyncLazyPersistService upon async lazy persist task fail
*/
public void onFailLazyPersist(String bpId, long blockId);
+
+ /**
+ * Move block from one storage to another storage
+ */
+ public ReplicaInfo moveBlockAcrossStorage(final ExtendedBlock block,
+ StorageType targetStorageType) throws IOException;
}
\ No newline at end of file
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/BlockPoolSlice.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/BlockPoolSlice.java
index 5c8709cf55782..5a69e1e4d666d 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/BlockPoolSlice.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/BlockPoolSlice.java
@@ -22,10 +22,13 @@
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
+import java.io.FileOutputStream;
import java.io.FileWriter;
import java.io.IOException;
import java.io.InputStream;
+import java.io.OutputStreamWriter;
import java.io.RandomAccessFile;
+import java.io.Writer;
import java.util.Scanner;
import org.apache.commons.io.FileUtils;
@@ -157,6 +160,10 @@ File getRbwDir() {
return rbwDir;
}
+ File getTmpDir() {
+ return tmpDir;
+ }
+
/** Run DU on local drives. It must be synchronized from caller. */
void decDfsUsed(long value) {
dfsUsage.decDfsUsed(value);
@@ -182,7 +189,7 @@ long loadDfsUsed() {
Scanner sc;
try {
- sc = new Scanner(new File(currentDir, DU_CACHE_FILE));
+ sc = new Scanner(new File(currentDir, DU_CACHE_FILE), "UTF-8");
} catch (FileNotFoundException fnfe) {
return -1;
}
@@ -223,23 +230,18 @@ void saveDfsUsed() {
outFile.getParent());
}
- FileWriter out = null;
try {
long used = getDfsUsed();
- if (used > 0) {
- out = new FileWriter(outFile);
+ try (Writer out = new OutputStreamWriter(
+ new FileOutputStream(outFile), "UTF-8")) {
// mtime is written last, so that truncated writes won't be valid.
out.write(Long.toString(used) + " " + Long.toString(Time.now()));
out.flush();
- out.close();
- out = null;
}
} catch (IOException ioe) {
// If write failed, the volume might be bad. Since the cache file is
// not critical, log the error and continue.
FsDatasetImpl.LOG.warn("Failed to write dfsUsed to " + outFile, ioe);
- } finally {
- IOUtils.cleanup(null, out);
}
}
@@ -443,7 +445,7 @@ void addToReplicasMap(ReplicaMap volumeMap, File dir,
File.pathSeparator + "." + file.getName() + ".restart");
Scanner sc = null;
try {
- sc = new Scanner(restartMeta);
+ sc = new Scanner(restartMeta, "UTF-8");
// The restart meta file exists
if (sc.hasNextLong() && (sc.nextLong() > Time.now())) {
// It didn't expire. Load the replica as a RBW.
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java
index 4a89778ad3cdb..3eda38e703caa 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java
@@ -126,7 +126,7 @@ class FsDatasetImpl implements FsDatasetSpi {
@Override // FsDatasetSpi
public List getVolumes() {
- return volumes.volumes;
+ return volumes.getVolumes();
}
@Override
@@ -139,9 +139,10 @@ public StorageReport[] getStorageReports(String bpid)
throws IOException {
StorageReport[] reports;
synchronized (statsLock) {
- reports = new StorageReport[volumes.volumes.size()];
+ List curVolumes = getVolumes();
+ reports = new StorageReport[curVolumes.size()];
int i = 0;
- for (FsVolumeImpl volume : volumes.volumes) {
+ for (FsVolumeImpl volume : curVolumes) {
reports[i++] = new StorageReport(volume.toDatanodeStorage(),
false,
volume.getCapacity(),
@@ -663,13 +664,21 @@ static File moveBlockFiles(Block b, File srcfile, File destdir)
* @return the new meta and block files.
* @throws IOException
*/
- static File[] copyBlockFiles(long blockId, long genStamp,
- File srcMeta, File srcFile, File destRoot)
+ static File[] copyBlockFiles(long blockId, long genStamp, File srcMeta,
+ File srcFile, File destRoot, boolean calculateChecksum)
throws IOException {
final File destDir = DatanodeUtil.idToBlockDir(destRoot, blockId);
final File dstFile = new File(destDir, srcFile.getName());
final File dstMeta = FsDatasetUtil.getMetaFile(dstFile, genStamp);
- computeChecksum(srcMeta, dstMeta, srcFile);
+ if (calculateChecksum) {
+ computeChecksum(srcMeta, dstMeta, srcFile);
+ } else {
+ try {
+ Storage.nativeCopyFileUnbuffered(srcMeta, dstMeta, true);
+ } catch (IOException e) {
+ throw new IOException("Failed to copy " + srcMeta + " to " + dstMeta, e);
+ }
+ }
try {
Storage.nativeCopyFileUnbuffered(srcFile, dstFile, true);
@@ -677,13 +686,72 @@ static File[] copyBlockFiles(long blockId, long genStamp,
throw new IOException("Failed to copy " + srcFile + " to " + dstFile, e);
}
if (LOG.isDebugEnabled()) {
- LOG.debug("Copied " + srcMeta + " to " + dstMeta +
- " and calculated checksum");
- LOG.debug("Copied " + srcFile + " to " + dstFile);
+ if (calculateChecksum) {
+ LOG.debug("Copied " + srcMeta + " to " + dstMeta
+ + " and calculated checksum");
+ } else {
+ LOG.debug("Copied " + srcFile + " to " + dstFile);
+ }
}
return new File[] {dstMeta, dstFile};
}
+ /**
+ * Move block files from one storage to another storage.
+ * @return Returns the Old replicaInfo
+ * @throws IOException
+ */
+ @Override
+ public ReplicaInfo moveBlockAcrossStorage(ExtendedBlock block,
+ StorageType targetStorageType) throws IOException {
+ ReplicaInfo replicaInfo = getReplicaInfo(block);
+ if (replicaInfo.getState() != ReplicaState.FINALIZED) {
+ throw new ReplicaNotFoundException(
+ ReplicaNotFoundException.UNFINALIZED_REPLICA + block);
+ }
+ if (replicaInfo.getNumBytes() != block.getNumBytes()) {
+ throw new IOException("Corrupted replica " + replicaInfo
+ + " with a length of " + replicaInfo.getNumBytes()
+ + " expected length is " + block.getNumBytes());
+ }
+ if (replicaInfo.getVolume().getStorageType() == targetStorageType) {
+ throw new ReplicaAlreadyExistsException("Replica " + replicaInfo
+ + " already exists on storage " + targetStorageType);
+ }
+
+ if (replicaInfo.isOnTransientStorage()) {
+ // Block movement from RAM_DISK will be done by LazyPersist mechanism
+ throw new IOException("Replica " + replicaInfo
+ + " cannot be moved from storageType : "
+ + replicaInfo.getVolume().getStorageType());
+ }
+
+ FsVolumeImpl targetVolume = volumes.getNextVolume(targetStorageType,
+ block.getNumBytes());
+ File oldBlockFile = replicaInfo.getBlockFile();
+ File oldMetaFile = replicaInfo.getMetaFile();
+
+ // Copy files to temp dir first
+ File[] blockFiles = copyBlockFiles(block.getBlockId(),
+ block.getGenerationStamp(), oldMetaFile, oldBlockFile,
+ targetVolume.getTmpDir(block.getBlockPoolId()),
+ replicaInfo.isOnTransientStorage());
+
+ ReplicaInfo newReplicaInfo = new ReplicaInPipeline(
+ replicaInfo.getBlockId(), replicaInfo.getGenerationStamp(),
+ targetVolume, blockFiles[0].getParentFile(), 0);
+ newReplicaInfo.setNumBytes(blockFiles[1].length());
+ // Finalize the copied files
+ newReplicaInfo = finalizeReplica(block.getBlockPoolId(), newReplicaInfo);
+
+ removeOldReplica(replicaInfo, newReplicaInfo, oldBlockFile, oldMetaFile,
+ oldBlockFile.length(), oldMetaFile.length(), block.getBlockPoolId());
+
+ // Replace the old block if any to reschedule the scanning.
+ datanode.getBlockScanner().addBlock(block);
+ return replicaInfo;
+ }
+
/**
* Compute and store the checksum for a block file that does not already have
* its checksum computed.
@@ -702,7 +770,6 @@ private static void computeChecksum(File srcMeta, File dstMeta, File blockFile)
final byte[] crcs = new byte[checksum.getChecksumSize(data.length)];
DataOutputStream metaOut = null;
- InputStream dataIn = null;
try {
File parentFile = dstMeta.getParentFile();
if (parentFile != null) {
@@ -715,22 +782,23 @@ private static void computeChecksum(File srcMeta, File dstMeta, File blockFile)
new FileOutputStream(dstMeta), HdfsConstants.SMALL_BUFFER_SIZE));
BlockMetadataHeader.writeHeader(metaOut, checksum);
- dataIn = isNativeIOAvailable ?
+ int offset = 0;
+ try (InputStream dataIn = isNativeIOAvailable ?
NativeIO.getShareDeleteFileInputStream(blockFile) :
- new FileInputStream(blockFile);
+ new FileInputStream(blockFile)) {
- int offset = 0;
- for(int n; (n = dataIn.read(data, offset, data.length - offset)) != -1; ) {
- if (n > 0) {
- n += offset;
- offset = n % checksum.getBytesPerChecksum();
- final int length = n - offset;
+ for (int n; (n = dataIn.read(data, offset, data.length - offset)) != -1; ) {
+ if (n > 0) {
+ n += offset;
+ offset = n % checksum.getBytesPerChecksum();
+ final int length = n - offset;
- if (length > 0) {
- checksum.calculateChunkedSums(data, 0, length, crcs, 0);
- metaOut.write(crcs, 0, checksum.getChecksumSize(length));
+ if (length > 0) {
+ checksum.calculateChunkedSums(data, 0, length, crcs, 0);
+ metaOut.write(crcs, 0, checksum.getChecksumSize(length));
- System.arraycopy(data, length, data, 0, offset);
+ System.arraycopy(data, length, data, 0, offset);
+ }
}
}
}
@@ -739,7 +807,7 @@ private static void computeChecksum(File srcMeta, File dstMeta, File blockFile)
checksum.calculateChunkedSums(data, 0, offset, crcs, 0);
metaOut.write(crcs, 0, 4);
} finally {
- IOUtils.cleanup(LOG, dataIn, metaOut);
+ IOUtils.cleanup(LOG, metaOut);
}
}
@@ -1326,7 +1394,8 @@ public Map getBlockReports(String bpid) {
Map> uc =
new HashMap>();
- for (FsVolumeSpi v : volumes.volumes) {
+ List curVolumes = getVolumes();
+ for (FsVolumeSpi v : curVolumes) {
finalized.put(v.getStorageID(), new ArrayList());
uc.put(v.getStorageID(), new ArrayList());
}
@@ -1353,7 +1422,7 @@ public Map getBlockReports(String bpid) {
}
}
- for (FsVolumeImpl v : volumes.volumes) {
+ for (FsVolumeImpl v : curVolumes) {
ArrayList finalizedList = finalized.get(v.getStorageID());
ArrayList ucList = uc.get(v.getStorageID());
blockReportsMap.put(v.toDatanodeStorage(),
@@ -1532,11 +1601,6 @@ public void invalidate(String bpid, Block invalidBlks[]) throws IOException {
}
f = info.getBlockFile();
v = (FsVolumeImpl)info.getVolume();
- if (f == null) {
- errors.add("Failed to delete replica " + invalidBlks[i]
- + ": File not found, volume=" + v);
- continue;
- }
if (v == null) {
errors.add("Failed to delete replica " + invalidBlks[i]
+ ". No volume for this replica, file=" + f);
@@ -2221,7 +2285,7 @@ private static class VolumeInfo {
private Collection getVolumeInfo() {
Collection info = new ArrayList();
- for (FsVolumeImpl volume : volumes.volumes) {
+ for (FsVolumeImpl volume : getVolumes()) {
long used = 0;
long free = 0;
try {
@@ -2255,8 +2319,9 @@ public Map getVolumeInfoMap() {
@Override //FsDatasetSpi
public synchronized void deleteBlockPool(String bpid, boolean force)
throws IOException {
+ List curVolumes = getVolumes();
if (!force) {
- for (FsVolumeImpl volume : volumes.volumes) {
+ for (FsVolumeImpl volume : curVolumes) {
if (!volume.isBPDirEmpty(bpid)) {
LOG.warn(bpid + " has some block files, cannot delete unless forced");
throw new IOException("Cannot delete block pool, "
@@ -2264,7 +2329,7 @@ public synchronized void deleteBlockPool(String bpid, boolean force)
}
}
}
- for (FsVolumeImpl volume : volumes.volumes) {
+ for (FsVolumeImpl volume : curVolumes) {
volume.deleteBPDirectories(bpid, force);
}
}
@@ -2282,13 +2347,14 @@ public BlockLocalPathInfo getBlockLocalPathInfo(ExtendedBlock block)
@Override // FsDatasetSpi
public HdfsBlocksMetadata getHdfsBlocksMetadata(String poolId,
long[] blockIds) throws IOException {
+ List curVolumes = getVolumes();
// List of VolumeIds, one per volume on the datanode
- List blocksVolumeIds = new ArrayList(volumes.volumes.size());
+ List blocksVolumeIds = new ArrayList<>(curVolumes.size());
// List of indexes into the list of VolumeIds, pointing at the VolumeId of
// the volume that the block is on
List blocksVolumeIndexes = new ArrayList(blockIds.length);
// Initialize the list of VolumeIds simply by enumerating the volumes
- for (int i = 0; i < volumes.volumes.size(); i++) {
+ for (int i = 0; i < curVolumes.size(); i++) {
blocksVolumeIds.add(ByteBuffer.allocate(4).putInt(i).array());
}
// Determine the index of the VolumeId of each block's volume, by comparing
@@ -2301,7 +2367,7 @@ public HdfsBlocksMetadata getHdfsBlocksMetadata(String poolId,
int volumeIndex = 0;
if (info != null) {
FsVolumeSpi blockVolume = info.getVolume();
- for (FsVolumeImpl volume : volumes.volumes) {
+ for (FsVolumeImpl volume : curVolumes) {
// This comparison of references should be safe
if (blockVolume == volume) {
isValid = true;
@@ -2442,6 +2508,35 @@ private void setupAsyncLazyPersistThread(final FsVolumeImpl v) {
}
}
+ private void removeOldReplica(ReplicaInfo replicaInfo,
+ ReplicaInfo newReplicaInfo, File blockFile, File metaFile,
+ long blockFileUsed, long metaFileUsed, final String bpid) {
+ // Before deleting the files from old storage we must notify the
+ // NN that the files are on the new storage. Else a blockReport from
+ // the transient storage might cause the NN to think the blocks are lost.
+ // Replicas must be evicted from client short-circuit caches, because the
+ // storage will no longer be same, and thus will require validating
+ // checksum. This also stops a client from holding file descriptors,
+ // which would prevent the OS from reclaiming the memory.
+ ExtendedBlock extendedBlock =
+ new ExtendedBlock(bpid, newReplicaInfo);
+ datanode.getShortCircuitRegistry().processBlockInvalidation(
+ ExtendedBlockId.fromExtendedBlock(extendedBlock));
+ datanode.notifyNamenodeReceivedBlock(
+ extendedBlock, null, newReplicaInfo.getStorageUuid());
+
+ // Remove the old replicas
+ if (blockFile.delete() || !blockFile.exists()) {
+ ((FsVolumeImpl) replicaInfo.getVolume()).decDfsUsed(bpid, blockFileUsed);
+ if (metaFile.delete() || !metaFile.exists()) {
+ ((FsVolumeImpl) replicaInfo.getVolume()).decDfsUsed(bpid, metaFileUsed);
+ }
+ }
+
+ // If deletion failed then the directory scanner will cleanup the blocks
+ // eventually.
+ }
+
class LazyWriter implements Runnable {
private volatile boolean shouldRun = true;
final int checkpointerInterval;
@@ -2525,7 +2620,7 @@ private boolean transientFreeSpaceBelowThreshold() throws IOException {
// Don't worry about fragmentation for now. We don't expect more than one
// transient volume per DN.
- for (FsVolumeImpl v : volumes.volumes) {
+ for (FsVolumeImpl v : getVolumes()) {
if (v.isTransientStorage()) {
capacity += v.getCapacity();
free += v.getAvailable();
@@ -2601,30 +2696,8 @@ private void evictBlocks() throws IOException {
}
}
- // Before deleting the files from transient storage we must notify the
- // NN that the files are on the new storage. Else a blockReport from
- // the transient storage might cause the NN to think the blocks are lost.
- // Replicas must be evicted from client short-circuit caches, because the
- // storage will no longer be transient, and thus will require validating
- // checksum. This also stops a client from holding file descriptors,
- // which would prevent the OS from reclaiming the memory.
- ExtendedBlock extendedBlock =
- new ExtendedBlock(bpid, newReplicaInfo);
- datanode.getShortCircuitRegistry().processBlockInvalidation(
- ExtendedBlockId.fromExtendedBlock(extendedBlock));
- datanode.notifyNamenodeReceivedBlock(
- extendedBlock, null, newReplicaInfo.getStorageUuid());
-
- // Remove the old replicas from transient storage.
- if (blockFile.delete() || !blockFile.exists()) {
- ((FsVolumeImpl) replicaInfo.getVolume()).decDfsUsed(bpid, blockFileUsed);
- if (metaFile.delete() || !metaFile.exists()) {
- ((FsVolumeImpl) replicaInfo.getVolume()).decDfsUsed(bpid, metaFileUsed);
- }
- }
-
- // If deletion failed then the directory scanner will cleanup the blocks
- // eventually.
+ removeOldReplica(replicaInfo, newReplicaInfo, blockFile, metaFile,
+ blockFileUsed, metaFileUsed, bpid);
}
}
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsVolumeImpl.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsVolumeImpl.java
index 1d7540ccb1c43..48427ad052a03 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsVolumeImpl.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsVolumeImpl.java
@@ -129,6 +129,10 @@ File getLazyPersistDir(String bpid) throws IOException {
return getBlockPoolSlice(bpid).getLazypersistDir();
}
+ File getTmpDir(String bpid) throws IOException {
+ return getBlockPoolSlice(bpid).getTmpDir();
+ }
+
void decDfsUsed(String bpid, long value) {
synchronized(dataset) {
BlockPoolSlice bp = bpSlices.get(bpid);
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsVolumeList.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsVolumeList.java
index 837ddf720afb0..c02603d61422f 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsVolumeList.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsVolumeList.java
@@ -19,10 +19,13 @@
import java.io.IOException;
import java.util.ArrayList;
+import java.util.Arrays;
import java.util.Collections;
import java.util.Iterator;
import java.util.List;
+import java.util.concurrent.atomic.AtomicReference;
+import com.google.common.collect.Lists;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hdfs.StorageType;
import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsVolumeSpi;
@@ -31,11 +34,9 @@
import org.apache.hadoop.util.Time;
class FsVolumeList {
- /**
- * Read access to this unmodifiable list is not synchronized.
- * This list is replaced on modification holding "this" lock.
- */
- volatile List volumes = null;
+ private final AtomicReference volumes =
+ new AtomicReference<>(new FsVolumeImpl[0]);
+ private Object checkDirsMutex = new Object();
private final VolumeChoosingPolicy blockChooser;
private volatile int numFailedVolumes;
@@ -49,19 +50,27 @@ class FsVolumeList {
int numberOfFailedVolumes() {
return numFailedVolumes;
}
-
+
+ /**
+ * Return an immutable list view of all the volumes.
+ */
+ List getVolumes() {
+ return Collections.unmodifiableList(Arrays.asList(volumes.get()));
+ }
+
/**
- * Get next volume. Synchronized to ensure {@link #curVolume} is updated
- * by a single thread and next volume is chosen with no concurrent
- * update to {@link #volumes}.
+ * Get next volume.
+ *
* @param blockSize free space needed on the volume
* @param storageType the desired {@link StorageType}
* @return next volume to store the block in.
*/
- synchronized FsVolumeImpl getNextVolume(StorageType storageType,
- long blockSize) throws IOException {
- final List list = new ArrayList(volumes.size());
- for(FsVolumeImpl v : volumes) {
+ FsVolumeImpl getNextVolume(StorageType storageType, long blockSize)
+ throws IOException {
+ // Get a snapshot of currently available volumes.
+ final FsVolumeImpl[] curVolumes = volumes.get();
+ final List list = new ArrayList<>(curVolumes.length);
+ for(FsVolumeImpl v : curVolumes) {
if (v.getStorageType() == storageType) {
list.add(v);
}
@@ -70,16 +79,16 @@ synchronized FsVolumeImpl getNextVolume(StorageType storageType,
}
/**
- * Get next volume. Synchronized to ensure {@link #curVolume} is updated
- * by a single thread and next volume is chosen with no concurrent
- * update to {@link #volumes}.
+ * Get next volume.
+ *
* @param blockSize free space needed on the volume
* @return next volume to store the block in.
*/
- synchronized FsVolumeImpl getNextTransientVolume(
- long blockSize) throws IOException {
- final List list = new ArrayList(volumes.size());
- for(FsVolumeImpl v : volumes) {
+ FsVolumeImpl getNextTransientVolume(long blockSize) throws IOException {
+ // Get a snapshot of currently available volumes.
+ final List curVolumes = getVolumes();
+ final List list = new ArrayList<>(curVolumes.size());
+ for(FsVolumeImpl v : curVolumes) {
if (v.isTransientStorage()) {
list.add(v);
}
@@ -89,7 +98,7 @@ synchronized FsVolumeImpl getNextTransientVolume(
long getDfsUsed() throws IOException {
long dfsUsed = 0L;
- for (FsVolumeImpl v : volumes) {
+ for (FsVolumeImpl v : volumes.get()) {
dfsUsed += v.getDfsUsed();
}
return dfsUsed;
@@ -97,7 +106,7 @@ long getDfsUsed() throws IOException {
long getBlockPoolUsed(String bpid) throws IOException {
long dfsUsed = 0L;
- for (FsVolumeImpl v : volumes) {
+ for (FsVolumeImpl v : volumes.get()) {
dfsUsed += v.getBlockPoolUsed(bpid);
}
return dfsUsed;
@@ -105,7 +114,7 @@ long getBlockPoolUsed(String bpid) throws IOException {
long getCapacity() {
long capacity = 0L;
- for (FsVolumeImpl v : volumes) {
+ for (FsVolumeImpl v : volumes.get()) {
capacity += v.getCapacity();
}
return capacity;
@@ -113,7 +122,7 @@ long getCapacity() {
long getRemaining() throws IOException {
long remaining = 0L;
- for (FsVolumeSpi vol : volumes) {
+ for (FsVolumeSpi vol : volumes.get()) {
remaining += vol.getAvailable();
}
return remaining;
@@ -127,7 +136,7 @@ void getAllVolumesMap(final String bpid,
final List exceptions = Collections.synchronizedList(
new ArrayList());
List replicaAddingThreads = new ArrayList();
- for (final FsVolumeImpl v : volumes) {
+ for (final FsVolumeImpl v : volumes.get()) {
Thread t = new Thread() {
public void run() {
try {
@@ -167,76 +176,115 @@ public void run() {
* Calls {@link FsVolumeImpl#checkDirs()} on each volume, removing any
* volumes from the active list that result in a DiskErrorException.
*
- * This method is synchronized to allow only one instance of checkDirs()
- * call
+ * Use checkDirsMutext to allow only one instance of checkDirs() call
+ *
* @return list of all the removed volumes.
*/
- synchronized List checkDirs() {
- ArrayList removedVols = null;
-
- // Make a copy of volumes for performing modification
- final List volumeList = new ArrayList(volumes);
+ List checkDirs() {
+ synchronized(checkDirsMutex) {
+ ArrayList removedVols = null;
+
+ // Make a copy of volumes for performing modification
+ final List volumeList = getVolumes();
- for(Iterator i = volumeList.iterator(); i.hasNext(); ) {
- final FsVolumeImpl fsv = i.next();
- try {
- fsv.checkDirs();
- } catch (DiskErrorException e) {
- FsDatasetImpl.LOG.warn("Removing failed volume " + fsv + ": ",e);
- if (removedVols == null) {
- removedVols = new ArrayList(1);
+ for(Iterator i = volumeList.iterator(); i.hasNext(); ) {
+ final FsVolumeImpl fsv = i.next();
+ try {
+ fsv.checkDirs();
+ } catch (DiskErrorException e) {
+ FsDatasetImpl.LOG.warn("Removing failed volume " + fsv + ": ",e);
+ if (removedVols == null) {
+ removedVols = new ArrayList(1);
+ }
+ removedVols.add(fsv);
+ removeVolume(fsv);
+ numFailedVolumes++;
}
- removedVols.add(fsv);
- fsv.shutdown();
- i.remove(); // Remove the volume
- numFailedVolumes++;
}
- }
-
- if (removedVols != null && removedVols.size() > 0) {
- // Replace volume list
- volumes = Collections.unmodifiableList(volumeList);
- FsDatasetImpl.LOG.warn("Completed checkDirs. Removed " + removedVols.size()
- + " volumes. Current volumes: " + this);
- }
+
+ if (removedVols != null && removedVols.size() > 0) {
+ FsDatasetImpl.LOG.warn("Completed checkDirs. Removed " + removedVols.size()
+ + " volumes. Current volumes: " + this);
+ }
- return removedVols;
+ return removedVols;
+ }
}
@Override
public String toString() {
- return volumes.toString();
+ return Arrays.toString(volumes.get());
}
/**
* Dynamically add new volumes to the existing volumes that this DN manages.
* @param newVolume the instance of new FsVolumeImpl.
*/
- synchronized void addVolume(FsVolumeImpl newVolume) {
+ void addVolume(FsVolumeImpl newVolume) {
// Make a copy of volumes to add new volumes.
- final List volumeList = volumes == null ?
- new ArrayList() :
- new ArrayList(volumes);
- volumeList.add(newVolume);
- volumes = Collections.unmodifiableList(volumeList);
+ while (true) {
+ final FsVolumeImpl[] curVolumes = volumes.get();
+ final List volumeList = Lists.newArrayList(curVolumes);
+ volumeList.add(newVolume);
+ if (volumes.compareAndSet(curVolumes,
+ volumeList.toArray(new FsVolumeImpl[volumeList.size()]))) {
+ break;
+ } else {
+ if (FsDatasetImpl.LOG.isDebugEnabled()) {
+ FsDatasetImpl.LOG.debug(
+ "The volume list has been changed concurrently, " +
+ "retry to remove volume: " + newVolume);
+ }
+ }
+ }
+
FsDatasetImpl.LOG.info("Added new volume: " + newVolume.toString());
}
/**
- * Dynamically remove volume to the list.
+ * Dynamically remove a volume in the list.
+ * @param target the volume instance to be removed.
+ */
+ private void removeVolume(FsVolumeImpl target) {
+ while (true) {
+ final FsVolumeImpl[] curVolumes = volumes.get();
+ final List volumeList = Lists.newArrayList(curVolumes);
+ if (volumeList.remove(target)) {
+ if (volumes.compareAndSet(curVolumes,
+ volumeList.toArray(new FsVolumeImpl[volumeList.size()]))) {
+ target.shutdown();
+ FsDatasetImpl.LOG.info("Removed volume: " + target);
+ break;
+ } else {
+ if (FsDatasetImpl.LOG.isDebugEnabled()) {
+ FsDatasetImpl.LOG.debug(
+ "The volume list has been changed concurrently, " +
+ "retry to remove volume: " + target);
+ }
+ }
+ } else {
+ if (FsDatasetImpl.LOG.isDebugEnabled()) {
+ FsDatasetImpl.LOG.debug("Volume " + target +
+ " does not exist or is removed by others.");
+ }
+ break;
+ }
+ }
+ }
+
+ /**
+ * Dynamically remove volume in the list.
* @param volume the volume to be removed.
*/
- synchronized void removeVolume(String volume) {
+ void removeVolume(String volume) {
// Make a copy of volumes to remove one volume.
- final List volumeList = new ArrayList(volumes);
+ final FsVolumeImpl[] curVolumes = volumes.get();
+ final List volumeList = Lists.newArrayList(curVolumes);
for (Iterator it = volumeList.iterator(); it.hasNext(); ) {
FsVolumeImpl fsVolume = it.next();
if (fsVolume.getBasePath().equals(volume)) {
- fsVolume.shutdown();
- it.remove();
- volumes = Collections.unmodifiableList(volumeList);
- FsDatasetImpl.LOG.info("Removed volume: " + volume);
- break;
+ // Make sure the removed volume is the one in the curVolumes.
+ removeVolume(fsVolume);
}
}
}
@@ -247,7 +295,7 @@ void addBlockPool(final String bpid, final Configuration conf) throws IOExceptio
final List exceptions = Collections.synchronizedList(
new ArrayList());
List blockPoolAddingThreads = new ArrayList();
- for (final FsVolumeImpl v : volumes) {
+ for (final FsVolumeImpl v : volumes.get()) {
Thread t = new Thread() {
public void run() {
try {
@@ -285,13 +333,13 @@ public void run() {
}
void removeBlockPool(String bpid) {
- for (FsVolumeImpl v : volumes) {
+ for (FsVolumeImpl v : volumes.get()) {
v.shutdownBlockPool(bpid);
}
}
void shutdown() {
- for (FsVolumeImpl volume : volumes) {
+ for (FsVolumeImpl volume : volumes.get()) {
if(volume != null) {
volume.shutdown();
}
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/RamDiskAsyncLazyPersistService.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/RamDiskAsyncLazyPersistService.java
index 5fdcc2f9d87a6..c9aba8a292a5b 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/RamDiskAsyncLazyPersistService.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/RamDiskAsyncLazyPersistService.java
@@ -232,7 +232,7 @@ public void run() {
try {
// No FsDatasetImpl lock for the file copy
File targetFiles[] = FsDatasetImpl.copyBlockFiles(
- blockId, genStamp, metaFile, blockFile, lazyPersistDir);
+ blockId, genStamp, metaFile, blockFile, lazyPersistDir, true);
// Lock FsDataSetImpl during onCompleteLazyPersist callback
datanode.getFSDataset().onCompleteLazyPersist(bpId, blockId,
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/webhdfs/ExceptionHandler.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/webhdfs/ExceptionHandler.java
index fea40d7ce9644..a7bb4907d9cff 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/webhdfs/ExceptionHandler.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/webhdfs/ExceptionHandler.java
@@ -17,6 +17,7 @@
*/
package org.apache.hadoop.hdfs.server.datanode.web.webhdfs;
+import com.google.common.base.Charsets;
import com.sun.jersey.api.ParamException;
import com.sun.jersey.api.container.ContainerException;
import io.netty.buffer.Unpooled;
@@ -39,7 +40,7 @@
import static io.netty.handler.codec.http.HttpResponseStatus.INTERNAL_SERVER_ERROR;
import static io.netty.handler.codec.http.HttpResponseStatus.NOT_FOUND;
import static io.netty.handler.codec.http.HttpVersion.HTTP_1_1;
-import static org.apache.hadoop.hdfs.server.datanode.web.webhdfs.WebHdfsHandler.APPLICATION_JSON;
+import static org.apache.hadoop.hdfs.server.datanode.web.webhdfs.WebHdfsHandler.APPLICATION_JSON_UTF8;
class ExceptionHandler {
static Log LOG = WebHdfsHandler.LOG;
@@ -82,11 +83,11 @@ static DefaultFullHttpResponse exceptionCaught(Throwable cause) {
s = INTERNAL_SERVER_ERROR;
}
- final byte[] js = JsonUtil.toJsonString(e).getBytes();
+ final byte[] js = JsonUtil.toJsonString(e).getBytes(Charsets.UTF_8);
DefaultFullHttpResponse resp =
new DefaultFullHttpResponse(HTTP_1_1, s, Unpooled.wrappedBuffer(js));
- resp.headers().set(CONTENT_TYPE, APPLICATION_JSON);
+ resp.headers().set(CONTENT_TYPE, APPLICATION_JSON_UTF8);
resp.headers().set(CONTENT_LENGTH, js.length);
return resp;
}
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/webhdfs/WebHdfsHandler.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/webhdfs/WebHdfsHandler.java
index cf7021828836b..f02780a01d783 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/webhdfs/WebHdfsHandler.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/webhdfs/WebHdfsHandler.java
@@ -29,6 +29,7 @@
import io.netty.handler.codec.http.HttpRequest;
import io.netty.handler.codec.http.QueryStringDecoder;
import io.netty.handler.stream.ChunkedStream;
+import org.apache.commons.io.Charsets;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
@@ -77,7 +78,8 @@ public class WebHdfsHandler extends SimpleChannelInboundHandler {
public static final int WEBHDFS_PREFIX_LENGTH = WEBHDFS_PREFIX.length();
public static final String APPLICATION_OCTET_STREAM =
"application/octet-stream";
- public static final String APPLICATION_JSON = "application/json";
+ public static final String APPLICATION_JSON_UTF8 =
+ "application/json; charset=utf-8";
private final Configuration conf;
private final Configuration confForCreate;
@@ -224,11 +226,11 @@ private void onGetFileChecksum(ChannelHandlerContext ctx) throws IOException {
} finally {
IOUtils.cleanup(LOG, dfsclient);
}
- final byte[] js = JsonUtil.toJsonString(checksum).getBytes();
+ final byte[] js = JsonUtil.toJsonString(checksum).getBytes(Charsets.UTF_8);
DefaultFullHttpResponse resp =
new DefaultFullHttpResponse(HTTP_1_1, OK, Unpooled.wrappedBuffer(js));
- resp.headers().set(CONTENT_TYPE, APPLICATION_JSON);
+ resp.headers().set(CONTENT_TYPE, APPLICATION_JSON_UTF8);
resp.headers().set(CONTENT_LENGTH, js.length);
resp.headers().set(CONNECTION, CLOSE);
ctx.writeAndFlush(resp).addListener(ChannelFutureListener.CLOSE);
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/mover/Mover.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/mover/Mover.java
index 59814af5512a9..a22f920099a40 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/mover/Mover.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/mover/Mover.java
@@ -48,8 +48,10 @@
import org.apache.hadoop.util.ToolRunner;
import java.io.BufferedReader;
+import java.io.FileInputStream;
import java.io.FileReader;
import java.io.IOException;
+import java.io.InputStreamReader;
import java.net.URI;
import java.text.DateFormat;
import java.util.*;
@@ -86,8 +88,8 @@ private Source getSource(MLocation ml) {
return get(sources, ml);
}
- private StorageGroup getTarget(MLocation ml) {
- return get(targets, ml);
+ private StorageGroup getTarget(String uuid, StorageType storageType) {
+ return targets.get(uuid, storageType);
}
private static G get(StorageGroupMap map, MLocation ml) {
@@ -387,6 +389,11 @@ boolean scheduleMoveReplica(DBlock db, MLocation ml,
boolean scheduleMoveReplica(DBlock db, Source source,
List targetTypes) {
+ // Match storage on the same node
+ if (chooseTargetInSameNode(db, source, targetTypes)) {
+ return true;
+ }
+
if (dispatcher.getCluster().isNodeGroupAware()) {
if (chooseTarget(db, source, targetTypes, Matcher.SAME_NODE_GROUP)) {
return true;
@@ -401,6 +408,26 @@ boolean scheduleMoveReplica(DBlock db, Source source,
return chooseTarget(db, source, targetTypes, Matcher.ANY_OTHER);
}
+ /**
+ * Choose the target storage within same Datanode if possible.
+ */
+ boolean chooseTargetInSameNode(DBlock db, Source source,
+ List targetTypes) {
+ for (StorageType t : targetTypes) {
+ StorageGroup target = storages.getTarget(source.getDatanodeInfo()
+ .getDatanodeUuid(), t);
+ if (target == null) {
+ continue;
+ }
+ final PendingMove pm = source.addPendingMove(db, target);
+ if (pm != null) {
+ dispatcher.executePendingMove(pm);
+ return true;
+ }
+ }
+ return false;
+ }
+
boolean chooseTarget(DBlock db, Source source,
List targetTypes, Matcher matcher) {
final NetworkTopology cluster = dispatcher.getCluster();
@@ -554,7 +581,8 @@ private static Options buildCliOptions() {
private static String[] readPathFile(String file) throws IOException {
List list = Lists.newArrayList();
- BufferedReader reader = new BufferedReader(new FileReader(file));
+ BufferedReader reader = new BufferedReader(
+ new InputStreamReader(new FileInputStream(file), "UTF-8"));
try {
String line;
while ((line = reader.readLine()) != null) {
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/AclEntryStatusFormat.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/AclEntryStatusFormat.java
new file mode 100644
index 0000000000000..82aa214bf7436
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/AclEntryStatusFormat.java
@@ -0,0 +1,136 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hdfs.server.namenode;
+
+import java.util.List;
+
+import org.apache.hadoop.fs.permission.AclEntry;
+import org.apache.hadoop.fs.permission.AclEntryScope;
+import org.apache.hadoop.fs.permission.AclEntryType;
+import org.apache.hadoop.fs.permission.FsAction;
+import org.apache.hadoop.hdfs.util.LongBitFormat;
+
+import com.google.common.collect.ImmutableList;
+
+/**
+ * Class to pack an AclEntry into an integer.
+ * An ACL entry is represented by a 32-bit integer in Big Endian format.
+ * The bits can be divided in four segments:
+ * [0:1) || [1:3) || [3:6) || [6:7) || [7:32)
+ *
+ * [0:1) -- the scope of the entry (AclEntryScope)
+ * [1:3) -- the type of the entry (AclEntryType)
+ * [3:6) -- the permission of the entry (FsAction)
+ * [6:7) -- A flag to indicate whether Named entry or not
+ * [7:32) -- the name of the entry, which is an ID that points to a
+ * string in the StringTableSection.
+ */
+public enum AclEntryStatusFormat {
+
+ SCOPE(null, 1),
+ TYPE(SCOPE.BITS, 2),
+ PERMISSION(TYPE.BITS, 3),
+ NAMED_ENTRY_CHECK(PERMISSION.BITS, 1),
+ NAME(NAMED_ENTRY_CHECK.BITS, 25);
+
+ private final LongBitFormat BITS;
+
+ private AclEntryStatusFormat(LongBitFormat previous, int length) {
+ BITS = new LongBitFormat(name(), previous, length, 0);
+ }
+
+ static AclEntryScope getScope(int aclEntry) {
+ int ordinal = (int) SCOPE.BITS.retrieve(aclEntry);
+ return AclEntryScope.values()[ordinal];
+ }
+
+ static AclEntryType getType(int aclEntry) {
+ int ordinal = (int) TYPE.BITS.retrieve(aclEntry);
+ return AclEntryType.values()[ordinal];
+ }
+
+ static FsAction getPermission(int aclEntry) {
+ int ordinal = (int) PERMISSION.BITS.retrieve(aclEntry);
+ return FsAction.values()[ordinal];
+ }
+
+ static String getName(int aclEntry) {
+ int nameExists = (int) NAMED_ENTRY_CHECK.BITS.retrieve(aclEntry);
+ if (nameExists == 0) {
+ return null;
+ }
+ int id = (int) NAME.BITS.retrieve(aclEntry);
+ AclEntryType type = getType(aclEntry);
+ if (type == AclEntryType.USER) {
+ return SerialNumberManager.INSTANCE.getUser(id);
+ } else if (type == AclEntryType.GROUP) {
+ return SerialNumberManager.INSTANCE.getGroup(id);
+ }
+ return null;
+ }
+
+ static int toInt(AclEntry aclEntry) {
+ long aclEntryInt = 0;
+ aclEntryInt = SCOPE.BITS
+ .combine(aclEntry.getScope().ordinal(), aclEntryInt);
+ aclEntryInt = TYPE.BITS.combine(aclEntry.getType().ordinal(), aclEntryInt);
+ aclEntryInt = PERMISSION.BITS.combine(aclEntry.getPermission().ordinal(),
+ aclEntryInt);
+ if (aclEntry.getName() != null) {
+ aclEntryInt = NAMED_ENTRY_CHECK.BITS.combine(1, aclEntryInt);
+ if (aclEntry.getType() == AclEntryType.USER) {
+ int userId = SerialNumberManager.INSTANCE.getUserSerialNumber(aclEntry
+ .getName());
+ aclEntryInt = NAME.BITS.combine(userId, aclEntryInt);
+ } else if (aclEntry.getType() == AclEntryType.GROUP) {
+ int groupId = SerialNumberManager.INSTANCE
+ .getGroupSerialNumber(aclEntry.getName());
+ aclEntryInt = NAME.BITS.combine(groupId, aclEntryInt);
+ }
+ }
+ return (int) aclEntryInt;
+ }
+
+ static AclEntry toAclEntry(int aclEntry) {
+ AclEntry.Builder builder = new AclEntry.Builder();
+ builder.setScope(getScope(aclEntry)).setType(getType(aclEntry))
+ .setPermission(getPermission(aclEntry));
+ if (getName(aclEntry) != null) {
+ builder.setName(getName(aclEntry));
+ }
+ return builder.build();
+ }
+
+ public static int[] toInt(List aclEntries) {
+ int[] entries = new int[aclEntries.size()];
+ for (int i = 0; i < entries.length; i++) {
+ entries[i] = toInt(aclEntries.get(i));
+ }
+ return entries;
+ }
+
+ public static ImmutableList toAclEntries(int[] entries) {
+ ImmutableList.Builder b = new ImmutableList.Builder();
+ for (int entry : entries) {
+ AclEntry aclEntry = toAclEntry(entry);
+ b.add(aclEntry);
+ }
+ return b.build();
+ }
+}
\ No newline at end of file
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/AclFeature.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/AclFeature.java
index 1c5f469b3b090..e097b05bc803d 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/AclFeature.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/AclFeature.java
@@ -21,6 +21,7 @@
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.fs.permission.AclEntry;
+import com.google.common.base.Preconditions;
import com.google.common.collect.ImmutableList;
/**
@@ -31,13 +32,28 @@ public class AclFeature implements INode.Feature {
public static final ImmutableList EMPTY_ENTRY_LIST =
ImmutableList.of();
- private final ImmutableList entries;
+ private final int [] entries;
- public AclFeature(ImmutableList entries) {
+ public AclFeature(int[] entries) {
this.entries = entries;
}
- public ImmutableList getEntries() {
- return entries;
+ /**
+ * Get the number of entries present
+ */
+ int getEntriesSize() {
+ return entries.length;
+ }
+
+ /**
+ * Get the entry at the specified position
+ * @param pos Position of the entry to be obtained
+ * @return integer representation of AclEntry
+ * @throws IndexOutOfBoundsException if pos out of bound
+ */
+ int getEntryAt(int pos) {
+ Preconditions.checkPositionIndex(pos, entries.length,
+ "Invalid position for AclEntry");
+ return entries[pos];
}
}
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/AclStorage.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/AclStorage.java
index c15d64e607e8e..a86604686fd56 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/AclStorage.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/AclStorage.java
@@ -20,6 +20,7 @@
import java.util.Collections;
import java.util.List;
+import com.google.common.annotations.VisibleForTesting;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.Lists;
@@ -76,7 +77,8 @@ public static void copyINodeDefaultAcl(INode child) {
}
// Split parent's entries into access vs. default.
- List featureEntries = parent.getAclFeature().getEntries();
+ List featureEntries = getEntriesFromAclFeature(parent
+ .getAclFeature());
ScopedAclEntries scopedEntries = new ScopedAclEntries(featureEntries);
List parentDefaultEntries = scopedEntries.getDefaultEntries();
@@ -153,7 +155,25 @@ public static void copyINodeDefaultAcl(INode child) {
*/
public static List readINodeAcl(INode inode, int snapshotId) {
AclFeature f = inode.getAclFeature(snapshotId);
- return f == null ? ImmutableList. of() : f.getEntries();
+ return getEntriesFromAclFeature(f);
+ }
+
+ /**
+ * Build list of AclEntries from the AclFeature
+ * @param aclFeature AclFeature
+ * @return List of entries
+ */
+ @VisibleForTesting
+ static ImmutableList getEntriesFromAclFeature(AclFeature aclFeature) {
+ if (aclFeature == null) {
+ return ImmutableList. of();
+ }
+ ImmutableList.Builder b = new ImmutableList.Builder();
+ for (int pos = 0, entry; pos < aclFeature.getEntriesSize(); pos++) {
+ entry = aclFeature.getEntryAt(pos);
+ b.add(AclEntryStatusFormat.toAclEntry(entry));
+ }
+ return b.build();
}
/**
@@ -179,7 +199,7 @@ public static List readINodeLogicalAcl(INode inode) {
final List existingAcl;
// Split ACL entries stored in the feature into access vs. default.
- List featureEntries = f.getEntries();
+ List featureEntries = getEntriesFromAclFeature(f);
ScopedAclEntries scoped = new ScopedAclEntries(featureEntries);
List accessEntries = scoped.getAccessEntries();
List defaultEntries = scoped.getDefaultEntries();
@@ -220,39 +240,6 @@ public static List readINodeLogicalAcl(INode inode) {
return existingAcl;
}
- /**
- * Completely removes the ACL from an inode.
- *
- * @param inode INode to update
- * @param snapshotId int latest snapshot ID of inode
- * @throws QuotaExceededException if quota limit is exceeded
- */
- public static void removeINodeAcl(INode inode, int snapshotId)
- throws QuotaExceededException {
- AclFeature f = inode.getAclFeature();
- if (f == null) {
- return;
- }
-
- FsPermission perm = inode.getFsPermission();
- List featureEntries = f.getEntries();
- if (featureEntries.get(0).getScope() == AclEntryScope.ACCESS) {
- // Restore group permissions from the feature's entry to permission
- // bits, overwriting the mask, which is not part of a minimal ACL.
- AclEntry groupEntryKey = new AclEntry.Builder()
- .setScope(AclEntryScope.ACCESS).setType(AclEntryType.GROUP).build();
- int groupEntryIndex = Collections.binarySearch(featureEntries,
- groupEntryKey, AclTransformation.ACL_ENTRY_COMPARATOR);
- assert groupEntryIndex >= 0;
- FsAction groupPerm = featureEntries.get(groupEntryIndex).getPermission();
- FsPermission newPerm = new FsPermission(perm.getUserAction(), groupPerm,
- perm.getOtherAction(), perm.getStickyBit());
- inode.setPermission(newPerm, snapshotId);
- }
-
- inode.removeAclFeature(snapshotId);
- }
-
/**
* Updates an inode with a new ACL. This method takes a full logical ACL and
* stores the entries to the inode's {@link FsPermission} and
@@ -330,7 +317,7 @@ private static AclFeature createAclFeature(List accessEntries,
// Add all default entries to the feature.
featureEntries.addAll(defaultEntries);
- return new AclFeature(ImmutableList.copyOf(featureEntries));
+ return new AclFeature(AclEntryStatusFormat.toInt(featureEntries));
}
/**
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EncryptionZoneManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EncryptionZoneManager.java
index 0d7ced9274d8e..3fe748d0eb926 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EncryptionZoneManager.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EncryptionZoneManager.java
@@ -199,9 +199,9 @@ String getKeyName(final INodesInPath iip) {
private EncryptionZoneInt getEncryptionZoneForPath(INodesInPath iip) {
assert dir.hasReadLock();
Preconditions.checkNotNull(iip);
- final INode[] inodes = iip.getINodes();
- for (int i = inodes.length - 1; i >= 0; i--) {
- final INode inode = inodes[i];
+ List inodes = iip.getReadOnlyINodes();
+ for (int i = inodes.size() - 1; i >= 0; i--) {
+ final INode inode = inodes.get(i);
if (inode != null) {
final EncryptionZoneInt ezi = encryptionZones.get(inode.getId());
if (ezi != null) {
@@ -249,6 +249,10 @@ void checkMoveValidity(INodesInPath srcIIP, INodesInPath dstIIP, String src)
final boolean dstInEZ = (dstEZI != null);
if (srcInEZ) {
if (!dstInEZ) {
+ if (srcEZI.getINodeId() == srcIIP.getLastINode().getId()) {
+ // src is ez root and dest is not in an ez. Allow the rename.
+ return;
+ }
throw new IOException(
src + " can't be moved from an encryption zone.");
}
@@ -259,9 +263,7 @@ void checkMoveValidity(INodesInPath srcIIP, INodesInPath dstIIP, String src)
}
}
- if (srcInEZ || dstInEZ) {
- Preconditions.checkState(srcEZI != null, "couldn't find src EZ?");
- Preconditions.checkState(dstEZI != null, "couldn't find dst EZ?");
+ if (srcInEZ) {
if (srcEZI != dstEZI) {
final String srcEZPath = getFullPathName(srcEZI);
final String dstEZPath = getFullPathName(dstEZI);
@@ -285,12 +287,12 @@ XAttr createEncryptionZone(String src, CipherSuite suite,
CryptoProtocolVersion version, String keyName)
throws IOException {
assert dir.hasWriteLock();
- if (dir.isNonEmptyDirectory(src)) {
+ final INodesInPath srcIIP = dir.getINodesInPath4Write(src, false);
+ if (dir.isNonEmptyDirectory(srcIIP)) {
throw new IOException(
"Attempt to create an encryption zone for a non-empty directory.");
}
- final INodesInPath srcIIP = dir.getINodesInPath4Write(src, false);
if (srcIIP != null &&
srcIIP.getLastINode() != null &&
!srcIIP.getLastINode().isDirectory()) {
@@ -311,7 +313,8 @@ XAttr createEncryptionZone(String src, CipherSuite suite,
xattrs.add(ezXAttr);
// updating the xattr will call addEncryptionZone,
// done this way to handle edit log loading
- dir.unprotectedSetXAttrs(src, xattrs, EnumSet.of(XAttrSetFlag.CREATE));
+ FSDirXAttrOp.unprotectedSetXAttrs(dir, src, xattrs,
+ EnumSet.of(XAttrSetFlag.CREATE));
return ezXAttr;
}
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirAclOp.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirAclOp.java
new file mode 100644
index 0000000000000..7aaa21c99a163
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirAclOp.java
@@ -0,0 +1,247 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.namenode;
+
+import org.apache.hadoop.fs.permission.AclEntry;
+import org.apache.hadoop.fs.permission.AclEntryScope;
+import org.apache.hadoop.fs.permission.AclEntryType;
+import org.apache.hadoop.fs.permission.AclStatus;
+import org.apache.hadoop.fs.permission.FsAction;
+import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.hdfs.DFSConfigKeys;
+import org.apache.hadoop.hdfs.protocol.AclException;
+import org.apache.hadoop.hdfs.protocol.HdfsConstants;
+import org.apache.hadoop.hdfs.protocol.HdfsFileStatus;
+
+import java.io.IOException;
+import java.util.Collections;
+import java.util.List;
+
+class FSDirAclOp {
+ static HdfsFileStatus modifyAclEntries(
+ FSDirectory fsd, final String srcArg, List aclSpec)
+ throws IOException {
+ String src = srcArg;
+ checkAclsConfigFlag(fsd);
+ FSPermissionChecker pc = fsd.getPermissionChecker();
+ byte[][] pathComponents = FSDirectory.getPathComponentsForReservedPath(src);
+ src = fsd.resolvePath(pc, src, pathComponents);
+ INodesInPath iip;
+ fsd.writeLock();
+ try {
+ iip = fsd.getINodesInPath4Write(FSDirectory.normalizePath(src), true);
+ fsd.checkOwner(pc, iip);
+ INode inode = FSDirectory.resolveLastINode(iip);
+ int snapshotId = iip.getLatestSnapshotId();
+ List existingAcl = AclStorage.readINodeLogicalAcl(inode);
+ List newAcl = AclTransformation.mergeAclEntries(
+ existingAcl, aclSpec);
+ AclStorage.updateINodeAcl(inode, newAcl, snapshotId);
+ fsd.getEditLog().logSetAcl(src, newAcl);
+ } finally {
+ fsd.writeUnlock();
+ }
+ return fsd.getAuditFileInfo(iip);
+ }
+
+ static HdfsFileStatus removeAclEntries(
+ FSDirectory fsd, final String srcArg, List aclSpec)
+ throws IOException {
+ String src = srcArg;
+ checkAclsConfigFlag(fsd);
+ FSPermissionChecker pc = fsd.getPermissionChecker();
+ byte[][] pathComponents = FSDirectory.getPathComponentsForReservedPath(src);
+ src = fsd.resolvePath(pc, src, pathComponents);
+ INodesInPath iip;
+ fsd.writeLock();
+ try {
+ iip = fsd.getINodesInPath4Write(FSDirectory.normalizePath(src), true);
+ fsd.checkOwner(pc, iip);
+ INode inode = FSDirectory.resolveLastINode(iip);
+ int snapshotId = iip.getLatestSnapshotId();
+ List existingAcl = AclStorage.readINodeLogicalAcl(inode);
+ List newAcl = AclTransformation.filterAclEntriesByAclSpec(
+ existingAcl, aclSpec);
+ AclStorage.updateINodeAcl(inode, newAcl, snapshotId);
+ fsd.getEditLog().logSetAcl(src, newAcl);
+ } finally {
+ fsd.writeUnlock();
+ }
+ return fsd.getAuditFileInfo(iip);
+ }
+
+ static HdfsFileStatus removeDefaultAcl(FSDirectory fsd, final String srcArg)
+ throws IOException {
+ String src = srcArg;
+ checkAclsConfigFlag(fsd);
+ FSPermissionChecker pc = fsd.getPermissionChecker();
+ byte[][] pathComponents = FSDirectory.getPathComponentsForReservedPath(src);
+ src = fsd.resolvePath(pc, src, pathComponents);
+ INodesInPath iip;
+ fsd.writeLock();
+ try {
+ iip = fsd.getINodesInPath4Write(FSDirectory.normalizePath(src), true);
+ fsd.checkOwner(pc, iip);
+ INode inode = FSDirectory.resolveLastINode(iip);
+ int snapshotId = iip.getLatestSnapshotId();
+ List existingAcl = AclStorage.readINodeLogicalAcl(inode);
+ List newAcl = AclTransformation.filterDefaultAclEntries(
+ existingAcl);
+ AclStorage.updateINodeAcl(inode, newAcl, snapshotId);
+ fsd.getEditLog().logSetAcl(src, newAcl);
+ } finally {
+ fsd.writeUnlock();
+ }
+ return fsd.getAuditFileInfo(iip);
+ }
+
+ static HdfsFileStatus removeAcl(FSDirectory fsd, final String srcArg)
+ throws IOException {
+ String src = srcArg;
+ checkAclsConfigFlag(fsd);
+ FSPermissionChecker pc = fsd.getPermissionChecker();
+ byte[][] pathComponents = FSDirectory.getPathComponentsForReservedPath(src);
+ src = fsd.resolvePath(pc, src, pathComponents);
+ INodesInPath iip;
+ fsd.writeLock();
+ try {
+ iip = fsd.getINodesInPath4Write(src);
+ fsd.checkOwner(pc, iip);
+ unprotectedRemoveAcl(fsd, iip);
+ } finally {
+ fsd.writeUnlock();
+ }
+ fsd.getEditLog().logSetAcl(src, AclFeature.EMPTY_ENTRY_LIST);
+ return fsd.getAuditFileInfo(iip);
+ }
+
+ static HdfsFileStatus setAcl(
+ FSDirectory fsd, final String srcArg, List aclSpec)
+ throws IOException {
+ String src = srcArg;
+ checkAclsConfigFlag(fsd);
+ byte[][] pathComponents = FSDirectory.getPathComponentsForReservedPath(src);
+ FSPermissionChecker pc = fsd.getPermissionChecker();
+ src = fsd.resolvePath(pc, src, pathComponents);
+ INodesInPath iip;
+ fsd.writeLock();
+ try {
+ iip = fsd.getINodesInPath4Write(src);
+ fsd.checkOwner(pc, iip);
+ List newAcl = unprotectedSetAcl(fsd, src, aclSpec);
+ fsd.getEditLog().logSetAcl(src, newAcl);
+ } finally {
+ fsd.writeUnlock();
+ }
+ return fsd.getAuditFileInfo(iip);
+ }
+
+ static AclStatus getAclStatus(
+ FSDirectory fsd, String src) throws IOException {
+ checkAclsConfigFlag(fsd);
+ FSPermissionChecker pc = fsd.getPermissionChecker();
+ byte[][] pathComponents = FSDirectory.getPathComponentsForReservedPath(src);
+ src = fsd.resolvePath(pc, src, pathComponents);
+ String srcs = FSDirectory.normalizePath(src);
+ fsd.readLock();
+ try {
+ // There is no real inode for the path ending in ".snapshot", so return a
+ // non-null, unpopulated AclStatus. This is similar to getFileInfo.
+ if (srcs.endsWith(HdfsConstants.SEPARATOR_DOT_SNAPSHOT_DIR) &&
+ fsd.getINode4DotSnapshot(srcs) != null) {
+ return new AclStatus.Builder().owner("").group("").build();
+ }
+ INodesInPath iip = fsd.getINodesInPath(srcs, true);
+ if (fsd.isPermissionEnabled()) {
+ fsd.checkTraverse(pc, iip);
+ }
+ INode inode = FSDirectory.resolveLastINode(iip);
+ int snapshotId = iip.getPathSnapshotId();
+ List acl = AclStorage.readINodeAcl(inode, snapshotId);
+ FsPermission fsPermission = inode.getFsPermission(snapshotId);
+ return new AclStatus.Builder()
+ .owner(inode.getUserName()).group(inode.getGroupName())
+ .stickyBit(fsPermission.getStickyBit())
+ .setPermission(fsPermission)
+ .addEntries(acl).build();
+ } finally {
+ fsd.readUnlock();
+ }
+ }
+
+ static List unprotectedSetAcl(
+ FSDirectory fsd, String src, List aclSpec)
+ throws IOException {
+ assert fsd.hasWriteLock();
+ final INodesInPath iip = fsd.getINodesInPath4Write(
+ FSDirectory.normalizePath(src), true);
+
+ // ACL removal is logged to edits as OP_SET_ACL with an empty list.
+ if (aclSpec.isEmpty()) {
+ unprotectedRemoveAcl(fsd, iip);
+ return AclFeature.EMPTY_ENTRY_LIST;
+ }
+
+ INode inode = FSDirectory.resolveLastINode(iip);
+ int snapshotId = iip.getLatestSnapshotId();
+ List existingAcl = AclStorage.readINodeLogicalAcl(inode);
+ List newAcl = AclTransformation.replaceAclEntries(existingAcl,
+ aclSpec);
+ AclStorage.updateINodeAcl(inode, newAcl, snapshotId);
+ return newAcl;
+ }
+
+ private static void checkAclsConfigFlag(FSDirectory fsd) throws AclException {
+ if (!fsd.isAclsEnabled()) {
+ throw new AclException(String.format(
+ "The ACL operation has been rejected. "
+ + "Support for ACLs has been disabled by setting %s to false.",
+ DFSConfigKeys.DFS_NAMENODE_ACLS_ENABLED_KEY));
+ }
+ }
+
+ private static void unprotectedRemoveAcl(FSDirectory fsd, INodesInPath iip)
+ throws IOException {
+ assert fsd.hasWriteLock();
+ INode inode = FSDirectory.resolveLastINode(iip);
+ int snapshotId = iip.getLatestSnapshotId();
+ AclFeature f = inode.getAclFeature();
+ if (f == null) {
+ return;
+ }
+
+ FsPermission perm = inode.getFsPermission();
+ List featureEntries = AclStorage.getEntriesFromAclFeature(f);
+ if (featureEntries.get(0).getScope() == AclEntryScope.ACCESS) {
+ // Restore group permissions from the feature's entry to permission
+ // bits, overwriting the mask, which is not part of a minimal ACL.
+ AclEntry groupEntryKey = new AclEntry.Builder()
+ .setScope(AclEntryScope.ACCESS).setType(AclEntryType.GROUP).build();
+ int groupEntryIndex = Collections.binarySearch(
+ featureEntries, groupEntryKey,
+ AclTransformation.ACL_ENTRY_COMPARATOR);
+ assert groupEntryIndex >= 0;
+ FsAction groupPerm = featureEntries.get(groupEntryIndex).getPermission();
+ FsPermission newPerm = new FsPermission(perm.getUserAction(), groupPerm,
+ perm.getOtherAction(), perm.getStickyBit());
+ inode.setPermission(newPerm, snapshotId);
+ }
+
+ inode.removeAclFeature(snapshotId);
+ }
+}
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirAttrOp.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirAttrOp.java
new file mode 100644
index 0000000000000..6c1890ecc6096
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirAttrOp.java
@@ -0,0 +1,459 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.namenode;
+
+import org.apache.hadoop.HadoopIllegalArgumentException;
+import org.apache.hadoop.fs.PathIsNotDirectoryException;
+import org.apache.hadoop.fs.UnresolvedLinkException;
+import org.apache.hadoop.fs.XAttr;
+import org.apache.hadoop.fs.XAttrSetFlag;
+import org.apache.hadoop.fs.permission.FsAction;
+import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.hdfs.protocol.Block;
+import org.apache.hadoop.hdfs.protocol.BlockStoragePolicy;
+import org.apache.hadoop.hdfs.protocol.HdfsConstants;
+import org.apache.hadoop.hdfs.protocol.HdfsFileStatus;
+import org.apache.hadoop.hdfs.protocol.QuotaExceededException;
+import org.apache.hadoop.hdfs.protocol.SnapshotAccessControlException;
+import org.apache.hadoop.hdfs.server.blockmanagement.BlockManager;
+import org.apache.hadoop.hdfs.server.blockmanagement.BlockStoragePolicySuite;
+import org.apache.hadoop.security.AccessControlException;
+
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.EnumSet;
+import java.util.List;
+
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_ACCESSTIME_PRECISION_KEY;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_STORAGE_POLICY_ENABLED_KEY;
+
+public class FSDirAttrOp {
+ static HdfsFileStatus setPermission(
+ FSDirectory fsd, final String srcArg, FsPermission permission)
+ throws IOException {
+ String src = srcArg;
+ FSPermissionChecker pc = fsd.getPermissionChecker();
+ byte[][] pathComponents = FSDirectory.getPathComponentsForReservedPath(src);
+ INodesInPath iip;
+ fsd.writeLock();
+ try {
+ src = fsd.resolvePath(pc, src, pathComponents);
+ iip = fsd.getINodesInPath4Write(src);
+ fsd.checkOwner(pc, iip);
+ unprotectedSetPermission(fsd, src, permission);
+ } finally {
+ fsd.writeUnlock();
+ }
+ fsd.getEditLog().logSetPermissions(src, permission);
+ return fsd.getAuditFileInfo(iip);
+ }
+
+ static HdfsFileStatus setOwner(
+ FSDirectory fsd, String src, String username, String group)
+ throws IOException {
+ FSPermissionChecker pc = fsd.getPermissionChecker();
+ byte[][] pathComponents = FSDirectory.getPathComponentsForReservedPath(src);
+ INodesInPath iip;
+ fsd.writeLock();
+ try {
+ src = fsd.resolvePath(pc, src, pathComponents);
+ iip = fsd.getINodesInPath4Write(src);
+ fsd.checkOwner(pc, iip);
+ if (!pc.isSuperUser()) {
+ if (username != null && !pc.getUser().equals(username)) {
+ throw new AccessControlException("Non-super user cannot change owner");
+ }
+ if (group != null && !pc.containsGroup(group)) {
+ throw new AccessControlException("User does not belong to " + group);
+ }
+ }
+ unprotectedSetOwner(fsd, src, username, group);
+ } finally {
+ fsd.writeUnlock();
+ }
+ fsd.getEditLog().logSetOwner(src, username, group);
+ return fsd.getAuditFileInfo(iip);
+ }
+
+ static HdfsFileStatus setTimes(
+ FSDirectory fsd, String src, long mtime, long atime)
+ throws IOException {
+ if (!fsd.isAccessTimeSupported() && atime != -1) {
+ throw new IOException(
+ "Access time for hdfs is not configured. " +
+ " Please set " + DFS_NAMENODE_ACCESSTIME_PRECISION_KEY
+ + " configuration parameter.");
+ }
+
+ FSPermissionChecker pc = fsd.getPermissionChecker();
+ byte[][] pathComponents = FSDirectory.getPathComponentsForReservedPath(src);
+
+ INodesInPath iip;
+ fsd.writeLock();
+ try {
+ src = fsd.resolvePath(pc, src, pathComponents);
+ iip = fsd.getINodesInPath4Write(src);
+ // Write access is required to set access and modification times
+ if (fsd.isPermissionEnabled()) {
+ fsd.checkPathAccess(pc, iip, FsAction.WRITE);
+ }
+ final INode inode = iip.getLastINode();
+ if (inode == null) {
+ throw new FileNotFoundException("File/Directory " + src +
+ " does not exist.");
+ }
+ boolean changed = unprotectedSetTimes(fsd, inode, mtime, atime, true,
+ iip.getLatestSnapshotId());
+ if (changed) {
+ fsd.getEditLog().logTimes(src, mtime, atime);
+ }
+ } finally {
+ fsd.writeUnlock();
+ }
+ return fsd.getAuditFileInfo(iip);
+ }
+
+ static boolean setReplication(
+ FSDirectory fsd, BlockManager bm, String src, final short replication)
+ throws IOException {
+ bm.verifyReplication(src, replication, null);
+ final boolean isFile;
+ FSPermissionChecker pc = fsd.getPermissionChecker();
+ byte[][] pathComponents = FSDirectory.getPathComponentsForReservedPath(src);
+ fsd.writeLock();
+ try {
+ src = fsd.resolvePath(pc, src, pathComponents);
+ final INodesInPath iip = fsd.getINodesInPath4Write(src);
+ if (fsd.isPermissionEnabled()) {
+ fsd.checkPathAccess(pc, iip, FsAction.WRITE);
+ }
+
+ final short[] blockRepls = new short[2]; // 0: old, 1: new
+ final Block[] blocks = unprotectedSetReplication(fsd, src, replication,
+ blockRepls);
+ isFile = blocks != null;
+ if (isFile) {
+ fsd.getEditLog().logSetReplication(src, replication);
+ bm.setReplication(blockRepls[0], blockRepls[1], src, blocks);
+ }
+ } finally {
+ fsd.writeUnlock();
+ }
+ return isFile;
+ }
+
+ static HdfsFileStatus setStoragePolicy(
+ FSDirectory fsd, BlockManager bm, String src, final String policyName)
+ throws IOException {
+ if (!fsd.isStoragePolicyEnabled()) {
+ throw new IOException(
+ "Failed to set storage policy since "
+ + DFS_STORAGE_POLICY_ENABLED_KEY + " is set to false.");
+ }
+ FSPermissionChecker pc = fsd.getPermissionChecker();
+ byte[][] pathComponents = FSDirectory.getPathComponentsForReservedPath(src);
+ INodesInPath iip;
+ fsd.writeLock();
+ try {
+ src = FSDirectory.resolvePath(src, pathComponents, fsd);
+ iip = fsd.getINodesInPath4Write(src);
+
+ if (fsd.isPermissionEnabled()) {
+ fsd.checkPathAccess(pc, iip, FsAction.WRITE);
+ }
+
+ // get the corresponding policy and make sure the policy name is valid
+ BlockStoragePolicy policy = bm.getStoragePolicy(policyName);
+ if (policy == null) {
+ throw new HadoopIllegalArgumentException(
+ "Cannot find a block policy with the name " + policyName);
+ }
+ unprotectedSetStoragePolicy(fsd, bm, iip, policy.getId());
+ fsd.getEditLog().logSetStoragePolicy(src, policy.getId());
+ } finally {
+ fsd.writeUnlock();
+ }
+ return fsd.getAuditFileInfo(iip);
+ }
+
+ static BlockStoragePolicy[] getStoragePolicies(BlockManager bm)
+ throws IOException {
+ return bm.getStoragePolicies();
+ }
+
+ static long getPreferredBlockSize(FSDirectory fsd, String src)
+ throws IOException {
+ FSPermissionChecker pc = fsd.getPermissionChecker();
+ byte[][] pathComponents = FSDirectory.getPathComponentsForReservedPath(src);
+ fsd.readLock();
+ try {
+ src = fsd.resolvePath(pc, src, pathComponents);
+ final INodesInPath iip = fsd.getINodesInPath(src, false);
+ if (fsd.isPermissionEnabled()) {
+ fsd.checkTraverse(pc, iip);
+ }
+ return INodeFile.valueOf(iip.getLastINode(), src)
+ .getPreferredBlockSize();
+ } finally {
+ fsd.readUnlock();
+ }
+ }
+
+ /**
+ * Set the namespace quota and diskspace quota for a directory.
+ *
+ * Note: This does not support ".inodes" relative path.
+ */
+ static void setQuota(FSDirectory fsd, String src, long nsQuota, long dsQuota)
+ throws IOException {
+ if (fsd.isPermissionEnabled()) {
+ FSPermissionChecker pc = fsd.getPermissionChecker();
+ pc.checkSuperuserPrivilege();
+ }
+
+ fsd.writeLock();
+ try {
+ INodeDirectory changed = unprotectedSetQuota(fsd, src, nsQuota, dsQuota);
+ if (changed != null) {
+ final Quota.Counts q = changed.getQuotaCounts();
+ fsd.getEditLog().logSetQuota(
+ src, q.get(Quota.NAMESPACE), q.get(Quota.DISKSPACE));
+ }
+ } finally {
+ fsd.writeUnlock();
+ }
+ }
+
+ static void unprotectedSetPermission(
+ FSDirectory fsd, String src, FsPermission permissions)
+ throws FileNotFoundException, UnresolvedLinkException,
+ QuotaExceededException, SnapshotAccessControlException {
+ assert fsd.hasWriteLock();
+ final INodesInPath inodesInPath = fsd.getINodesInPath4Write(src, true);
+ final INode inode = inodesInPath.getLastINode();
+ if (inode == null) {
+ throw new FileNotFoundException("File does not exist: " + src);
+ }
+ int snapshotId = inodesInPath.getLatestSnapshotId();
+ inode.setPermission(permissions, snapshotId);
+ }
+
+ static void unprotectedSetOwner(
+ FSDirectory fsd, String src, String username, String groupname)
+ throws FileNotFoundException, UnresolvedLinkException,
+ QuotaExceededException, SnapshotAccessControlException {
+ assert fsd.hasWriteLock();
+ final INodesInPath inodesInPath = fsd.getINodesInPath4Write(src, true);
+ INode inode = inodesInPath.getLastINode();
+ if (inode == null) {
+ throw new FileNotFoundException("File does not exist: " + src);
+ }
+ if (username != null) {
+ inode = inode.setUser(username, inodesInPath.getLatestSnapshotId());
+ }
+ if (groupname != null) {
+ inode.setGroup(groupname, inodesInPath.getLatestSnapshotId());
+ }
+ }
+
+ static boolean setTimes(
+ FSDirectory fsd, INode inode, long mtime, long atime, boolean force,
+ int latestSnapshotId) throws QuotaExceededException {
+ fsd.writeLock();
+ try {
+ return unprotectedSetTimes(fsd, inode, mtime, atime, force,
+ latestSnapshotId);
+ } finally {
+ fsd.writeUnlock();
+ }
+ }
+
+ static boolean unprotectedSetTimes(
+ FSDirectory fsd, String src, long mtime, long atime, boolean force)
+ throws UnresolvedLinkException, QuotaExceededException {
+ assert fsd.hasWriteLock();
+ final INodesInPath i = fsd.getINodesInPath(src, true);
+ return unprotectedSetTimes(fsd, i.getLastINode(), mtime, atime,
+ force, i.getLatestSnapshotId());
+ }
+
+ /**
+ * See {@link org.apache.hadoop.hdfs.protocol.ClientProtocol#setQuota(String, long, long)}
+ * for the contract.
+ * Sets quota for for a directory.
+ * @return INodeDirectory if any of the quotas have changed. null otherwise.
+ * @throws FileNotFoundException if the path does not exist.
+ * @throws PathIsNotDirectoryException if the path is not a directory.
+ * @throws QuotaExceededException if the directory tree size is
+ * greater than the given quota
+ * @throws UnresolvedLinkException if a symlink is encountered in src.
+ * @throws SnapshotAccessControlException if path is in RO snapshot
+ */
+ static INodeDirectory unprotectedSetQuota(
+ FSDirectory fsd, String src, long nsQuota, long dsQuota)
+ throws FileNotFoundException, PathIsNotDirectoryException,
+ QuotaExceededException, UnresolvedLinkException,
+ SnapshotAccessControlException {
+ assert fsd.hasWriteLock();
+ // sanity check
+ if ((nsQuota < 0 && nsQuota != HdfsConstants.QUOTA_DONT_SET &&
+ nsQuota != HdfsConstants.QUOTA_RESET) ||
+ (dsQuota < 0 && dsQuota != HdfsConstants.QUOTA_DONT_SET &&
+ dsQuota != HdfsConstants.QUOTA_RESET)) {
+ throw new IllegalArgumentException("Illegal value for nsQuota or " +
+ "dsQuota : " + nsQuota + " and " +
+ dsQuota);
+ }
+
+ String srcs = FSDirectory.normalizePath(src);
+ final INodesInPath iip = fsd.getINodesInPath4Write(srcs, true);
+ INodeDirectory dirNode = INodeDirectory.valueOf(iip.getLastINode(), srcs);
+ if (dirNode.isRoot() && nsQuota == HdfsConstants.QUOTA_RESET) {
+ throw new IllegalArgumentException("Cannot clear namespace quota on root.");
+ } else { // a directory inode
+ final Quota.Counts oldQuota = dirNode.getQuotaCounts();
+ final long oldNsQuota = oldQuota.get(Quota.NAMESPACE);
+ final long oldDsQuota = oldQuota.get(Quota.DISKSPACE);
+ if (nsQuota == HdfsConstants.QUOTA_DONT_SET) {
+ nsQuota = oldNsQuota;
+ }
+ if (dsQuota == HdfsConstants.QUOTA_DONT_SET) {
+ dsQuota = oldDsQuota;
+ }
+ if (oldNsQuota == nsQuota && oldDsQuota == dsQuota) {
+ return null;
+ }
+
+ final int latest = iip.getLatestSnapshotId();
+ dirNode.recordModification(latest);
+ dirNode.setQuota(nsQuota, dsQuota);
+ return dirNode;
+ }
+ }
+
+ static Block[] unprotectedSetReplication(
+ FSDirectory fsd, String src, short replication, short[] blockRepls)
+ throws QuotaExceededException, UnresolvedLinkException,
+ SnapshotAccessControlException {
+ assert fsd.hasWriteLock();
+
+ final INodesInPath iip = fsd.getINodesInPath4Write(src, true);
+ final INode inode = iip.getLastINode();
+ if (inode == null || !inode.isFile()) {
+ return null;
+ }
+ INodeFile file = inode.asFile();
+ final short oldBR = file.getBlockReplication();
+
+ // before setFileReplication, check for increasing block replication.
+ // if replication > oldBR, then newBR == replication.
+ // if replication < oldBR, we don't know newBR yet.
+ if (replication > oldBR) {
+ long dsDelta = (replication - oldBR)*(file.diskspaceConsumed()/oldBR);
+ fsd.updateCount(iip, 0, dsDelta, true);
+ }
+
+ file.setFileReplication(replication, iip.getLatestSnapshotId());
+
+ final short newBR = file.getBlockReplication();
+ // check newBR < oldBR case.
+ if (newBR < oldBR) {
+ long dsDelta = (newBR - oldBR)*(file.diskspaceConsumed()/newBR);
+ fsd.updateCount(iip, 0, dsDelta, true);
+ }
+
+ if (blockRepls != null) {
+ blockRepls[0] = oldBR;
+ blockRepls[1] = newBR;
+ }
+ return file.getBlocks();
+ }
+
+ static void unprotectedSetStoragePolicy(
+ FSDirectory fsd, BlockManager bm, INodesInPath iip, byte policyId)
+ throws IOException {
+ assert fsd.hasWriteLock();
+ final INode inode = iip.getLastINode();
+ if (inode == null) {
+ throw new FileNotFoundException("File/Directory does not exist: "
+ + iip.getPath());
+ }
+ final int snapshotId = iip.getLatestSnapshotId();
+ if (inode.isFile()) {
+ BlockStoragePolicy newPolicy = bm.getStoragePolicy(policyId);
+ if (newPolicy.isCopyOnCreateFile()) {
+ throw new HadoopIllegalArgumentException(
+ "Policy " + newPolicy + " cannot be set after file creation.");
+ }
+
+ BlockStoragePolicy currentPolicy =
+ bm.getStoragePolicy(inode.getLocalStoragePolicyID());
+
+ if (currentPolicy != null && currentPolicy.isCopyOnCreateFile()) {
+ throw new HadoopIllegalArgumentException(
+ "Existing policy " + currentPolicy.getName() +
+ " cannot be changed after file creation.");
+ }
+ inode.asFile().setStoragePolicyID(policyId, snapshotId);
+ } else if (inode.isDirectory()) {
+ setDirStoragePolicy(fsd, inode.asDirectory(), policyId, snapshotId);
+ } else {
+ throw new FileNotFoundException(iip.getPath()
+ + " is not a file or directory");
+ }
+ }
+
+ private static void setDirStoragePolicy(
+ FSDirectory fsd, INodeDirectory inode, byte policyId,
+ int latestSnapshotId) throws IOException {
+ List existingXAttrs = XAttrStorage.readINodeXAttrs(inode);
+ XAttr xAttr = BlockStoragePolicySuite.buildXAttr(policyId);
+ List newXAttrs = FSDirXAttrOp.setINodeXAttrs(fsd, existingXAttrs,
+ Arrays.asList(xAttr),
+ EnumSet.of(
+ XAttrSetFlag.CREATE,
+ XAttrSetFlag.REPLACE));
+ XAttrStorage.updateINodeXAttrs(inode, newXAttrs, latestSnapshotId);
+ }
+
+ private static boolean unprotectedSetTimes(
+ FSDirectory fsd, INode inode, long mtime, long atime, boolean force,
+ int latest) throws QuotaExceededException {
+ assert fsd.hasWriteLock();
+ boolean status = false;
+ if (mtime != -1) {
+ inode = inode.setModificationTime(mtime, latest);
+ status = true;
+ }
+ if (atime != -1) {
+ long inodeTime = inode.getAccessTime();
+
+ // if the last access time update was within the last precision interval, then
+ // no need to store access time
+ if (atime <= inodeTime + fsd.getFSNamesystem().getAccessTimePrecision()
+ && !force) {
+ status = false;
+ } else {
+ inode.setAccessTime(atime, latest);
+ status = true;
+ }
+ }
+ return status;
+ }
+}
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirConcatOp.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirConcatOp.java
index 12feb33e04524..43d3b205f0e32 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirConcatOp.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirConcatOp.java
@@ -53,15 +53,17 @@ static HdfsFileStatus concat(
}
}
+ final INodesInPath trgIip = fsd.getINodesInPath4Write(target);
// write permission for the target
if (fsd.isPermissionEnabled()) {
FSPermissionChecker pc = fsd.getPermissionChecker();
- fsd.checkPathAccess(pc, target, FsAction.WRITE);
+ fsd.checkPathAccess(pc, trgIip, FsAction.WRITE);
// and srcs
for(String aSrc: srcs) {
- fsd.checkPathAccess(pc, aSrc, FsAction.READ); // read the file
- fsd.checkParentAccess(pc, aSrc, FsAction.WRITE); // for delete
+ final INodesInPath srcIip = fsd.getINodesInPath4Write(aSrc);
+ fsd.checkPathAccess(pc, srcIip, FsAction.READ); // read the file
+ fsd.checkParentAccess(pc, srcIip, FsAction.WRITE); // for delete
}
}
@@ -72,7 +74,6 @@ static HdfsFileStatus concat(
// replication and blocks sizes should be the same for ALL the blocks
// check the target
- final INodesInPath trgIip = fsd.getINodesInPath4Write(target);
if (fsd.getEZForPath(trgIip) != null) {
throw new HadoopIllegalArgumentException(
"concat can not be called for files in an encryption zone.");
@@ -167,7 +168,7 @@ static HdfsFileStatus concat(
fsd.writeUnlock();
}
fsd.getEditLog().logConcat(target, srcs, timestamp, logRetryCache);
- return fsd.getAuditFileInfo(target, false);
+ return fsd.getAuditFileInfo(trgIip);
}
/**
@@ -186,9 +187,8 @@ static void unprotectedConcat(
// do the move
final INodesInPath trgIIP = fsd.getINodesInPath4Write(target, true);
- final INode[] trgINodes = trgIIP.getINodes();
final INodeFile trgInode = trgIIP.getLastINode().asFile();
- INodeDirectory trgParent = trgINodes[trgINodes.length-2].asDirectory();
+ INodeDirectory trgParent = trgIIP.getINode(-2).asDirectory();
final int trgLatestSnapshot = trgIIP.getLatestSnapshotId();
final INodeFile [] allSrcInodes = new INodeFile[srcs.length];
@@ -228,6 +228,6 @@ static void unprotectedConcat(
trgInode.setModificationTime(timestamp, trgLatestSnapshot);
trgParent.updateModificationTime(timestamp, trgLatestSnapshot);
// update quota on the parent directory ('count' files removed, 0 space)
- FSDirectory.unprotectedUpdateCount(trgIIP, trgINodes.length - 1, -count, 0);
+ FSDirectory.unprotectedUpdateCount(trgIIP, trgIIP.length() - 1, -count, 0);
}
}
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirMkdirOp.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirMkdirOp.java
new file mode 100644
index 0000000000000..4ea77e6650a14
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirMkdirOp.java
@@ -0,0 +1,237 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.namenode;
+
+import org.apache.hadoop.fs.FileAlreadyExistsException;
+import org.apache.hadoop.fs.InvalidPathException;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.UnresolvedLinkException;
+import org.apache.hadoop.fs.permission.AclEntry;
+import org.apache.hadoop.fs.permission.FsAction;
+import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.fs.permission.PermissionStatus;
+import org.apache.hadoop.hdfs.DFSUtil;
+import org.apache.hadoop.hdfs.protocol.AclException;
+import org.apache.hadoop.hdfs.protocol.HdfsFileStatus;
+import org.apache.hadoop.hdfs.protocol.QuotaExceededException;
+import org.apache.hadoop.hdfs.protocol.SnapshotAccessControlException;
+import org.apache.hadoop.hdfs.server.namenode.snapshot.Snapshot;
+
+import java.io.IOException;
+import java.util.List;
+
+import static org.apache.hadoop.util.Time.now;
+
+class FSDirMkdirOp {
+ static HdfsFileStatus mkdirs(
+ FSNamesystem fsn, String src, PermissionStatus permissions,
+ boolean createParent) throws IOException {
+ FSDirectory fsd = fsn.getFSDirectory();
+ if(NameNode.stateChangeLog.isDebugEnabled()) {
+ NameNode.stateChangeLog.debug("DIR* NameSystem.mkdirs: " + src);
+ }
+ if (!DFSUtil.isValidName(src)) {
+ throw new InvalidPathException(src);
+ }
+ FSPermissionChecker pc = fsd.getPermissionChecker();
+ byte[][] pathComponents = FSDirectory.getPathComponentsForReservedPath(src);
+ src = fsd.resolvePath(pc, src, pathComponents);
+ INodesInPath iip = fsd.getINodesInPath4Write(src);
+ if (fsd.isPermissionEnabled()) {
+ fsd.checkTraverse(pc, iip);
+ }
+
+ if (!isDirMutable(fsd, iip)) {
+ if (fsd.isPermissionEnabled()) {
+ fsd.checkAncestorAccess(pc, iip, FsAction.WRITE);
+ }
+
+ if (!createParent) {
+ fsd.verifyParentDir(iip, src);
+ }
+
+ // validate that we have enough inodes. This is, at best, a
+ // heuristic because the mkdirs() operation might need to
+ // create multiple inodes.
+ fsn.checkFsObjectLimit();
+ iip = mkdirsRecursively(fsd, iip, permissions, false, now());
+ if (iip == null) {
+ throw new IOException("Failed to create directory: " + src);
+ }
+ }
+ return fsd.getAuditFileInfo(iip);
+ }
+
+ static INode unprotectedMkdir(
+ FSDirectory fsd, long inodeId, String src,
+ PermissionStatus permissions, List aclEntries, long timestamp)
+ throws QuotaExceededException, UnresolvedLinkException, AclException {
+ assert fsd.hasWriteLock();
+ byte[][] components = INode.getPathComponents(src);
+ final INodesInPath iip = fsd.getExistingPathINodes(components);
+ final int pos = iip.length() - 1;
+ final INodesInPath newiip = unprotectedMkdir(fsd, inodeId, iip, pos,
+ components[pos], permissions, aclEntries, timestamp);
+ return newiip.getINode(pos);
+ }
+
+ /**
+ * Create a directory
+ * If ancestor directories do not exist, automatically create them.
+
+ * @param fsd FSDirectory
+ * @param iip the INodesInPath instance containing all the existing INodes
+ * and null elements for non-existing components in the path
+ * @param permissions the permission of the directory
+ * @param inheritPermission
+ * if the permission of the directory should inherit from its parent or not.
+ * u+wx is implicitly added to the automatically created directories,
+ * and to the given directory if inheritPermission is true
+ * @param now creation time
+ * @return non-null INodesInPath instance if operation succeeds
+ * @throws QuotaExceededException if directory creation violates
+ * any quota limit
+ * @throws UnresolvedLinkException if a symlink is encountered in src.
+ * @throws SnapshotAccessControlException if path is in RO snapshot
+ */
+ static INodesInPath mkdirsRecursively(FSDirectory fsd, INodesInPath iip,
+ PermissionStatus permissions, boolean inheritPermission, long now)
+ throws FileAlreadyExistsException, QuotaExceededException,
+ UnresolvedLinkException, SnapshotAccessControlException,
+ AclException {
+ final int lastInodeIndex = iip.length() - 1;
+ final byte[][] components = iip.getPathComponents();
+ final String[] names = new String[components.length];
+ for (int i = 0; i < components.length; i++) {
+ names[i] = DFSUtil.bytes2String(components[i]);
+ }
+
+ fsd.writeLock();
+ try {
+ if (iip.isSnapshot()) {
+ throw new SnapshotAccessControlException(
+ "Modification on RO snapshot is disallowed");
+ }
+ final int length = iip.length();
+ // find the index of the first null in inodes[]
+ StringBuilder pathbuilder = new StringBuilder();
+ int i = 1;
+ INode curNode;
+ for(; i < length && (curNode = iip.getINode(i)) != null; i++) {
+ pathbuilder.append(Path.SEPARATOR).append(names[i]);
+ if (!curNode.isDirectory()) {
+ throw new FileAlreadyExistsException("Parent path is not a directory: "
+ + pathbuilder + " " + curNode.getLocalName());
+ }
+ }
+
+ // default to creating parent dirs with the given perms
+ PermissionStatus parentPermissions = permissions;
+
+ // if not inheriting and it's the last inode, there's no use in
+ // computing perms that won't be used
+ if (inheritPermission || (i < lastInodeIndex)) {
+ // if inheriting (ie. creating a file or symlink), use the parent dir,
+ // else the supplied permissions
+ // NOTE: the permissions of the auto-created directories violate posix
+ FsPermission parentFsPerm = inheritPermission ?
+ iip.getINode(i-1).getFsPermission() : permissions.getPermission();
+
+ // ensure that the permissions allow user write+execute
+ if (!parentFsPerm.getUserAction().implies(FsAction.WRITE_EXECUTE)) {
+ parentFsPerm = new FsPermission(
+ parentFsPerm.getUserAction().or(FsAction.WRITE_EXECUTE),
+ parentFsPerm.getGroupAction(),
+ parentFsPerm.getOtherAction()
+ );
+ }
+
+ if (!parentPermissions.getPermission().equals(parentFsPerm)) {
+ parentPermissions = new PermissionStatus(
+ parentPermissions.getUserName(),
+ parentPermissions.getGroupName(),
+ parentFsPerm
+ );
+ // when inheriting, use same perms for entire path
+ if (inheritPermission) permissions = parentPermissions;
+ }
+ }
+
+ // create directories beginning from the first null index
+ for(; i < length; i++) {
+ pathbuilder.append(Path.SEPARATOR).append(names[i]);
+ iip = unprotectedMkdir(fsd, fsd.allocateNewInodeId(), iip, i,
+ components[i], (i < lastInodeIndex) ? parentPermissions :
+ permissions, null, now);
+ if (iip.getINode(i) == null) {
+ return null;
+ }
+ // Directory creation also count towards FilesCreated
+ // to match count of FilesDeleted metric.
+ NameNode.getNameNodeMetrics().incrFilesCreated();
+
+ final String cur = pathbuilder.toString();
+ fsd.getEditLog().logMkDir(cur, iip.getINode(i));
+ if(NameNode.stateChangeLog.isDebugEnabled()) {
+ NameNode.stateChangeLog.debug(
+ "mkdirs: created directory " + cur);
+ }
+ }
+ } finally {
+ fsd.writeUnlock();
+ }
+ return iip;
+ }
+
+ /**
+ * Check whether the path specifies a directory
+ * @throws SnapshotAccessControlException if path is in RO snapshot
+ */
+ private static boolean isDirMutable(FSDirectory fsd, INodesInPath iip)
+ throws SnapshotAccessControlException {
+ fsd.readLock();
+ try {
+ INode node = iip.getLastINode();
+ return node != null && node.isDirectory();
+ } finally {
+ fsd.readUnlock();
+ }
+ }
+
+ /** create a directory at index pos.
+ * The parent path to the directory is at [0, pos-1].
+ * All ancestors exist. Newly created one stored at index pos.
+ */
+ private static INodesInPath unprotectedMkdir(
+ FSDirectory fsd, long inodeId, INodesInPath inodesInPath, int pos,
+ byte[] name, PermissionStatus permission, List aclEntries,
+ long timestamp)
+ throws QuotaExceededException, AclException {
+ assert fsd.hasWriteLock();
+ final INodeDirectory dir = new INodeDirectory(inodeId, name, permission,
+ timestamp);
+ if (fsd.addChild(inodesInPath, pos, dir, true)) {
+ if (aclEntries != null) {
+ AclStorage.updateINodeAcl(dir, aclEntries, Snapshot.CURRENT_STATE_ID);
+ }
+ return INodesInPath.replace(inodesInPath, pos, dir);
+ } else {
+ return inodesInPath;
+ }
+ }
+}
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirRenameOp.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirRenameOp.java
new file mode 100644
index 0000000000000..4239f46094555
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirRenameOp.java
@@ -0,0 +1,723 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.namenode;
+
+import org.apache.hadoop.fs.FileAlreadyExistsException;
+import org.apache.hadoop.fs.InvalidPathException;
+import org.apache.hadoop.fs.Options;
+import org.apache.hadoop.fs.ParentNotDirectoryException;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.permission.FsAction;
+import org.apache.hadoop.hdfs.DFSUtil;
+import org.apache.hadoop.hdfs.DistributedFileSystem;
+import org.apache.hadoop.hdfs.protocol.HdfsFileStatus;
+import org.apache.hadoop.hdfs.protocol.QuotaExceededException;
+import org.apache.hadoop.hdfs.protocol.SnapshotException;
+import org.apache.hadoop.hdfs.server.namenode.INode.BlocksMapUpdateInfo;
+import org.apache.hadoop.hdfs.server.namenode.snapshot.Snapshot;
+import org.apache.hadoop.hdfs.util.ReadOnlyList;
+import org.apache.hadoop.util.ChunkedArrayList;
+import org.apache.hadoop.util.Time;
+
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.util.AbstractMap;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Map;
+
+import static org.apache.hadoop.hdfs.protocol.FSLimitException.MaxDirectoryItemsExceededException;
+import static org.apache.hadoop.hdfs.protocol.FSLimitException.PathComponentTooLongException;
+
+class FSDirRenameOp {
+ @Deprecated
+ static RenameOldResult renameToInt(
+ FSDirectory fsd, final String srcArg, final String dstArg,
+ boolean logRetryCache)
+ throws IOException {
+ String src = srcArg;
+ String dst = dstArg;
+ if (NameNode.stateChangeLog.isDebugEnabled()) {
+ NameNode.stateChangeLog.debug("DIR* NameSystem.renameTo: " + src +
+ " to " + dst);
+ }
+ if (!DFSUtil.isValidName(dst)) {
+ throw new IOException("Invalid name: " + dst);
+ }
+ FSPermissionChecker pc = fsd.getPermissionChecker();
+
+ byte[][] srcComponents = FSDirectory.getPathComponentsForReservedPath(src);
+ byte[][] dstComponents = FSDirectory.getPathComponentsForReservedPath(dst);
+ HdfsFileStatus resultingStat = null;
+ src = fsd.resolvePath(pc, src, srcComponents);
+ dst = fsd.resolvePath(pc, dst, dstComponents);
+ @SuppressWarnings("deprecation")
+ final boolean status = renameTo(fsd, pc, src, dst, logRetryCache);
+ if (status) {
+ INodesInPath dstIIP = fsd.getINodesInPath(dst, false);
+ resultingStat = fsd.getAuditFileInfo(dstIIP);
+ }
+ return new RenameOldResult(status, resultingStat);
+ }
+
+ /**
+ * Verify quota for rename operation where srcInodes[srcInodes.length-1] moves
+ * dstInodes[dstInodes.length-1]
+ */
+ private static void verifyQuotaForRename(FSDirectory fsd, INodesInPath src,
+ INodesInPath dst) throws QuotaExceededException {
+ if (!fsd.getFSNamesystem().isImageLoaded() || fsd.shouldSkipQuotaChecks()) {
+ // Do not check quota if edits log is still being processed
+ return;
+ }
+ int i = 0;
+ while(src.getINode(i) == dst.getINode(i)) { i++; }
+ // src[i - 1] is the last common ancestor.
+
+ final Quota.Counts delta = src.getLastINode().computeQuotaUsage();
+
+ // Reduce the required quota by dst that is being removed
+ final INode dstINode = dst.getLastINode();
+ if (dstINode != null) {
+ delta.subtract(dstINode.computeQuotaUsage());
+ }
+ FSDirectory.verifyQuota(dst, dst.length() - 1, delta.get(Quota.NAMESPACE),
+ delta.get(Quota.DISKSPACE), src.getINode(i - 1));
+ }
+
+ /**
+ * Checks file system limits (max component length and max directory items)
+ * during a rename operation.
+ */
+ static void verifyFsLimitsForRename(FSDirectory fsd, INodesInPath srcIIP,
+ INodesInPath dstIIP)
+ throws PathComponentTooLongException, MaxDirectoryItemsExceededException {
+ byte[] dstChildName = dstIIP.getLastLocalName();
+ final String parentPath = dstIIP.getParentPath();
+ fsd.verifyMaxComponentLength(dstChildName, parentPath);
+ // Do not enforce max directory items if renaming within same directory.
+ if (srcIIP.getINode(-2) != dstIIP.getINode(-2)) {
+ fsd.verifyMaxDirItems(dstIIP.getINode(-2).asDirectory(), parentPath);
+ }
+ }
+
+ /**
+ *
+ * Note: This is to be used by {@link FSEditLogLoader} only.
+ *
+ */
+ @Deprecated
+ @SuppressWarnings("deprecation")
+ static boolean unprotectedRenameTo(FSDirectory fsd, String src, String dst,
+ long timestamp) throws IOException {
+ if (fsd.isDir(dst)) {
+ dst += Path.SEPARATOR + new Path(src).getName();
+ }
+ final INodesInPath srcIIP = fsd.getINodesInPath4Write(src, false);
+ final INodesInPath dstIIP = fsd.getINodesInPath4Write(dst, false);
+ return unprotectedRenameTo(fsd, src, dst, srcIIP, dstIIP, timestamp);
+ }
+
+ /**
+ * Change a path name
+ *
+ * @param fsd FSDirectory
+ * @param src source path
+ * @param dst destination path
+ * @return true if rename succeeds; false otherwise
+ * @deprecated See {@link #renameToInt(FSDirectory, String, String,
+ * boolean, Options.Rename...)}
+ */
+ @Deprecated
+ static boolean unprotectedRenameTo(FSDirectory fsd, String src, String dst,
+ final INodesInPath srcIIP, final INodesInPath dstIIP, long timestamp)
+ throws IOException {
+ assert fsd.hasWriteLock();
+ final INode srcInode = srcIIP.getLastINode();
+ try {
+ validateRenameSource(srcIIP);
+ } catch (SnapshotException e) {
+ throw e;
+ } catch (IOException ignored) {
+ return false;
+ }
+
+ // validate the destination
+ if (dst.equals(src)) {
+ return true;
+ }
+
+ try {
+ validateDestination(src, dst, srcInode);
+ } catch (IOException ignored) {
+ return false;
+ }
+
+ if (dstIIP.getLastINode() != null) {
+ NameNode.stateChangeLog.warn("DIR* FSDirectory.unprotectedRenameTo: " +
+ "failed to rename " + src + " to " + dst + " because destination " +
+ "exists");
+ return false;
+ }
+ INode dstParent = dstIIP.getINode(-2);
+ if (dstParent == null) {
+ NameNode.stateChangeLog.warn("DIR* FSDirectory.unprotectedRenameTo: " +
+ "failed to rename " + src + " to " + dst + " because destination's " +
+ "parent does not exist");
+ return false;
+ }
+
+ fsd.ezManager.checkMoveValidity(srcIIP, dstIIP, src);
+ // Ensure dst has quota to accommodate rename
+ verifyFsLimitsForRename(fsd, srcIIP, dstIIP);
+ verifyQuotaForRename(fsd, srcIIP, dstIIP);
+
+ RenameOperation tx = new RenameOperation(fsd, src, dst, srcIIP, dstIIP);
+
+ boolean added = false;
+
+ try {
+ // remove src
+ final long removedSrc = fsd.removeLastINode(tx.srcIIP);
+ if (removedSrc == -1) {
+ NameNode.stateChangeLog.warn("DIR* FSDirectory.unprotectedRenameTo: "
+ + "failed to rename " + src + " to " + dst + " because the source" +
+ " can not be removed");
+ return false;
+ }
+
+ added = tx.addSourceToDestination();
+ if (added) {
+ if (NameNode.stateChangeLog.isDebugEnabled()) {
+ NameNode.stateChangeLog.debug("DIR* FSDirectory" +
+ ".unprotectedRenameTo: " + src + " is renamed to " + dst);
+ }
+
+ tx.updateMtimeAndLease(timestamp);
+ tx.updateQuotasInSourceTree();
+
+ return true;
+ }
+ } finally {
+ if (!added) {
+ tx.restoreSource();
+ }
+ }
+ NameNode.stateChangeLog.warn("DIR* FSDirectory.unprotectedRenameTo: " +
+ "failed to rename " + src + " to " + dst);
+ return false;
+ }
+
+ /**
+ * The new rename which has the POSIX semantic.
+ */
+ static Map.Entry renameToInt(
+ FSDirectory fsd, final String srcArg, final String dstArg,
+ boolean logRetryCache, Options.Rename... options)
+ throws IOException {
+ String src = srcArg;
+ String dst = dstArg;
+ if (NameNode.stateChangeLog.isDebugEnabled()) {
+ NameNode.stateChangeLog.debug("DIR* NameSystem.renameTo: with options -" +
+ " " + src + " to " + dst);
+ }
+ if (!DFSUtil.isValidName(dst)) {
+ throw new InvalidPathException("Invalid name: " + dst);
+ }
+ final FSPermissionChecker pc = fsd.getPermissionChecker();
+
+ byte[][] srcComponents = FSDirectory.getPathComponentsForReservedPath(src);
+ byte[][] dstComponents = FSDirectory.getPathComponentsForReservedPath(dst);
+ BlocksMapUpdateInfo collectedBlocks = new BlocksMapUpdateInfo();
+ src = fsd.resolvePath(pc, src, srcComponents);
+ dst = fsd.resolvePath(pc, dst, dstComponents);
+ renameTo(fsd, pc, src, dst, collectedBlocks, logRetryCache, options);
+ INodesInPath dstIIP = fsd.getINodesInPath(dst, false);
+ HdfsFileStatus resultingStat = fsd.getAuditFileInfo(dstIIP);
+
+ return new AbstractMap.SimpleImmutableEntry<>(
+ collectedBlocks, resultingStat);
+ }
+
+ /**
+ * @see #unprotectedRenameTo(FSDirectory, String, String, long,
+ * org.apache.hadoop.fs.Options.Rename...)
+ */
+ static void renameTo(FSDirectory fsd, FSPermissionChecker pc, String src,
+ String dst, BlocksMapUpdateInfo collectedBlocks, boolean logRetryCache,
+ Options.Rename... options) throws IOException {
+ final INodesInPath srcIIP = fsd.getINodesInPath4Write(src, false);
+ final INodesInPath dstIIP = fsd.getINodesInPath4Write(dst, false);
+ if (fsd.isPermissionEnabled()) {
+ // Rename does not operate on link targets
+ // Do not resolveLink when checking permissions of src and dst
+ // Check write access to parent of src
+ fsd.checkPermission(pc, srcIIP, false, null, FsAction.WRITE, null, null,
+ false);
+ // Check write access to ancestor of dst
+ fsd.checkPermission(pc, dstIIP, false, FsAction.WRITE, null, null, null,
+ false);
+ }
+
+ if (NameNode.stateChangeLog.isDebugEnabled()) {
+ NameNode.stateChangeLog.debug("DIR* FSDirectory.renameTo: " + src + " to "
+ + dst);
+ }
+ final long mtime = Time.now();
+ fsd.writeLock();
+ try {
+ if (unprotectedRenameTo(fsd, src, dst, srcIIP, dstIIP, mtime,
+ collectedBlocks, options)) {
+ fsd.getFSNamesystem().incrDeletedFileCount(1);
+ }
+ } finally {
+ fsd.writeUnlock();
+ }
+ fsd.getEditLog().logRename(src, dst, mtime, logRetryCache, options);
+ }
+
+ /**
+ * Rename src to dst.
+ *
+ * Note: This is to be used by {@link org.apache.hadoop.hdfs.server
+ * .namenode.FSEditLogLoader} only.
+ *
+ *
+ * @param fsd FSDirectory
+ * @param src source path
+ * @param dst destination path
+ * @param timestamp modification time
+ * @param options Rename options
+ */
+ static boolean unprotectedRenameTo(
+ FSDirectory fsd, String src, String dst, long timestamp,
+ Options.Rename... options)
+ throws IOException {
+ BlocksMapUpdateInfo collectedBlocks = new BlocksMapUpdateInfo();
+ final INodesInPath srcIIP = fsd.getINodesInPath4Write(src, false);
+ final INodesInPath dstIIP = fsd.getINodesInPath4Write(dst, false);
+ boolean ret = unprotectedRenameTo(fsd, src, dst, srcIIP, dstIIP, timestamp,
+ collectedBlocks, options);
+ if (!collectedBlocks.getToDeleteList().isEmpty()) {
+ fsd.getFSNamesystem().removeBlocksAndUpdateSafemodeTotal(collectedBlocks);
+ }
+ return ret;
+ }
+
+ /**
+ * Rename src to dst.
+ * See {@link DistributedFileSystem#rename(Path, Path, Options.Rename...)}
+ * for details related to rename semantics and exceptions.
+ *
+ * @param fsd FSDirectory
+ * @param src source path
+ * @param dst destination path
+ * @param timestamp modification time
+ * @param collectedBlocks blocks to be removed
+ * @param options Rename options
+ */
+ static boolean unprotectedRenameTo(FSDirectory fsd, String src, String dst,
+ final INodesInPath srcIIP, final INodesInPath dstIIP, long timestamp,
+ BlocksMapUpdateInfo collectedBlocks, Options.Rename... options)
+ throws IOException {
+ assert fsd.hasWriteLock();
+ boolean overwrite = options != null
+ && Arrays.asList(options).contains(Options.Rename.OVERWRITE);
+
+ final String error;
+ final INode srcInode = srcIIP.getLastINode();
+ validateRenameSource(srcIIP);
+
+ // validate the destination
+ if (dst.equals(src)) {
+ throw new FileAlreadyExistsException("The source " + src +
+ " and destination " + dst + " are the same");
+ }
+ validateDestination(src, dst, srcInode);
+
+ if (dstIIP.length() == 1) {
+ error = "rename destination cannot be the root";
+ NameNode.stateChangeLog.warn("DIR* FSDirectory.unprotectedRenameTo: " +
+ error);
+ throw new IOException(error);
+ }
+
+ fsd.ezManager.checkMoveValidity(srcIIP, dstIIP, src);
+ final INode dstInode = dstIIP.getLastINode();
+ List snapshottableDirs = new ArrayList<>();
+ if (dstInode != null) { // Destination exists
+ validateOverwrite(src, dst, overwrite, srcInode, dstInode);
+ FSDirSnapshotOp.checkSnapshot(dstInode, snapshottableDirs);
+ }
+
+ INode dstParent = dstIIP.getINode(-2);
+ if (dstParent == null) {
+ error = "rename destination parent " + dst + " not found.";
+ NameNode.stateChangeLog.warn("DIR* FSDirectory.unprotectedRenameTo: " +
+ error);
+ throw new FileNotFoundException(error);
+ }
+ if (!dstParent.isDirectory()) {
+ error = "rename destination parent " + dst + " is a file.";
+ NameNode.stateChangeLog.warn("DIR* FSDirectory.unprotectedRenameTo: " +
+ error);
+ throw new ParentNotDirectoryException(error);
+ }
+
+ // Ensure dst has quota to accommodate rename
+ verifyFsLimitsForRename(fsd, srcIIP, dstIIP);
+ verifyQuotaForRename(fsd, srcIIP, dstIIP);
+
+ RenameOperation tx = new RenameOperation(fsd, src, dst, srcIIP, dstIIP);
+
+ boolean undoRemoveSrc = true;
+ final long removedSrc = fsd.removeLastINode(tx.srcIIP);
+ if (removedSrc == -1) {
+ error = "Failed to rename " + src + " to " + dst +
+ " because the source can not be removed";
+ NameNode.stateChangeLog.warn("DIR* FSDirectory.unprotectedRenameTo: " +
+ error);
+ throw new IOException(error);
+ }
+
+ boolean undoRemoveDst = false;
+ INode removedDst = null;
+ long removedNum = 0;
+ try {
+ if (dstInode != null) { // dst exists remove it
+ if ((removedNum = fsd.removeLastINode(tx.dstIIP)) != -1) {
+ removedDst = tx.dstIIP.getLastINode();
+ undoRemoveDst = true;
+ }
+ }
+
+ // add src as dst to complete rename
+ if (tx.addSourceToDestination()) {
+ undoRemoveSrc = false;
+ if (NameNode.stateChangeLog.isDebugEnabled()) {
+ NameNode.stateChangeLog.debug("DIR* FSDirectory.unprotectedRenameTo: "
+ + src + " is renamed to " + dst);
+ }
+
+ tx.updateMtimeAndLease(timestamp);
+
+ // Collect the blocks and remove the lease for previous dst
+ boolean filesDeleted = false;
+ if (removedDst != null) {
+ undoRemoveDst = false;
+ if (removedNum > 0) {
+ List removedINodes = new ChunkedArrayList<>();
+ if (!removedDst.isInLatestSnapshot(tx.dstIIP.getLatestSnapshotId())) {
+ removedDst.destroyAndCollectBlocks(collectedBlocks,
+ removedINodes);
+ filesDeleted = true;
+ } else {
+ filesDeleted = removedDst.cleanSubtree(
+ Snapshot.CURRENT_STATE_ID, tx.dstIIP.getLatestSnapshotId(),
+ collectedBlocks, removedINodes, true)
+ .get(Quota.NAMESPACE) >= 0;
+ }
+ fsd.getFSNamesystem().removePathAndBlocks(src, null,
+ removedINodes, false);
+ }
+ }
+
+ if (snapshottableDirs.size() > 0) {
+ // There are snapshottable directories (without snapshots) to be
+ // deleted. Need to update the SnapshotManager.
+ fsd.getFSNamesystem().removeSnapshottableDirs(snapshottableDirs);
+ }
+
+ tx.updateQuotasInSourceTree();
+ return filesDeleted;
+ }
+ } finally {
+ if (undoRemoveSrc) {
+ tx.restoreSource();
+ }
+
+ if (undoRemoveDst) {
+ // Rename failed - restore dst
+ if (dstParent.isDirectory() &&
+ dstParent.asDirectory().isWithSnapshot()) {
+ dstParent.asDirectory().undoRename4DstParent(removedDst,
+ dstIIP.getLatestSnapshotId());
+ } else {
+ fsd.addLastINodeNoQuotaCheck(tx.dstIIP, removedDst);
+ }
+ if (removedDst.isReference()) {
+ final INodeReference removedDstRef = removedDst.asReference();
+ final INodeReference.WithCount wc = (INodeReference.WithCount)
+ removedDstRef.getReferredINode().asReference();
+ wc.addReference(removedDstRef);
+ }
+ }
+ }
+ NameNode.stateChangeLog.warn("DIR* FSDirectory.unprotectedRenameTo: " +
+ "failed to rename " + src + " to " + dst);
+ throw new IOException("rename from " + src + " to " + dst + " failed.");
+ }
+
+ /**
+ * @deprecated Use {@link #renameToInt(FSDirectory, String, String,
+ * boolean, Options.Rename...)}
+ */
+ @Deprecated
+ @SuppressWarnings("deprecation")
+ private static boolean renameTo(FSDirectory fsd, FSPermissionChecker pc,
+ String src, String dst, boolean logRetryCache) throws IOException {
+ // Rename does not operate on link targets
+ // Do not resolveLink when checking permissions of src and dst
+ // Check write access to parent of src
+ final INodesInPath srcIIP = fsd.getINodesInPath4Write(src, false);
+ // Note: We should not be doing this. This is move() not renameTo().
+ final String actualDst = fsd.isDir(dst) ?
+ dst + Path.SEPARATOR + new Path(src).getName() : dst;
+ final INodesInPath dstIIP = fsd.getINodesInPath4Write(actualDst, false);
+ if (fsd.isPermissionEnabled()) {
+ fsd.checkPermission(pc, srcIIP, false, null, FsAction.WRITE, null, null,
+ false);
+ // Check write access to ancestor of dst
+ fsd.checkPermission(pc, dstIIP, false, FsAction.WRITE, null, null,
+ null, false);
+ }
+
+ if (NameNode.stateChangeLog.isDebugEnabled()) {
+ NameNode.stateChangeLog.debug("DIR* FSDirectory.renameTo: " + src + " to "
+ + dst);
+ }
+ final long mtime = Time.now();
+ boolean stat = false;
+ fsd.writeLock();
+ try {
+ stat = unprotectedRenameTo(fsd, src, actualDst, srcIIP, dstIIP, mtime);
+ } finally {
+ fsd.writeUnlock();
+ }
+ if (stat) {
+ fsd.getEditLog().logRename(src, dst, mtime, logRetryCache);
+ return true;
+ }
+ return false;
+ }
+
+ private static void validateDestination(
+ String src, String dst, INode srcInode)
+ throws IOException {
+ String error;
+ if (srcInode.isSymlink() &&
+ dst.equals(srcInode.asSymlink().getSymlinkString())) {
+ throw new FileAlreadyExistsException("Cannot rename symlink " + src
+ + " to its target " + dst);
+ }
+ // dst cannot be a directory or a file under src
+ if (dst.startsWith(src)
+ && dst.charAt(src.length()) == Path.SEPARATOR_CHAR) {
+ error = "Rename destination " + dst
+ + " is a directory or file under source " + src;
+ NameNode.stateChangeLog.warn("DIR* FSDirectory.unprotectedRenameTo: "
+ + error);
+ throw new IOException(error);
+ }
+ }
+
+ private static void validateOverwrite(
+ String src, String dst, boolean overwrite, INode srcInode, INode dstInode)
+ throws IOException {
+ String error;// It's OK to rename a file to a symlink and vice versa
+ if (dstInode.isDirectory() != srcInode.isDirectory()) {
+ error = "Source " + src + " and destination " + dst
+ + " must both be directories";
+ NameNode.stateChangeLog.warn("DIR* FSDirectory.unprotectedRenameTo: "
+ + error);
+ throw new IOException(error);
+ }
+ if (!overwrite) { // If destination exists, overwrite flag must be true
+ error = "rename destination " + dst + " already exists";
+ NameNode.stateChangeLog.warn("DIR* FSDirectory.unprotectedRenameTo: "
+ + error);
+ throw new FileAlreadyExistsException(error);
+ }
+ if (dstInode.isDirectory()) {
+ final ReadOnlyList children = dstInode.asDirectory()
+ .getChildrenList(Snapshot.CURRENT_STATE_ID);
+ if (!children.isEmpty()) {
+ error = "rename destination directory is not empty: " + dst;
+ NameNode.stateChangeLog.warn("DIR* FSDirectory.unprotectedRenameTo: "
+ + error);
+ throw new IOException(error);
+ }
+ }
+ }
+
+ private static void validateRenameSource(INodesInPath srcIIP)
+ throws IOException {
+ String error;
+ final INode srcInode = srcIIP.getLastINode();
+ // validate source
+ if (srcInode == null) {
+ error = "rename source " + srcIIP.getPath() + " is not found.";
+ NameNode.stateChangeLog.warn("DIR* FSDirectory.unprotectedRenameTo: "
+ + error);
+ throw new FileNotFoundException(error);
+ }
+ if (srcIIP.length() == 1) {
+ error = "rename source cannot be the root";
+ NameNode.stateChangeLog.warn("DIR* FSDirectory.unprotectedRenameTo: "
+ + error);
+ throw new IOException(error);
+ }
+ // srcInode and its subtree cannot contain snapshottable directories with
+ // snapshots
+ FSDirSnapshotOp.checkSnapshot(srcInode, null);
+ }
+
+ private static class RenameOperation {
+ private final FSDirectory fsd;
+ private final INodesInPath srcIIP;
+ private final INodesInPath dstIIP;
+ private final String src;
+ private final String dst;
+ private final INodeReference.WithCount withCount;
+ private final int srcRefDstSnapshot;
+ private final INodeDirectory srcParent;
+ private final byte[] srcChildName;
+ private final boolean isSrcInSnapshot;
+ private final boolean srcChildIsReference;
+ private final Quota.Counts oldSrcCounts;
+ private INode srcChild;
+
+ RenameOperation(FSDirectory fsd, String src, String dst,
+ INodesInPath srcIIP, INodesInPath dstIIP)
+ throws QuotaExceededException {
+ this.fsd = fsd;
+ this.dstIIP = dstIIP;
+ this.src = src;
+ this.dst = dst;
+ srcChild = srcIIP.getLastINode();
+ srcChildName = srcChild.getLocalNameBytes();
+ isSrcInSnapshot = srcChild.isInLatestSnapshot(srcIIP.getLatestSnapshotId());
+ srcChildIsReference = srcChild.isReference();
+ srcParent = srcIIP.getINode(-2).asDirectory();
+
+ // Record the snapshot on srcChild. After the rename, before any new
+ // snapshot is taken on the dst tree, changes will be recorded in the
+ // latest snapshot of the src tree.
+ if (isSrcInSnapshot) {
+ srcChild.recordModification(srcIIP.getLatestSnapshotId());
+ }
+
+ // check srcChild for reference
+ srcRefDstSnapshot = srcChildIsReference ?
+ srcChild.asReference().getDstSnapshotId() : Snapshot.CURRENT_STATE_ID;
+ oldSrcCounts = Quota.Counts.newInstance();
+ if (isSrcInSnapshot) {
+ final INodeReference.WithName withName =
+ srcIIP.getINode(-2).asDirectory().replaceChild4ReferenceWithName(
+ srcChild, srcIIP.getLatestSnapshotId());
+ withCount = (INodeReference.WithCount) withName.getReferredINode();
+ srcChild = withName;
+ srcIIP = INodesInPath.replace(srcIIP, srcIIP.length() - 1, srcChild);
+ // get the counts before rename
+ withCount.getReferredINode().computeQuotaUsage(oldSrcCounts, true);
+ } else if (srcChildIsReference) {
+ // srcChild is reference but srcChild is not in latest snapshot
+ withCount = (INodeReference.WithCount) srcChild.asReference()
+ .getReferredINode();
+ } else {
+ withCount = null;
+ }
+ this.srcIIP = srcIIP;
+ }
+
+ boolean addSourceToDestination() {
+ final INode dstParent = dstIIP.getINode(-2);
+ srcChild = srcIIP.getLastINode();
+ final byte[] dstChildName = dstIIP.getLastLocalName();
+ final INode toDst;
+ if (withCount == null) {
+ srcChild.setLocalName(dstChildName);
+ toDst = srcChild;
+ } else {
+ withCount.getReferredINode().setLocalName(dstChildName);
+ int dstSnapshotId = dstIIP.getLatestSnapshotId();
+ toDst = new INodeReference.DstReference(dstParent.asDirectory(),
+ withCount, dstSnapshotId);
+ }
+ return fsd.addLastINodeNoQuotaCheck(dstIIP, toDst);
+ }
+
+ void updateMtimeAndLease(long timestamp) throws QuotaExceededException {
+ srcParent.updateModificationTime(timestamp, srcIIP.getLatestSnapshotId());
+ final INode dstParent = dstIIP.getINode(-2);
+ dstParent.updateModificationTime(timestamp, dstIIP.getLatestSnapshotId());
+ // update moved lease with new filename
+ fsd.getFSNamesystem().unprotectedChangeLease(src, dst);
+ }
+
+ void restoreSource() throws QuotaExceededException {
+ // Rename failed - restore src
+ final INode oldSrcChild = srcChild;
+ // put it back
+ if (withCount == null) {
+ srcChild.setLocalName(srcChildName);
+ } else if (!srcChildIsReference) { // src must be in snapshot
+ // the withCount node will no longer be used thus no need to update
+ // its reference number here
+ srcChild = withCount.getReferredINode();
+ srcChild.setLocalName(srcChildName);
+ } else {
+ withCount.removeReference(oldSrcChild.asReference());
+ srcChild = new INodeReference.DstReference(srcParent, withCount,
+ srcRefDstSnapshot);
+ withCount.getReferredINode().setLocalName(srcChildName);
+ }
+
+ if (isSrcInSnapshot) {
+ srcParent.undoRename4ScrParent(oldSrcChild.asReference(), srcChild);
+ } else {
+ // srcParent is not an INodeDirectoryWithSnapshot, we only need to add
+ // the srcChild back
+ fsd.addLastINodeNoQuotaCheck(srcIIP, srcChild);
+ }
+ }
+
+ void updateQuotasInSourceTree() throws QuotaExceededException {
+ // update the quota usage in src tree
+ if (isSrcInSnapshot) {
+ // get the counts after rename
+ Quota.Counts newSrcCounts = srcChild.computeQuotaUsage(
+ Quota.Counts.newInstance(), false);
+ newSrcCounts.subtract(oldSrcCounts);
+ srcParent.addSpaceConsumed(newSrcCounts.get(Quota.NAMESPACE),
+ newSrcCounts.get(Quota.DISKSPACE), false);
+ }
+ }
+ }
+
+ static class RenameOldResult {
+ final boolean success;
+ final HdfsFileStatus auditStat;
+
+ RenameOldResult(boolean success, HdfsFileStatus auditStat) {
+ this.success = success;
+ this.auditStat = auditStat;
+ }
+ }
+}
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirSnapshotOp.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirSnapshotOp.java
new file mode 100644
index 0000000000000..833bc30245a8b
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirSnapshotOp.java
@@ -0,0 +1,238 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.namenode;
+
+import org.apache.hadoop.HadoopIllegalArgumentException;
+import org.apache.hadoop.fs.InvalidPathException;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.permission.FsAction;
+import org.apache.hadoop.hdfs.DFSUtil;
+import org.apache.hadoop.hdfs.protocol.FSLimitException;
+import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport;
+import org.apache.hadoop.hdfs.protocol.SnapshotException;
+import org.apache.hadoop.hdfs.protocol.SnapshottableDirectoryStatus;
+import org.apache.hadoop.hdfs.server.namenode.snapshot.DirectorySnapshottableFeature;
+import org.apache.hadoop.hdfs.server.namenode.snapshot.Snapshot;
+import org.apache.hadoop.hdfs.server.namenode.snapshot.SnapshotManager;
+import org.apache.hadoop.util.ChunkedArrayList;
+
+import java.io.IOException;
+import java.util.List;
+
+class FSDirSnapshotOp {
+ /** Verify if the snapshot name is legal. */
+ static void verifySnapshotName(FSDirectory fsd, String snapshotName,
+ String path)
+ throws FSLimitException.PathComponentTooLongException {
+ if (snapshotName.contains(Path.SEPARATOR)) {
+ throw new HadoopIllegalArgumentException(
+ "Snapshot name cannot contain \"" + Path.SEPARATOR + "\"");
+ }
+ final byte[] bytes = DFSUtil.string2Bytes(snapshotName);
+ fsd.verifyINodeName(bytes);
+ fsd.verifyMaxComponentLength(bytes, path);
+ }
+
+ /** Allow snapshot on a directory. */
+ static void allowSnapshot(FSDirectory fsd, SnapshotManager snapshotManager,
+ String path) throws IOException {
+ fsd.writeLock();
+ try {
+ snapshotManager.setSnapshottable(path, true);
+ } finally {
+ fsd.writeUnlock();
+ }
+ fsd.getEditLog().logAllowSnapshot(path);
+ }
+
+ static void disallowSnapshot(
+ FSDirectory fsd, SnapshotManager snapshotManager,
+ String path) throws IOException {
+ fsd.writeLock();
+ try {
+ snapshotManager.resetSnapshottable(path);
+ } finally {
+ fsd.writeUnlock();
+ }
+ fsd.getEditLog().logDisallowSnapshot(path);
+ }
+
+ /**
+ * Create a snapshot
+ * @param snapshotRoot The directory path where the snapshot is taken
+ * @param snapshotName The name of the snapshot
+ */
+ static String createSnapshot(
+ FSDirectory fsd, SnapshotManager snapshotManager, String snapshotRoot,
+ String snapshotName, boolean logRetryCache)
+ throws IOException {
+ final INodesInPath iip = fsd.getINodesInPath4Write(snapshotRoot);
+ if (fsd.isPermissionEnabled()) {
+ FSPermissionChecker pc = fsd.getPermissionChecker();
+ fsd.checkOwner(pc, iip);
+ }
+
+ if (snapshotName == null || snapshotName.isEmpty()) {
+ snapshotName = Snapshot.generateDefaultSnapshotName();
+ } else if (!DFSUtil.isValidNameForComponent(snapshotName)) {
+ throw new InvalidPathException("Invalid snapshot name: " + snapshotName);
+ }
+
+ String snapshotPath = null;
+ verifySnapshotName(fsd, snapshotName, snapshotRoot);
+ fsd.writeLock();
+ try {
+ snapshotPath = snapshotManager.createSnapshot(iip, snapshotRoot,
+ snapshotName);
+ } finally {
+ fsd.writeUnlock();
+ }
+ fsd.getEditLog().logCreateSnapshot(snapshotRoot, snapshotName,
+ logRetryCache);
+
+ return snapshotPath;
+ }
+
+ static void renameSnapshot(FSDirectory fsd, SnapshotManager snapshotManager,
+ String path, String snapshotOldName, String snapshotNewName,
+ boolean logRetryCache) throws IOException {
+ final INodesInPath iip = fsd.getINodesInPath4Write(path);
+ if (fsd.isPermissionEnabled()) {
+ FSPermissionChecker pc = fsd.getPermissionChecker();
+ fsd.checkOwner(pc, iip);
+ }
+ verifySnapshotName(fsd, snapshotNewName, path);
+ fsd.writeLock();
+ try {
+ snapshotManager.renameSnapshot(iip, path, snapshotOldName,
+ snapshotNewName);
+ } finally {
+ fsd.writeUnlock();
+ }
+ fsd.getEditLog().logRenameSnapshot(path, snapshotOldName,
+ snapshotNewName, logRetryCache);
+ }
+
+ static SnapshottableDirectoryStatus[] getSnapshottableDirListing(
+ FSDirectory fsd, SnapshotManager snapshotManager) throws IOException {
+ FSPermissionChecker pc = fsd.getPermissionChecker();
+ fsd.readLock();
+ try {
+ final String user = pc.isSuperUser()? null : pc.getUser();
+ return snapshotManager.getSnapshottableDirListing(user);
+ } finally {
+ fsd.readUnlock();
+ }
+ }
+
+ static SnapshotDiffReport getSnapshotDiffReport(FSDirectory fsd,
+ SnapshotManager snapshotManager, String path,
+ String fromSnapshot, String toSnapshot) throws IOException {
+ SnapshotDiffReport diffs;
+ final FSPermissionChecker pc = fsd.getPermissionChecker();
+ fsd.readLock();
+ try {
+ if (fsd.isPermissionEnabled()) {
+ checkSubtreeReadPermission(fsd, pc, path, fromSnapshot);
+ checkSubtreeReadPermission(fsd, pc, path, toSnapshot);
+ }
+ INodesInPath iip = fsd.getINodesInPath(path, true);
+ diffs = snapshotManager.diff(iip, path, fromSnapshot, toSnapshot);
+ } finally {
+ fsd.readUnlock();
+ }
+ return diffs;
+ }
+
+ /**
+ * Delete a snapshot of a snapshottable directory
+ * @param snapshotRoot The snapshottable directory
+ * @param snapshotName The name of the to-be-deleted snapshot
+ * @throws IOException
+ */
+ static INode.BlocksMapUpdateInfo deleteSnapshot(
+ FSDirectory fsd, SnapshotManager snapshotManager, String snapshotRoot,
+ String snapshotName, boolean logRetryCache)
+ throws IOException {
+ final INodesInPath iip = fsd.getINodesInPath4Write(snapshotRoot);
+ if (fsd.isPermissionEnabled()) {
+ FSPermissionChecker pc = fsd.getPermissionChecker();
+ fsd.checkOwner(pc, iip);
+ }
+
+ INode.BlocksMapUpdateInfo collectedBlocks = new INode.BlocksMapUpdateInfo();
+ ChunkedArrayList removedINodes = new ChunkedArrayList();
+ fsd.writeLock();
+ try {
+ snapshotManager.deleteSnapshot(iip, snapshotName, collectedBlocks,
+ removedINodes);
+ fsd.removeFromInodeMap(removedINodes);
+ } finally {
+ fsd.writeUnlock();
+ }
+ removedINodes.clear();
+ fsd.getEditLog().logDeleteSnapshot(snapshotRoot, snapshotName,
+ logRetryCache);
+
+ return collectedBlocks;
+ }
+
+ private static void checkSubtreeReadPermission(
+ FSDirectory fsd, final FSPermissionChecker pc, String snapshottablePath,
+ String snapshot) throws IOException {
+ final String fromPath = snapshot == null ?
+ snapshottablePath : Snapshot.getSnapshotPath(snapshottablePath,
+ snapshot);
+ INodesInPath iip = fsd.getINodesInPath(fromPath, true);
+ fsd.checkPermission(pc, iip, false, null, null, FsAction.READ,
+ FsAction.READ);
+ }
+
+ /**
+ * Check if the given INode (or one of its descendants) is snapshottable and
+ * already has snapshots.
+ *
+ * @param target The given INode
+ * @param snapshottableDirs The list of directories that are snapshottable
+ * but do not have snapshots yet
+ */
+ static void checkSnapshot(
+ INode target, List snapshottableDirs)
+ throws SnapshotException {
+ if (target.isDirectory()) {
+ INodeDirectory targetDir = target.asDirectory();
+ DirectorySnapshottableFeature sf = targetDir
+ .getDirectorySnapshottableFeature();
+ if (sf != null) {
+ if (sf.getNumSnapshots() > 0) {
+ String fullPath = targetDir.getFullPathName();
+ throw new SnapshotException("The directory " + fullPath
+ + " cannot be deleted since " + fullPath
+ + " is snapshottable and already has snapshots");
+ } else {
+ if (snapshottableDirs != null) {
+ snapshottableDirs.add(targetDir);
+ }
+ }
+ }
+ for (INode child : targetDir.getChildrenList(Snapshot.CURRENT_STATE_ID)) {
+ checkSnapshot(child, snapshottableDirs);
+ }
+ }
+ }
+}
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirStatAndListingOp.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirStatAndListingOp.java
new file mode 100644
index 0000000000000..dc0fe1f5f00bb
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirStatAndListingOp.java
@@ -0,0 +1,501 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hdfs.server.namenode;
+
+import com.google.common.base.Preconditions;
+import org.apache.commons.io.Charsets;
+import org.apache.hadoop.fs.ContentSummary;
+import org.apache.hadoop.fs.DirectoryListingStartAfterNotFoundException;
+import org.apache.hadoop.fs.FileEncryptionInfo;
+import org.apache.hadoop.fs.InvalidPathException;
+import org.apache.hadoop.fs.UnresolvedLinkException;
+import org.apache.hadoop.fs.permission.FsAction;
+import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.hdfs.DFSUtil;
+import org.apache.hadoop.hdfs.protocol.DirectoryListing;
+import org.apache.hadoop.hdfs.protocol.FsPermissionExtension;
+import org.apache.hadoop.hdfs.protocol.HdfsConstants;
+import org.apache.hadoop.hdfs.protocol.HdfsFileStatus;
+import org.apache.hadoop.hdfs.protocol.HdfsLocatedFileStatus;
+import org.apache.hadoop.hdfs.protocol.LocatedBlock;
+import org.apache.hadoop.hdfs.protocol.LocatedBlocks;
+import org.apache.hadoop.hdfs.protocol.SnapshotException;
+import org.apache.hadoop.hdfs.server.blockmanagement.BlockStoragePolicySuite;
+import org.apache.hadoop.hdfs.server.namenode.snapshot.DirectorySnapshottableFeature;
+import org.apache.hadoop.hdfs.server.namenode.snapshot.Snapshot;
+import org.apache.hadoop.hdfs.util.ReadOnlyList;
+
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.util.Arrays;
+
+class FSDirStatAndListingOp {
+ static DirectoryListing getListingInt(FSDirectory fsd, final String srcArg,
+ byte[] startAfter, boolean needLocation) throws IOException {
+ FSPermissionChecker pc = fsd.getPermissionChecker();
+ byte[][] pathComponents = FSDirectory
+ .getPathComponentsForReservedPath(srcArg);
+ final String startAfterString = new String(startAfter, Charsets.UTF_8);
+ final String src = fsd.resolvePath(pc, srcArg, pathComponents);
+ final INodesInPath iip = fsd.getINodesInPath(src, true);
+
+ // Get file name when startAfter is an INodePath
+ if (FSDirectory.isReservedName(startAfterString)) {
+ byte[][] startAfterComponents = FSDirectory
+ .getPathComponentsForReservedPath(startAfterString);
+ try {
+ String tmp = FSDirectory.resolvePath(src, startAfterComponents, fsd);
+ byte[][] regularPath = INode.getPathComponents(tmp);
+ startAfter = regularPath[regularPath.length - 1];
+ } catch (IOException e) {
+ // Possibly the inode is deleted
+ throw new DirectoryListingStartAfterNotFoundException(
+ "Can't find startAfter " + startAfterString);
+ }
+ }
+
+ boolean isSuperUser = true;
+ if (fsd.isPermissionEnabled()) {
+ if (iip.getLastINode() != null && iip.getLastINode().isDirectory()) {
+ fsd.checkPathAccess(pc, iip, FsAction.READ_EXECUTE);
+ } else {
+ fsd.checkTraverse(pc, iip);
+ }
+ isSuperUser = pc.isSuperUser();
+ }
+ return getListing(fsd, iip, src, startAfter, needLocation, isSuperUser);
+ }
+
+ /**
+ * Get the file info for a specific file.
+ *
+ * @param srcArg The string representation of the path to the file
+ * @param resolveLink whether to throw UnresolvedLinkException
+ * if src refers to a symlink
+ *
+ * @return object containing information regarding the file
+ * or null if file not found
+ */
+ static HdfsFileStatus getFileInfo(
+ FSDirectory fsd, String srcArg, boolean resolveLink)
+ throws IOException {
+ String src = srcArg;
+ if (!DFSUtil.isValidName(src)) {
+ throw new InvalidPathException("Invalid file name: " + src);
+ }
+ FSPermissionChecker pc = fsd.getPermissionChecker();
+ byte[][] pathComponents = FSDirectory.getPathComponentsForReservedPath(src);
+ src = fsd.resolvePath(pc, src, pathComponents);
+ final INodesInPath iip = fsd.getINodesInPath(src, resolveLink);
+ boolean isSuperUser = true;
+ if (fsd.isPermissionEnabled()) {
+ fsd.checkPermission(pc, iip, false, null, null, null, null, false);
+ isSuperUser = pc.isSuperUser();
+ }
+ return getFileInfo(fsd, src, resolveLink,
+ FSDirectory.isReservedRawName(srcArg), isSuperUser);
+ }
+
+ /**
+ * Returns true if the file is closed
+ */
+ static boolean isFileClosed(FSDirectory fsd, String src) throws IOException {
+ FSPermissionChecker pc = fsd.getPermissionChecker();
+ byte[][] pathComponents = FSDirectory.getPathComponentsForReservedPath(src);
+ src = fsd.resolvePath(pc, src, pathComponents);
+ final INodesInPath iip = fsd.getINodesInPath(src, true);
+ if (fsd.isPermissionEnabled()) {
+ fsd.checkTraverse(pc, iip);
+ }
+ return !INodeFile.valueOf(iip.getLastINode(), src).isUnderConstruction();
+ }
+
+ static ContentSummary getContentSummary(
+ FSDirectory fsd, String src) throws IOException {
+ byte[][] pathComponents = FSDirectory.getPathComponentsForReservedPath(src);
+ FSPermissionChecker pc = fsd.getPermissionChecker();
+ src = fsd.resolvePath(pc, src, pathComponents);
+ final INodesInPath iip = fsd.getINodesInPath(src, false);
+ if (fsd.isPermissionEnabled()) {
+ fsd.checkPermission(pc, iip, false, null, null, null,
+ FsAction.READ_EXECUTE);
+ }
+ return getContentSummaryInt(fsd, iip);
+ }
+
+ private static byte getStoragePolicyID(byte inodePolicy, byte parentPolicy) {
+ return inodePolicy != BlockStoragePolicySuite.ID_UNSPECIFIED ? inodePolicy :
+ parentPolicy;
+ }
+
+ /**
+ * Get a partial listing of the indicated directory
+ *
+ * We will stop when any of the following conditions is met:
+ * 1) this.lsLimit files have been added
+ * 2) needLocation is true AND enough files have been added such
+ * that at least this.lsLimit block locations are in the response
+ *
+ * @param fsd FSDirectory
+ * @param iip the INodesInPath instance containing all the INodes along the
+ * path
+ * @param src the directory name
+ * @param startAfter the name to start listing after
+ * @param needLocation if block locations are returned
+ * @return a partial listing starting after startAfter
+ */
+ private static DirectoryListing getListing(FSDirectory fsd, INodesInPath iip,
+ String src, byte[] startAfter, boolean needLocation, boolean isSuperUser)
+ throws IOException {
+ String srcs = FSDirectory.normalizePath(src);
+ final boolean isRawPath = FSDirectory.isReservedRawName(src);
+
+ fsd.readLock();
+ try {
+ if (srcs.endsWith(HdfsConstants.SEPARATOR_DOT_SNAPSHOT_DIR)) {
+ return getSnapshotsListing(fsd, srcs, startAfter);
+ }
+ final int snapshot = iip.getPathSnapshotId();
+ final INode targetNode = iip.getLastINode();
+ if (targetNode == null)
+ return null;
+ byte parentStoragePolicy = isSuperUser ?
+ targetNode.getStoragePolicyID() : BlockStoragePolicySuite
+ .ID_UNSPECIFIED;
+
+ if (!targetNode.isDirectory()) {
+ return new DirectoryListing(
+ new HdfsFileStatus[]{createFileStatus(fsd,
+ HdfsFileStatus.EMPTY_NAME, targetNode, needLocation,
+ parentStoragePolicy, snapshot, isRawPath, iip)}, 0);
+ }
+
+ final INodeDirectory dirInode = targetNode.asDirectory();
+ final ReadOnlyList contents = dirInode.getChildrenList(snapshot);
+ int startChild = INodeDirectory.nextChild(contents, startAfter);
+ int totalNumChildren = contents.size();
+ int numOfListing = Math.min(totalNumChildren - startChild,
+ fsd.getLsLimit());
+ int locationBudget = fsd.getLsLimit();
+ int listingCnt = 0;
+ HdfsFileStatus listing[] = new HdfsFileStatus[numOfListing];
+ for (int i=0; i0; i++) {
+ INode cur = contents.get(startChild+i);
+ byte curPolicy = isSuperUser && !cur.isSymlink()?
+ cur.getLocalStoragePolicyID():
+ BlockStoragePolicySuite.ID_UNSPECIFIED;
+ listing[i] = createFileStatus(fsd, cur.getLocalNameBytes(), cur,
+ needLocation, getStoragePolicyID(curPolicy,
+ parentStoragePolicy), snapshot, isRawPath, iip);
+ listingCnt++;
+ if (needLocation) {
+ // Once we hit lsLimit locations, stop.
+ // This helps to prevent excessively large response payloads.
+ // Approximate #locations with locatedBlockCount() * repl_factor
+ LocatedBlocks blks =
+ ((HdfsLocatedFileStatus)listing[i]).getBlockLocations();
+ locationBudget -= (blks == null) ? 0 :
+ blks.locatedBlockCount() * listing[i].getReplication();
+ }
+ }
+ // truncate return array if necessary
+ if (listingCnt < numOfListing) {
+ listing = Arrays.copyOf(listing, listingCnt);
+ }
+ return new DirectoryListing(
+ listing, totalNumChildren-startChild-listingCnt);
+ } finally {
+ fsd.readUnlock();
+ }
+ }
+
+ /**
+ * Get a listing of all the snapshots of a snapshottable directory
+ */
+ private static DirectoryListing getSnapshotsListing(
+ FSDirectory fsd, String src, byte[] startAfter)
+ throws IOException {
+ Preconditions.checkState(fsd.hasReadLock());
+ Preconditions.checkArgument(
+ src.endsWith(HdfsConstants.SEPARATOR_DOT_SNAPSHOT_DIR),
+ "%s does not end with %s", src, HdfsConstants.SEPARATOR_DOT_SNAPSHOT_DIR);
+
+ final String dirPath = FSDirectory.normalizePath(src.substring(0,
+ src.length() - HdfsConstants.DOT_SNAPSHOT_DIR.length()));
+
+ final INode node = fsd.getINode(dirPath);
+ final INodeDirectory dirNode = INodeDirectory.valueOf(node, dirPath);
+ final DirectorySnapshottableFeature sf = dirNode.getDirectorySnapshottableFeature();
+ if (sf == null) {
+ throw new SnapshotException(
+ "Directory is not a snapshottable directory: " + dirPath);
+ }
+ final ReadOnlyList snapshots = sf.getSnapshotList();
+ int skipSize = ReadOnlyList.Util.binarySearch(snapshots, startAfter);
+ skipSize = skipSize < 0 ? -skipSize - 1 : skipSize + 1;
+ int numOfListing = Math.min(snapshots.size() - skipSize, fsd.getLsLimit());
+ final HdfsFileStatus listing[] = new HdfsFileStatus[numOfListing];
+ for (int i = 0; i < numOfListing; i++) {
+ Snapshot.Root sRoot = snapshots.get(i + skipSize).getRoot();
+ listing[i] = createFileStatus(fsd, sRoot.getLocalNameBytes(), sRoot,
+ BlockStoragePolicySuite.ID_UNSPECIFIED, Snapshot.CURRENT_STATE_ID,
+ false, INodesInPath.fromINode(sRoot));
+ }
+ return new DirectoryListing(
+ listing, snapshots.size() - skipSize - numOfListing);
+ }
+
+ /** Get the file info for a specific file.
+ * @param fsd FSDirectory
+ * @param src The string representation of the path to the file
+ * @param isRawPath true if a /.reserved/raw pathname was passed by the user
+ * @param includeStoragePolicy whether to include storage policy
+ * @return object containing information regarding the file
+ * or null if file not found
+ */
+ static HdfsFileStatus getFileInfo(
+ FSDirectory fsd, INodesInPath src, boolean isRawPath,
+ boolean includeStoragePolicy)
+ throws IOException {
+ fsd.readLock();
+ try {
+ final INode i = src.getLastINode();
+ byte policyId = includeStoragePolicy && i != null && !i.isSymlink() ?
+ i.getStoragePolicyID() : BlockStoragePolicySuite.ID_UNSPECIFIED;
+ return i == null ? null : createFileStatus(
+ fsd, HdfsFileStatus.EMPTY_NAME, i, policyId,
+ src.getPathSnapshotId(), isRawPath, src);
+ } finally {
+ fsd.readUnlock();
+ }
+ }
+
+ static HdfsFileStatus getFileInfo(
+ FSDirectory fsd, String src, boolean resolveLink, boolean isRawPath,
+ boolean includeStoragePolicy)
+ throws IOException {
+ String srcs = FSDirectory.normalizePath(src);
+ if (srcs.endsWith(HdfsConstants.SEPARATOR_DOT_SNAPSHOT_DIR)) {
+ if (fsd.getINode4DotSnapshot(srcs) != null) {
+ return new HdfsFileStatus(0, true, 0, 0, 0, 0, null, null, null, null,
+ HdfsFileStatus.EMPTY_NAME, -1L, 0, null,
+ BlockStoragePolicySuite.ID_UNSPECIFIED);
+ }
+ return null;
+ }
+
+ fsd.readLock();
+ try {
+ final INodesInPath iip = fsd.getINodesInPath(srcs, resolveLink);
+ return getFileInfo(fsd, iip, isRawPath, includeStoragePolicy);
+ } finally {
+ fsd.readUnlock();
+ }
+ }
+
+ /**
+ * Currently we only support "ls /xxx/.snapshot" which will return all the
+ * snapshots of a directory. The FSCommand Ls will first call getFileInfo to
+ * make sure the file/directory exists (before the real getListing call).
+ * Since we do not have a real INode for ".snapshot", we return an empty
+ * non-null HdfsFileStatus here.
+ */
+ private static HdfsFileStatus getFileInfo4DotSnapshot(
+ FSDirectory fsd, String src)
+ throws UnresolvedLinkException {
+ if (fsd.getINode4DotSnapshot(src) != null) {
+ return new HdfsFileStatus(0, true, 0, 0, 0, 0, null, null, null, null,
+ HdfsFileStatus.EMPTY_NAME, -1L, 0, null,
+ BlockStoragePolicySuite.ID_UNSPECIFIED);
+ }
+ return null;
+ }
+
+ /**
+ * create an hdfs file status from an inode
+ *
+ * @param fsd FSDirectory
+ * @param path the local name
+ * @param node inode
+ * @param needLocation if block locations need to be included or not
+ * @param isRawPath true if this is being called on behalf of a path in
+ * /.reserved/raw
+ * @return a file status
+ * @throws java.io.IOException if any error occurs
+ */
+ static HdfsFileStatus createFileStatus(
+ FSDirectory fsd, byte[] path, INode node, boolean needLocation,
+ byte storagePolicy, int snapshot, boolean isRawPath, INodesInPath iip)
+ throws IOException {
+ if (needLocation) {
+ return createLocatedFileStatus(fsd, path, node, storagePolicy,
+ snapshot, isRawPath, iip);
+ } else {
+ return createFileStatus(fsd, path, node, storagePolicy, snapshot,
+ isRawPath, iip);
+ }
+ }
+
+ /**
+ * Create FileStatus by file INode
+ */
+ static HdfsFileStatus createFileStatus(
+ FSDirectory fsd, byte[] path, INode node, byte storagePolicy,
+ int snapshot, boolean isRawPath, INodesInPath iip) throws IOException {
+ long size = 0; // length is zero for directories
+ short replication = 0;
+ long blocksize = 0;
+ final boolean isEncrypted;
+
+ final FileEncryptionInfo feInfo = isRawPath ? null :
+ fsd.getFileEncryptionInfo(node, snapshot, iip);
+
+ if (node.isFile()) {
+ final INodeFile fileNode = node.asFile();
+ size = fileNode.computeFileSize(snapshot);
+ replication = fileNode.getFileReplication(snapshot);
+ blocksize = fileNode.getPreferredBlockSize();
+ isEncrypted = (feInfo != null) ||
+ (isRawPath && fsd.isInAnEZ(INodesInPath.fromINode(node)));
+ } else {
+ isEncrypted = fsd.isInAnEZ(INodesInPath.fromINode(node));
+ }
+
+ int childrenNum = node.isDirectory() ?
+ node.asDirectory().getChildrenNum(snapshot) : 0;
+
+ return new HdfsFileStatus(
+ size,
+ node.isDirectory(),
+ replication,
+ blocksize,
+ node.getModificationTime(snapshot),
+ node.getAccessTime(snapshot),
+ getPermissionForFileStatus(node, snapshot, isEncrypted),
+ node.getUserName(snapshot),
+ node.getGroupName(snapshot),
+ node.isSymlink() ? node.asSymlink().getSymlink() : null,
+ path,
+ node.getId(),
+ childrenNum,
+ feInfo,
+ storagePolicy);
+ }
+
+ /**
+ * Create FileStatus with location info by file INode
+ */
+ private static HdfsLocatedFileStatus createLocatedFileStatus(
+ FSDirectory fsd, byte[] path, INode node, byte storagePolicy,
+ int snapshot, boolean isRawPath, INodesInPath iip) throws IOException {
+ assert fsd.hasReadLock();
+ long size = 0; // length is zero for directories
+ short replication = 0;
+ long blocksize = 0;
+ LocatedBlocks loc = null;
+ final boolean isEncrypted;
+ final FileEncryptionInfo feInfo = isRawPath ? null :
+ fsd.getFileEncryptionInfo(node, snapshot, iip);
+ if (node.isFile()) {
+ final INodeFile fileNode = node.asFile();
+ size = fileNode.computeFileSize(snapshot);
+ replication = fileNode.getFileReplication(snapshot);
+ blocksize = fileNode.getPreferredBlockSize();
+
+ final boolean inSnapshot = snapshot != Snapshot.CURRENT_STATE_ID;
+ final boolean isUc = !inSnapshot && fileNode.isUnderConstruction();
+ final long fileSize = !inSnapshot && isUc ?
+ fileNode.computeFileSizeNotIncludingLastUcBlock() : size;
+
+ loc = fsd.getFSNamesystem().getBlockManager().createLocatedBlocks(
+ fileNode.getBlocks(), fileSize, isUc, 0L, size, false,
+ inSnapshot, feInfo);
+ if (loc == null) {
+ loc = new LocatedBlocks();
+ }
+ isEncrypted = (feInfo != null) ||
+ (isRawPath && fsd.isInAnEZ(INodesInPath.fromINode(node)));
+ } else {
+ isEncrypted = fsd.isInAnEZ(INodesInPath.fromINode(node));
+ }
+ int childrenNum = node.isDirectory() ?
+ node.asDirectory().getChildrenNum(snapshot) : 0;
+
+ HdfsLocatedFileStatus status =
+ new HdfsLocatedFileStatus(size, node.isDirectory(), replication,
+ blocksize, node.getModificationTime(snapshot),
+ node.getAccessTime(snapshot),
+ getPermissionForFileStatus(node, snapshot, isEncrypted),
+ node.getUserName(snapshot), node.getGroupName(snapshot),
+ node.isSymlink() ? node.asSymlink().getSymlink() : null, path,
+ node.getId(), loc, childrenNum, feInfo, storagePolicy);
+ // Set caching information for the located blocks.
+ if (loc != null) {
+ CacheManager cacheManager = fsd.getFSNamesystem().getCacheManager();
+ for (LocatedBlock lb: loc.getLocatedBlocks()) {
+ cacheManager.setCachedLocations(lb);
+ }
+ }
+ return status;
+ }
+
+ /**
+ * Returns an inode's FsPermission for use in an outbound FileStatus. If the
+ * inode has an ACL or is for an encrypted file/dir, then this method will
+ * return an FsPermissionExtension.
+ *
+ * @param node INode to check
+ * @param snapshot int snapshot ID
+ * @param isEncrypted boolean true if the file/dir is encrypted
+ * @return FsPermission from inode, with ACL bit on if the inode has an ACL
+ * and encrypted bit on if it represents an encrypted file/dir.
+ */
+ private static FsPermission getPermissionForFileStatus(
+ INode node, int snapshot, boolean isEncrypted) {
+ FsPermission perm = node.getFsPermission(snapshot);
+ boolean hasAcl = node.getAclFeature(snapshot) != null;
+ if (hasAcl || isEncrypted) {
+ perm = new FsPermissionExtension(perm, hasAcl, isEncrypted);
+ }
+ return perm;
+ }
+
+ private static ContentSummary getContentSummaryInt(FSDirectory fsd,
+ INodesInPath iip) throws IOException {
+ fsd.readLock();
+ try {
+ INode targetNode = iip.getLastINode();
+ if (targetNode == null) {
+ throw new FileNotFoundException("File does not exist: " + iip.getPath());
+ }
+ else {
+ // Make it relinquish locks everytime contentCountLimit entries are
+ // processed. 0 means disabled. I.e. blocking for the entire duration.
+ ContentSummaryComputationContext cscc =
+ new ContentSummaryComputationContext(fsd, fsd.getFSNamesystem(),
+ fsd.getContentCountLimit());
+ ContentSummary cs = targetNode.computeAndConvertContentSummary(cscc);
+ fsd.addYieldCount(cscc.getYieldCount());
+ return cs;
+ }
+ } finally {
+ fsd.readUnlock();
+ }
+ }
+}
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirSymlinkOp.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirSymlinkOp.java
new file mode 100644
index 0000000000000..d232b87180df4
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirSymlinkOp.java
@@ -0,0 +1,129 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.namenode;
+
+import org.apache.hadoop.fs.InvalidPathException;
+import org.apache.hadoop.fs.UnresolvedLinkException;
+import org.apache.hadoop.fs.permission.FsAction;
+import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.fs.permission.PermissionStatus;
+import org.apache.hadoop.hdfs.DFSUtil;
+import org.apache.hadoop.hdfs.protocol.HdfsFileStatus;
+import org.apache.hadoop.hdfs.protocol.QuotaExceededException;
+
+import java.io.IOException;
+
+import static org.apache.hadoop.util.Time.now;
+
+class FSDirSymlinkOp {
+
+ static HdfsFileStatus createSymlinkInt(
+ FSNamesystem fsn, String target, final String linkArg,
+ PermissionStatus dirPerms, boolean createParent, boolean logRetryCache)
+ throws IOException {
+ FSDirectory fsd = fsn.getFSDirectory();
+ String link = linkArg;
+ if (!DFSUtil.isValidName(link)) {
+ throw new InvalidPathException("Invalid link name: " + link);
+ }
+ if (FSDirectory.isReservedName(target) || target.isEmpty()) {
+ throw new InvalidPathException("Invalid target name: " + target);
+ }
+
+ if (NameNode.stateChangeLog.isDebugEnabled()) {
+ NameNode.stateChangeLog.debug("DIR* NameSystem.createSymlink: target="
+ + target + " link=" + link);
+ }
+
+ FSPermissionChecker pc = fsn.getPermissionChecker();
+ byte[][] pathComponents = FSDirectory.getPathComponentsForReservedPath(link);
+ INodesInPath iip;
+ fsd.writeLock();
+ try {
+ link = fsd.resolvePath(pc, link, pathComponents);
+ iip = fsd.getINodesInPath4Write(link, false);
+ if (!createParent) {
+ fsd.verifyParentDir(iip, link);
+ }
+ if (!fsd.isValidToCreate(link, iip)) {
+ throw new IOException(
+ "failed to create link " + link +
+ " either because the filename is invalid or the file exists");
+ }
+ if (fsd.isPermissionEnabled()) {
+ fsd.checkAncestorAccess(pc, iip, FsAction.WRITE);
+ }
+ // validate that we have enough inodes.
+ fsn.checkFsObjectLimit();
+
+ // add symbolic link to namespace
+ addSymlink(fsd, link, iip, target, dirPerms, createParent, logRetryCache);
+ } finally {
+ fsd.writeUnlock();
+ }
+ NameNode.getNameNodeMetrics().incrCreateSymlinkOps();
+ return fsd.getAuditFileInfo(iip);
+ }
+
+ static INodeSymlink unprotectedAddSymlink(
+ FSDirectory fsd, INodesInPath iip, long id, String target, long mtime,
+ long atime, PermissionStatus perm)
+ throws UnresolvedLinkException, QuotaExceededException {
+ assert fsd.hasWriteLock();
+ final INodeSymlink symlink = new INodeSymlink(id, null, perm, mtime, atime,
+ target);
+ return fsd.addINode(iip, symlink) ? symlink : null;
+ }
+
+ /**
+ * Add the given symbolic link to the fs. Record it in the edits log.
+ */
+ private static INodeSymlink addSymlink(
+ FSDirectory fsd, String path, INodesInPath iip, String target,
+ PermissionStatus dirPerms, boolean createParent, boolean logRetryCache)
+ throws IOException {
+ final long mtime = now();
+ if (createParent) {
+ INodesInPath parentIIP = iip.getParentINodesInPath();
+ if (parentIIP == null || (parentIIP = FSDirMkdirOp.mkdirsRecursively(
+ fsd,
+ parentIIP, dirPerms, true, mtime)) == null) {
+ return null;
+ } else {
+ iip = INodesInPath.append(parentIIP, null, iip.getLastLocalName());
+ }
+ }
+ final String userName = dirPerms.getUserName();
+ long id = fsd.allocateNewInodeId();
+ PermissionStatus perm = new PermissionStatus(
+ userName, null, FsPermission.getDefault());
+ INodeSymlink newNode =
+ unprotectedAddSymlink(fsd, iip, id, target, mtime, mtime, perm);
+ if (newNode == null) {
+ NameNode.stateChangeLog.info("addSymlink: failed to add " + path);
+ return null;
+ }
+ fsd.getEditLog().logSymlink(path, target, mtime, mtime, newNode,
+ logRetryCache);
+
+ if(NameNode.stateChangeLog.isDebugEnabled()) {
+ NameNode.stateChangeLog.debug("addSymlink: " + path + " is added");
+ }
+ return newNode;
+ }
+}
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirXAttrOp.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirXAttrOp.java
new file mode 100644
index 0000000000000..45e63f2effe00
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirXAttrOp.java
@@ -0,0 +1,463 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.namenode;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Charsets;
+import com.google.common.base.Preconditions;
+import com.google.common.collect.Lists;
+import org.apache.hadoop.HadoopIllegalArgumentException;
+import org.apache.hadoop.fs.XAttr;
+import org.apache.hadoop.fs.XAttrSetFlag;
+import org.apache.hadoop.fs.permission.FsAction;
+import org.apache.hadoop.hdfs.DFSConfigKeys;
+import org.apache.hadoop.hdfs.XAttrHelper;
+import org.apache.hadoop.hdfs.protocol.HdfsFileStatus;
+import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos;
+import org.apache.hadoop.hdfs.protocolPB.PBHelper;
+import org.apache.hadoop.security.AccessControlException;
+
+import java.io.IOException;
+import java.util.EnumSet;
+import java.util.List;
+import java.util.ListIterator;
+
+import static org.apache.hadoop.hdfs.server.common.HdfsServerConstants.CRYPTO_XATTR_ENCRYPTION_ZONE;
+import static org.apache.hadoop.hdfs.server.common.HdfsServerConstants.SECURITY_XATTR_UNREADABLE_BY_SUPERUSER;
+
+class FSDirXAttrOp {
+ private static final XAttr KEYID_XATTR =
+ XAttrHelper.buildXAttr(CRYPTO_XATTR_ENCRYPTION_ZONE, null);
+ private static final XAttr UNREADABLE_BY_SUPERUSER_XATTR =
+ XAttrHelper.buildXAttr(SECURITY_XATTR_UNREADABLE_BY_SUPERUSER, null);
+
+ /**
+ * Set xattr for a file or directory.
+ *
+ * @param src
+ * - path on which it sets the xattr
+ * @param xAttr
+ * - xAttr details to set
+ * @param flag
+ * - xAttrs flags
+ * @throws IOException
+ */
+ static HdfsFileStatus setXAttr(
+ FSDirectory fsd, String src, XAttr xAttr, EnumSet flag,
+ boolean logRetryCache)
+ throws IOException {
+ checkXAttrsConfigFlag(fsd);
+ checkXAttrSize(fsd, xAttr);
+ FSPermissionChecker pc = fsd.getPermissionChecker();
+ XAttrPermissionFilter.checkPermissionForApi(
+ pc, xAttr, FSDirectory.isReservedRawName(src));
+ byte[][] pathComponents = FSDirectory.getPathComponentsForReservedPath(src);
+ src = fsd.resolvePath(pc, src, pathComponents);
+ List xAttrs = Lists.newArrayListWithCapacity(1);
+ xAttrs.add(xAttr);
+ INodesInPath iip;
+ fsd.writeLock();
+ try {
+ iip = fsd.getINodesInPath4Write(src);
+ checkXAttrChangeAccess(fsd, iip, xAttr, pc);
+ unprotectedSetXAttrs(fsd, src, xAttrs, flag);
+ } finally {
+ fsd.writeUnlock();
+ }
+ fsd.getEditLog().logSetXAttrs(src, xAttrs, logRetryCache);
+ return fsd.getAuditFileInfo(iip);
+ }
+
+ static List getXAttrs(FSDirectory fsd, final String srcArg,
+ List xAttrs)
+ throws IOException {
+ String src = srcArg;
+ checkXAttrsConfigFlag(fsd);
+ FSPermissionChecker pc = fsd.getPermissionChecker();
+ final boolean isRawPath = FSDirectory.isReservedRawName(src);
+ boolean getAll = xAttrs == null || xAttrs.isEmpty();
+ if (!getAll) {
+ XAttrPermissionFilter.checkPermissionForApi(pc, xAttrs, isRawPath);
+ }
+ byte[][] pathComponents = FSDirectory.getPathComponentsForReservedPath(src);
+ src = fsd.resolvePath(pc, src, pathComponents);
+ final INodesInPath iip = fsd.getINodesInPath(src, true);
+ if (fsd.isPermissionEnabled()) {
+ fsd.checkPathAccess(pc, iip, FsAction.READ);
+ }
+ List all = FSDirXAttrOp.getXAttrs(fsd, src);
+ List filteredAll = XAttrPermissionFilter.
+ filterXAttrsForApi(pc, all, isRawPath);
+
+ if (getAll) {
+ return filteredAll;
+ }
+ if (filteredAll == null || filteredAll.isEmpty()) {
+ return null;
+ }
+ List toGet = Lists.newArrayListWithCapacity(xAttrs.size());
+ for (XAttr xAttr : xAttrs) {
+ boolean foundIt = false;
+ for (XAttr a : filteredAll) {
+ if (xAttr.getNameSpace() == a.getNameSpace() && xAttr.getName().equals(
+ a.getName())) {
+ toGet.add(a);
+ foundIt = true;
+ break;
+ }
+ }
+ if (!foundIt) {
+ throw new IOException(
+ "At least one of the attributes provided was not found.");
+ }
+ }
+ return toGet;
+ }
+
+ static List listXAttrs(
+ FSDirectory fsd, String src) throws IOException {
+ FSDirXAttrOp.checkXAttrsConfigFlag(fsd);
+ final FSPermissionChecker pc = fsd.getPermissionChecker();
+ final boolean isRawPath = FSDirectory.isReservedRawName(src);
+ byte[][] pathComponents = FSDirectory.getPathComponentsForReservedPath(src);
+ src = fsd.resolvePath(pc, src, pathComponents);
+ final INodesInPath iip = fsd.getINodesInPath(src, true);
+ if (fsd.isPermissionEnabled()) {
+ /* To access xattr names, you need EXECUTE in the owning directory. */
+ fsd.checkParentAccess(pc, iip, FsAction.EXECUTE);
+ }
+ final List all = FSDirXAttrOp.getXAttrs(fsd, src);
+ return XAttrPermissionFilter.
+ filterXAttrsForApi(pc, all, isRawPath);
+ }
+
+ /**
+ * Remove an xattr for a file or directory.
+ *
+ * @param src
+ * - path to remove the xattr from
+ * @param xAttr
+ * - xAttr to remove
+ * @throws IOException
+ */
+ static HdfsFileStatus removeXAttr(
+ FSDirectory fsd, String src, XAttr xAttr, boolean logRetryCache)
+ throws IOException {
+ FSDirXAttrOp.checkXAttrsConfigFlag(fsd);
+ FSPermissionChecker pc = fsd.getPermissionChecker();
+ XAttrPermissionFilter.checkPermissionForApi(
+ pc, xAttr, FSDirectory.isReservedRawName(src));
+ byte[][] pathComponents = FSDirectory.getPathComponentsForReservedPath(
+ src);
+
+ List xAttrs = Lists.newArrayListWithCapacity(1);
+ xAttrs.add(xAttr);
+ INodesInPath iip;
+ fsd.writeLock();
+ try {
+ src = fsd.resolvePath(pc, src, pathComponents);
+ iip = fsd.getINodesInPath4Write(src);
+ checkXAttrChangeAccess(fsd, iip, xAttr, pc);
+
+ List removedXAttrs = unprotectedRemoveXAttrs(fsd, src, xAttrs);
+ if (removedXAttrs != null && !removedXAttrs.isEmpty()) {
+ fsd.getEditLog().logRemoveXAttrs(src, removedXAttrs, logRetryCache);
+ } else {
+ throw new IOException(
+ "No matching attributes found for remove operation");
+ }
+ } finally {
+ fsd.writeUnlock();
+ }
+ return fsd.getAuditFileInfo(iip);
+ }
+
+ static List