name property as
* a host:port.
+ * @param name property name.
+ * @param addr inetSocketAddress addr.
*/
public void setSocketAddr(String name, InetSocketAddress addr) {
set(name, NetUtils.getHostPortString(addr));
@@ -2724,6 +2728,7 @@ public Class getClass(String name, Class defaultValue) {
* @param name the conf key name.
* @param defaultValue default value.
* @param xface the interface implemented by the named class.
+ * @param Interface class type.
* @return property value as a Class,
* or defaultValue.
*/
@@ -2753,6 +2758,7 @@ else if (theClass != null)
* @param name the property name.
* @param xface the interface implemented by the classes named by
* name.
+ * @param Interface class type.
* @return a List of objects implementing xface.
*/
@SuppressWarnings("unchecked")
@@ -2785,15 +2791,16 @@ public void setClass(String name, Class theClass, Class xface) {
set(name, theClass.getName());
}
- /**
+ /**
* Get a local file under a directory named by dirsProp with
* the given path. If dirsProp contains multiple directories,
* then one is chosen based on path's hash code. If the selected
* directory does not exist, an attempt is made to create it.
- *
+ *
* @param dirsProp directory in which to locate the file.
* @param path file-path.
* @return local file under the directory with the given path.
+ * @throws IOException raised on errors performing I/O.
*/
public Path getLocalPath(String dirsProp, String path)
throws IOException {
@@ -2817,15 +2824,16 @@ public Path getLocalPath(String dirsProp, String path)
throw new IOException("No valid local directories in property: "+dirsProp);
}
- /**
+ /**
* Get a local file name under a directory named in dirsProp with
* the given path. If dirsProp contains multiple directories,
* then one is chosen based on path's hash code. If the selected
* directory does not exist, an attempt is made to create it.
- *
+ *
* @param dirsProp directory in which to locate the file.
* @param path file-path.
* @return local file under the directory with the given path.
+ * @throws IOException raised on errors performing I/O.
*/
public File getFile(String dirsProp, String path)
throws IOException {
@@ -3437,7 +3445,7 @@ void parseNext() throws IOException, XMLStreamException {
/**
* Add tags defined in HADOOP_TAGS_SYSTEM, HADOOP_TAGS_CUSTOM.
- * @param prop
+ * @param prop properties.
*/
public void addTags(Properties prop) {
// Get all system tags
@@ -3538,7 +3546,7 @@ private void loadProperty(Properties properties, String name, String attr,
/**
* Print a warning if a property with a given name already exists with a
- * different value
+ * different value.
*/
private void checkForOverride(Properties properties, String name, String attr, String value) {
String propertyValue = properties.getProperty(attr);
@@ -3548,11 +3556,12 @@ private void checkForOverride(Properties properties, String name, String attr, S
}
}
- /**
+ /**
* Write out the non-default properties in this configuration to the given
* {@link OutputStream} using UTF-8 encoding.
- *
+ *
* @param out the output stream to write to.
+ * @throws IOException raised on errors performing I/O.
*/
public void writeXml(OutputStream out) throws IOException {
writeXml(new OutputStreamWriter(out, "UTF-8"));
@@ -3582,7 +3591,9 @@ public void writeXml(Writer out) throws IOException {
* the configuration, this method throws an {@link IllegalArgumentException}.
*
*
+ * @param propertyName xml property name.
* @param out the writer to write to.
+ * @throws IOException raised on errors performing I/O.
*/
public void writeXml(@Nullable String propertyName, Writer out)
throws IOException, IllegalArgumentException {
@@ -3736,7 +3747,7 @@ private synchronized void appendXMLProperty(Document doc, Element conf,
* @param config the configuration
* @param propertyName property name
* @param out the Writer to write to
- * @throws IOException
+ * @throws IOException raised on errors performing I/O.
* @throws IllegalArgumentException when property name is not
* empty and the property is not found in configuration
**/
@@ -3783,7 +3794,7 @@ public static void dumpConfiguration(Configuration config,
*
* @param config the configuration
* @param out the Writer to write to
- * @throws IOException
+ * @throws IOException raised on errors performing I/O.
*/
public static void dumpConfiguration(Configuration config,
Writer out) throws IOException {
@@ -3812,7 +3823,7 @@ public static void dumpConfiguration(Configuration config,
* @param jsonGen json writer
* @param config configuration
* @param name property name
- * @throws IOException
+ * @throws IOException raised on errors performing I/O.
*/
private static void appendJSONProperty(JsonGenerator jsonGen,
Configuration config, String name, ConfigRedactor redactor)
@@ -3894,7 +3905,10 @@ synchronized boolean getQuietMode() {
return this.quietmode;
}
- /** For debugging. List non-default properties to the terminal and exit. */
+ /** For debugging. List non-default properties to the terminal and exit.
+ * @param args the argument to be parsed.
+ * @throws Exception exception.
+ */
public static void main(String[] args) throws Exception {
new Configuration().writeXml(System.out);
}
@@ -3928,8 +3942,8 @@ public void write(DataOutput out) throws IOException {
}
/**
- * get keys matching the the regex
- * @param regex
+ * get keys matching the the regex.
+ * @param regex the regex to match against.
* @return {@literal Map- * * The result is stored in the output buffer. Upon return, the output buffer's * position will have advanced by n, where n is the value returned by this * method; the output buffer's limit will not have changed. - *
- * + *
* Ifoutput.remaining() bytes are insufficient to hold the result,
* a ShortBufferException is thrown.
* - * * Upon finishing, this method resets this cipher object to the state it was * in when previously initialized. That is, the object is available to encrypt * or decrypt more data. - *
- * - * If any exception is thrown, this cipher object need to be reset before it + *
+ * If any exception is thrown, this cipher object need to be reset before it * can be used again. - * + * * @param output the output ByteBuffer * @return int number of bytes stored inoutput
- * @throws ShortBufferException
- * @throws IllegalBlockSizeException
- * @throws BadPaddingException
+ * @throws ShortBufferException if there is insufficient space in the output buffer.
+ * @throws IllegalBlockSizeException This exception is thrown when the length
+ * of data provided to a block cipher is incorrect.
+ * @throws BadPaddingException This exception is thrown when a particular
+ * padding mechanism is expected for the input
+ * data but the data is not padded properly.
*/
public int doFinal(ByteBuffer output) throws ShortBufferException,
IllegalBlockSizeException, BadPaddingException {
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProvider.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProvider.java
index dafdaf7e15b25c..4d1674bd7b883d 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProvider.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProvider.java
@@ -242,7 +242,7 @@ protected int addVersion() {
/**
* Serialize the metadata to a set of bytes.
* @return the serialized bytes
- * @throws IOException
+ * @throws IOException raised on errors performing I/O.
*/
protected byte[] serialize() throws IOException {
ByteArrayOutputStream buffer = new ByteArrayOutputStream();
@@ -281,7 +281,7 @@ protected byte[] serialize() throws IOException {
/**
* Deserialize a new metadata object from a set of bytes.
* @param bytes the serialized metadata
- * @throws IOException
+ * @throws IOException raised on errors performing I/O.
*/
protected Metadata(byte[] bytes) throws IOException {
String cipher = null;
@@ -450,7 +450,7 @@ public boolean isTransient() {
* when decrypting data.
* @param versionName the name of a specific version of the key
* @return the key material
- * @throws IOException
+ * @throws IOException raised on errors performing I/O.
*/
public abstract KeyVersion getKeyVersion(String versionName
) throws IOException;
@@ -458,14 +458,15 @@ public abstract KeyVersion getKeyVersion(String versionName
/**
* Get the key names for all keys.
* @return the list of key names
- * @throws IOException
+ * @throws IOException raised on errors performing I/O.
*/
public abstract Listuri has syntax error
*/
public AbstractFileSystem(final URI uri, final String supportedScheme,
@@ -281,11 +281,12 @@ public AbstractFileSystem(final URI uri, final String supportedScheme,
myUri = getUri(uri, supportedScheme, authorityNeeded, defaultPort);
statistics = getStatistics(uri);
}
-
+
/**
- * Check that the Uri's scheme matches
- * @param uri
- * @param supportedScheme
+ * Check that the Uri's scheme matches.
+ *
+ * @param uri name URI of the FS.
+ * @param supportedScheme supported scheme.
*/
public void checkScheme(URI uri, String supportedScheme) {
String scheme = uri.getScheme();
@@ -362,7 +363,7 @@ public URI getUri() {
* If the path is fully qualified URI, then its scheme and authority
* matches that of this file system. Otherwise the path must be
* slash-relative name.
- *
+ * @param path the path.
* @throws InvalidPathException if the path is invalid
*/
public void checkPath(Path path) {
@@ -431,7 +432,7 @@ public String getUriPath(final Path p) {
/**
* Make the path fully qualified to this file system
- * @param path
+ * @param path the path.
* @return the qualified path
*/
public Path makeQualified(Path path) {
@@ -496,9 +497,9 @@ public FsServerDefaults getServerDefaults(final Path f) throws IOException {
* through any internal symlinks or mount point
* @param p path to be resolved
* @return fully qualified path
- * @throws FileNotFoundException
- * @throws AccessControlException
- * @throws IOException
+ * @throws FileNotFoundException when file not find throw.
+ * @throws AccessControlException when accees control error throw.
+ * @throws IOException raised on errors performing I/O.
* @throws UnresolvedLinkException if symbolic link on path cannot be
* resolved internally
*/
@@ -513,6 +514,18 @@ public Path resolvePath(final Path p) throws FileNotFoundException,
* {@link FileContext#create(Path, EnumSet, Options.CreateOpts...)} except
* that the Path f must be fully qualified and the permission is absolute
* (i.e. umask has been applied).
+ *
+ * @param f the path.
+ * @param createFlag create_flag.
+ * @param opts create ops.
+ * @throws AccessControlException access controll exception.
+ * @throws FileAlreadyExistsException file already exception.
+ * @throws FileNotFoundException file not found exception.
+ * @throws ParentNotDirectoryException parent not dir exception.
+ * @throws UnsupportedFileSystemException unsupported file system exception.
+ * @throws UnresolvedLinkException unresolved link exception.
+ * @throws IOException raised on errors performing I/O.
+ * @return output stream.
*/
public final FSDataOutputStream create(final Path f,
final EnumSet* A higher number here does not necessarily improve performance, especially * for object stores, where multiple threads may be attempting to create an FS * instance for the same URI. - *
+ * * Default value: {@value}. */ public static final String FS_CREATION_PARALLEL_COUNT = @@ -181,8 +181,9 @@ public class CommonConfigurationKeysPublic { /** * Default value for {@link #FS_CREATION_PARALLEL_COUNT}. - * + ** Default value: {@value}. + *
*/ public static final int FS_CREATION_PARALLEL_COUNT_DEFAULT = 64; diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CompositeCrcFileChecksum.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CompositeCrcFileChecksum.java index e1ed5cbcfcaa60..bdbc8f3a33f4b1 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CompositeCrcFileChecksum.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CompositeCrcFileChecksum.java @@ -37,7 +37,13 @@ public class CompositeCrcFileChecksum extends FileChecksum { private DataChecksum.Type crcType; private int bytesPerCrc; - /** Create a CompositeCrcFileChecksum. */ + /** + * Create a CompositeCrcFileChecksum. + * + * @param crc crc. + * @param crcType crcType. + * @param bytesPerCrc bytesPerCrc. + */ public CompositeCrcFileChecksum( int crc, DataChecksum.Type crcType, int bytesPerCrc) { this.crc = crc; diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ContentSummary.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ContentSummary.java index 79850e1a2f2914..9f97a12fa60885 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ContentSummary.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ContentSummary.java @@ -149,17 +149,31 @@ public ContentSummary build() { @Deprecated public ContentSummary() {} - /** Constructor, deprecated by ContentSummary.Builder + /** + * Constructor, deprecated by ContentSummary.Builder * This constructor implicitly set spaceConsumed the same as length. * spaceConsumed and length must be set explicitly with - * ContentSummary.Builder + * ContentSummary.Builder. + * + * @param length length. + * @param fileCount file count. + * @param directoryCount directory count. * */ @Deprecated public ContentSummary(long length, long fileCount, long directoryCount) { this(length, fileCount, directoryCount, -1L, length, -1L); } - /** Constructor, deprecated by ContentSummary.Builder */ + /** + * Constructor, deprecated by ContentSummary.Builder. + * + * @param length length. + * @param fileCount file count. + * @param directoryCount directory count. + * @param quota quota. + * @param spaceConsumed space consumed. + * @param spaceQuota space quota. + * */ @Deprecated public ContentSummary( long length, long fileCount, long directoryCount, long quota, @@ -172,7 +186,11 @@ public ContentSummary( setSpaceQuota(spaceQuota); } - /** Constructor for ContentSummary.Builder*/ + /** + * Constructor for ContentSummary.Builder. + * + * @param builder builder. + */ private ContentSummary(Builder builder) { super(builder); this.length = builder.length; diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CreateFlag.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CreateFlag.java index 71993713ad2ebd..ca008e536931de 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CreateFlag.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CreateFlag.java @@ -189,6 +189,8 @@ public static void validate(Object path, boolean pathExists, /** * Validate the CreateFlag for the append operation. The flag must contain * APPEND, and cannot contain OVERWRITE. + * + * @param flag enum set flag. */ public static void validateForAppend(EnumSetf does not exist
* @throws AccessControlException if access denied
* @throws IOException If an IO Error occurred
- *
+ * @throws UnresolvedLinkException If unresolved link occurred.
+ *
* Exceptions applicable to file systems accessed over RPC:
* @throws RpcClientException If an exception occurred in the RPC client
* @throws RpcServerException If an exception occurred in the RPC server
* @throws UnexpectedServerException If server implementation throws
* undeclared exception to RPC server
- *
+ *
* RuntimeExceptions:
* @throws InvalidPathException If path f is not valid
*/
@@ -615,7 +623,7 @@ public Path resolvePath(final Path f) throws FileNotFoundException,
* A Fully-qualified path has scheme and authority specified and an absolute
* path.
* Use the default file system and working dir in this FileContext to qualify.
- * @param path
+ * @param path the path.
* @return qualified path
*/
public Path makeQualified(final Path path) {
@@ -754,6 +762,7 @@ public FSDataOutputStream build() throws IOException {
*
* Client should expect {@link FSDataOutputStreamBuilder#build()} throw the
* same exceptions as create(Path, EnumSet, CreateOpts...).
+ * @throws IOException If an I/O error occurred.
*/
public FSDataOutputStreamBuilderf is invalid
+ *
+ * @return if delete success true, not false.
*/
public boolean delete(final Path f, final boolean recursive)
throws AccessControlException, FileNotFoundException,
@@ -857,6 +868,7 @@ public Boolean next(final AbstractFileSystem fs, final Path p)
* @throws RpcServerException If an exception occurred in the RPC server
* @throws UnexpectedServerException If server implementation throws
* undeclared exception to RPC server
+ * @return input stream.
*/
public FSDataInputStream open(final Path f) throws AccessControlException,
FileNotFoundException, UnsupportedFileSystemException, IOException {
@@ -887,6 +899,7 @@ public FSDataInputStream next(final AbstractFileSystem fs, final Path p)
* @throws RpcServerException If an exception occurred in the RPC server
* @throws UnexpectedServerException If server implementation throws
* undeclared exception to RPC server
+ * @return output stream.
*/
public FSDataInputStream open(final Path f, final int bufferSize)
throws AccessControlException, FileNotFoundException,
@@ -996,6 +1009,7 @@ public Boolean next(final AbstractFileSystem fs, final Path p)
*
* @param src path to be renamed
* @param dst new path after rename
+ * @param options rename options.
*
* @throws AccessControlException If access is denied
* @throws FileAlreadyExistsException If dst already exists and
@@ -1047,7 +1061,7 @@ public Void next(final AbstractFileSystem fs, final Path p)
/**
* Set permission of a path.
- * @param f
+ * @param f the path.
* @param permission - the new absolute permission (umask is not applied)
*
* @throws AccessControlException If access is denied
@@ -1191,7 +1205,7 @@ public FileChecksum next(final AbstractFileSystem fs, final Path p)
* Set the verify checksum flag for the file system denoted by the path.
* This is only applicable if the
* corresponding FileSystem supports checksum. By default doesn't do anything.
- * @param verifyChecksum
+ * @param verifyChecksum verify check sum.
* @param f set the verifyChecksum for the Filesystem containing this path
*
* @throws AccessControlException If access is denied
@@ -1246,8 +1260,9 @@ public FileStatus next(final AbstractFileSystem fs, final Path p)
/**
* Synchronize client metadata state.
*
- * @throws IOException
- * @throws UnsupportedOperationException
+ * @throws IOException If an I/O error occurred.
+ * @throws UnsupportedOperationException If file system for f is
+ * not supported.
*/
public void msync() throws IOException, UnsupportedOperationException {
defaultFS.msync();
@@ -1608,9 +1623,12 @@ public RemoteIteratorfiles does not exist.
+ * @throws IOException If an I/O error occurred.
+ * @return file status array.
*/
public FileStatus[] listStatus(Path[] files) throws AccessControlException,
FileNotFoundException, IOException {
@@ -2049,36 +2074,29 @@ public LocatedFileStatus next() throws IOException {
* *
*
*
*
*
*
+ * *
src already exists.
+ * @throws FileNotFoundException if next file does not exist any more.
+ * @throws ParentNotDirectoryException If parent of src is not a
+ * directory.
+ * @throws UnsupportedFileSystemException If file system for
+ * src/dst is not supported.
+ * @throws IOException If an I/O error occurred.
+ * @return if success copy true, not false.
*/
public boolean copy(final Path src, final Path dst)
throws AccessControlException, FileAlreadyExistsException,
@@ -2149,8 +2179,8 @@ public boolean copy(final Path src, final Path dst)
/**
* Copy from src to dst, optionally deleting src and overwriting dst.
- * @param src
- * @param dst
+ * @param src src.
+ * @param dst dst.
* @param deleteSource - delete src if true
* @param overwrite overwrite dst if true; throw IOException if dst exists
* and overwrite is false.
@@ -2198,7 +2228,12 @@ public boolean copy(final Path src, final Path dst, boolean deleteSource,
EnumSetf does not exist.
+ * @throws UnresolvedLinkException If unresolved link occurred.
+ * @throws AccessControlException If access is denied.
+ * @throws IOException If an I/O error occurred.
+ * @return resolve path.
*/
protected Path resolve(final Path f) throws FileNotFoundException,
UnresolvedLinkException, AccessControlException, IOException {
@@ -2306,6 +2348,7 @@ public Path next(final AbstractFileSystem fs, final Path p)
* to, but not including the final path component.
* @param f path to resolve
* @return the new path object.
+ * @throws IOException If an I/O error occurred.
*/
protected Path resolveIntermediate(final Path f) throws IOException {
return new FSLinkResolver* The local implementation is {@link LocalFileSystem} and distributed * implementation is DistributedFileSystem. There are other implementations * for object stores and (outside the Apache Hadoop codebase), * third party filesystems. - *
+ *
* Notes ** Important note for developers - *
+ * * If you are making changes here to the public API or protected methods, * you must review the following subclasses and make sure that * they are filtering/passing through new methods as appropriate. - * * * {@link FilterFileSystem}: methods are passed through. If not, * then {@code TestFilterFileSystem.MustNotImplement} must be @@ -147,21 +147,22 @@ * {@link #hasPathCapability(Path, String)} then * {@link FilterFileSystem#hasPathCapability(Path, String)} * must return false, always. - * + ** {@link ChecksumFileSystem}: checksums are created and * verified. - *
+ * * {@code TestHarFileSystem} will need its {@code MustNotImplement} * interface updated. - * * + ** There are some external places your changes will break things. * Do co-ordinate changes here. - *
+ * * * HBase: HBoss - * + ** Hive: HiveShim23 + *
* {@code shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java} * *****************************************************************/ @@ -280,6 +281,8 @@ public FileSystem run() throws IOException { /** * Returns the configured FileSystem implementation. * @param conf the configuration to use + * @return FileSystem. + * @throws IOException If an I/O error occurred. */ public static FileSystem get(Configuration conf) throws IOException { return get(getDefaultUri(conf), conf); @@ -374,6 +377,7 @@ public String getScheme() { * implement that method. * * @see #canonicalizeUri(URI) + * @return the URI of this filesystem. */ protected URI getCanonicalUri() { return canonicalizeUri(getUri()); @@ -390,6 +394,7 @@ protected URI getCanonicalUri() { * not specified and if {@link #getDefaultPort()} returns a * default port. * + * @param uri url. * @return URI * @see NetUtils#getCanonicalUri(URI, int) */ @@ -453,11 +458,21 @@ public String getCanonicalServiceName() { : null; } - /** @deprecated call {@link #getUri()} instead.*/ + /** + * @return uri to string. + * @deprecated call {@link #getUri()} instead. + */ @Deprecated public String getName() { return getUri().toString(); } - /** @deprecated call {@link #get(URI, Configuration)} instead. */ + /** + * @deprecated call {@link #get(URI, Configuration)} instead. + * + * @param name name. + * @param conf configuration. + * @return file system. + * @throws IOException If an I/O error occurred. + */ @Deprecated public static FileSystem getNamed(String name, Configuration conf) throws IOException { @@ -512,6 +527,9 @@ public static LocalFileSystem getLocal(Configuration conf) * configuration and URI, cached and returned to the caller. ** If OVERWRITE option is not passed as an argument, rename fails * if the dst already exists. + *
** If OVERWRITE option is passed as an argument, rename overwrites * the dst if it is a file or an empty directory. Rename fails if dst is * a non-empty directory. - *
+ *
* Note that atomicity of rename is dependent on the file system * implementation. Please refer to the file system documentation for * details. This default implementation is non atomic. @@ -1555,9 +1620,11 @@ public boolean setReplication(Path src, short replication) * This method is deprecated since it is a temporary method added to * support the transition from FileSystem to FileContext for user * applications. + * * * @param src path to be renamed * @param dst new path after rename + * @param options rename options. * @throws FileNotFoundException src path does not exist, or the parent * path of dst does not exist. * @throws FileAlreadyExistsException dest path exists and is a file @@ -1652,6 +1719,9 @@ public boolean truncate(Path f, long newLength) throws IOException { /** * Delete a file/directory. + * @param f the path. + * @throws IOException IO failure. + * @return if delete success true, not false. * @deprecated Use {@link #delete(Path, boolean)} instead. */ @Deprecated @@ -1768,6 +1838,7 @@ public boolean exists(Path f) throws IOException { * @param f path to check * @throws IOException IO failure * @deprecated Use {@link #getFileStatus(Path)} instead + * @return if f is directory true, not false. */ @Deprecated public boolean isDirectory(Path f) throws IOException { @@ -1785,6 +1856,7 @@ public boolean isDirectory(Path f) throws IOException { * @param f path to check * @throws IOException IO failure * @deprecated Use {@link #getFileStatus(Path)} instead + * @return if f is file true, not false. */ @Deprecated public boolean isFile(Path f) throws IOException { @@ -1797,6 +1869,7 @@ public boolean isFile(Path f) throws IOException { /** * The number of bytes in a file. + * @param f the path. * @return the number of bytes; 0 for a directory * @deprecated Use {@link #getFileStatus(Path)} instead. * @throws FileNotFoundException if the path does not resolve @@ -1811,6 +1884,7 @@ public long getLength(Path f) throws IOException { * @param f path to use * @throws FileNotFoundException if the path does not resolve * @throws IOException IO failure + * @return content summary. */ public ContentSummary getContentSummary(Path f) throws IOException { FileStatus status = getFileStatus(f); @@ -1945,9 +2019,9 @@ public boolean hasMore() { * @param f Path to list * @param token opaque iteration token returned by previous call, or null * if this is the first call. - * @return - * @throws FileNotFoundException - * @throws IOException + * @return directory entries. + * @throws FileNotFoundException when the path does not exist. + * @throws IOException If an I/O error occurred. */ @InterfaceAudience.Private protected DirectoryEntries listStatusBatch(Path f, byte[] token) throws @@ -1978,6 +2052,8 @@ private void listStatus(ArrayList*
*
*
*
*
*
*
+ * returns true if the operation succeeded. When deleteSource is true, + * this means "after the copy, delete(source) returned true" + * If the destination is a directory, and mkdirs (dest) fails, + * the operation will return false rather than raise any exception. + *
+ * The overwrite flag is about overwriting files; it has no effect about + * handing an attempt to copy a file atop a directory (expect an IOException), + * or a directory over a path which contains a file (mkdir will fail, so + * "false"). + *+ * The operation is recursive, and the deleteSource operation takes place + * as each subdirectory is copied. Therefore, if an operation fails partway + * through, the source tree may be partially deleted. + *
+ * @param srcFS source filesystem + * @param srcStatus status of source + * @param dstFS destination filesystem + * @param dst path of source + * @param deleteSource delete the source? + * @param overwrite overwrite files at destination? + * @param conf configuration to use when opening files + * @return true if the operation succeeded. + * @throws IOException failure + */ public static boolean copy(FileSystem srcFS, FileStatus srcStatus, FileSystem dstFS, Path dst, boolean deleteSource, @@ -409,22 +470,27 @@ public static boolean copy(FileSystem srcFS, FileStatus srcStatus, if (!dstFS.mkdirs(dst)) { return false; } - FileStatus contents[] = srcFS.listStatus(src); - for (int i = 0; i < contents.length; i++) { - copy(srcFS, contents[i], dstFS, - new Path(dst, contents[i].getPath().getName()), - deleteSource, overwrite, conf); + RemoteIterator* The interface extends {@link IOStatisticsSource} so that there is no * need to cast an instance to see if is a source of statistics. * However, implementations MAY return null for their actual statistics. + *
*/ @InterfaceAudience.Public @InterfaceStability.Unstable diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/MultipartUploaderBuilder.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/MultipartUploaderBuilder.java index 381bfaa07f6d1a..e7b0865063ee5b 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/MultipartUploaderBuilder.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/MultipartUploaderBuilder.java @@ -25,34 +25,43 @@ /** * Builder interface for Multipart readers. - * @param
* .opt("foofs:option.a", true)
* .opt("foofs:option.b", "value")
- * .opt("barfs:cache", true)
+ * .opt("fs.s3a.open.option.etag", "9fe4c37c25b")
* .must("foofs:cache", true)
* .must("barfs:cache-size", 256 * 1024 * 1024)
* .build();
@@ -88,6 +88,9 @@
/** Keep track of the keys for mandatory options. */
private final Set mandatoryKeys = new HashSet<>();
+ /** Keep track of the optional keys. */
+ private final Set optionalKeys = new HashSet<>();
+
/**
* Constructor with both optional path and path handle.
* Either or both argument may be empty, but it is an error for
@@ -163,6 +166,7 @@ public PathHandle getPathHandle() {
@Override
public B opt(@Nonnull final String key, @Nonnull final String value) {
mandatoryKeys.remove(key);
+ optionalKeys.add(key);
options.set(key, value);
return getThisBuilder();
}
@@ -175,6 +179,7 @@ public B opt(@Nonnull final String key, @Nonnull final String value) {
@Override
public B opt(@Nonnull final String key, boolean value) {
mandatoryKeys.remove(key);
+ optionalKeys.add(key);
options.setBoolean(key, value);
return getThisBuilder();
}
@@ -187,10 +192,19 @@ public B opt(@Nonnull final String key, boolean value) {
@Override
public B opt(@Nonnull final String key, int value) {
mandatoryKeys.remove(key);
+ optionalKeys.add(key);
options.setInt(key, value);
return getThisBuilder();
}
+ @Override
+ public B opt(@Nonnull final String key, final long value) {
+ mandatoryKeys.remove(key);
+ optionalKeys.add(key);
+ options.setLong(key, value);
+ return getThisBuilder();
+ }
+
/**
* Set optional float parameter for the Builder.
*
@@ -199,6 +213,7 @@ public B opt(@Nonnull final String key, int value) {
@Override
public B opt(@Nonnull final String key, float value) {
mandatoryKeys.remove(key);
+ optionalKeys.add(key);
options.setFloat(key, value);
return getThisBuilder();
}
@@ -211,6 +226,7 @@ public B opt(@Nonnull final String key, float value) {
@Override
public B opt(@Nonnull final String key, double value) {
mandatoryKeys.remove(key);
+ optionalKeys.add(key);
options.setDouble(key, value);
return getThisBuilder();
}
@@ -223,6 +239,7 @@ public B opt(@Nonnull final String key, double value) {
@Override
public B opt(@Nonnull final String key, @Nonnull final String... values) {
mandatoryKeys.remove(key);
+ optionalKeys.add(key);
options.setStrings(key, values);
return getThisBuilder();
}
@@ -248,6 +265,7 @@ public B must(@Nonnull final String key, @Nonnull final String value) {
@Override
public B must(@Nonnull final String key, boolean value) {
mandatoryKeys.add(key);
+ optionalKeys.remove(key);
options.setBoolean(key, value);
return getThisBuilder();
}
@@ -260,10 +278,19 @@ public B must(@Nonnull final String key, boolean value) {
@Override
public B must(@Nonnull final String key, int value) {
mandatoryKeys.add(key);
+ optionalKeys.remove(key);
options.setInt(key, value);
return getThisBuilder();
}
+ @Override
+ public B must(@Nonnull final String key, final long value) {
+ mandatoryKeys.add(key);
+ optionalKeys.remove(key);
+ options.setLong(key, value);
+ return getThisBuilder();
+ }
+
/**
* Set mandatory float option.
*
@@ -272,6 +299,7 @@ public B must(@Nonnull final String key, int value) {
@Override
public B must(@Nonnull final String key, float value) {
mandatoryKeys.add(key);
+ optionalKeys.remove(key);
options.setFloat(key, value);
return getThisBuilder();
}
@@ -284,6 +312,7 @@ public B must(@Nonnull final String key, float value) {
@Override
public B must(@Nonnull final String key, double value) {
mandatoryKeys.add(key);
+ optionalKeys.remove(key);
options.setDouble(key, value);
return getThisBuilder();
}
@@ -296,6 +325,7 @@ public B must(@Nonnull final String key, double value) {
@Override
public B must(@Nonnull final String key, @Nonnull final String... values) {
mandatoryKeys.add(key);
+ optionalKeys.remove(key);
options.setStrings(key, values);
return getThisBuilder();
}
@@ -310,10 +340,18 @@ public Configuration getOptions() {
/**
* Get all the keys that are set as mandatory keys.
+ * @return mandatory keys.
*/
public Set getMandatoryKeys() {
return Collections.unmodifiableSet(mandatoryKeys);
}
+ /**
+ * Get all the keys that are set as optional keys.
+ * @return optional keys.
+ */
+ public Set getOptionalKeys() {
+ return Collections.unmodifiableSet(optionalKeys);
+ }
/**
* Reject a configuration if one or more mandatory keys are
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/impl/AbstractMultipartUploader.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/impl/AbstractMultipartUploader.java
index 416924e18d87c9..f9ae9f55cc17f1 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/impl/AbstractMultipartUploader.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/impl/AbstractMultipartUploader.java
@@ -127,7 +127,7 @@ protected void checkPutArguments(Path filePath,
* {@inheritDoc}.
* @param path path to abort uploads under.
* @return a future to -1.
- * @throws IOException
+ * @throws IOException raised on errors performing I/O.
*/
public CompletableFuture abortUploadsUnderPath(Path path)
throws IOException {
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/impl/FileSystemMultipartUploader.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/impl/FileSystemMultipartUploader.java
index 2339e42128973a..481d927672dc31 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/impl/FileSystemMultipartUploader.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/impl/FileSystemMultipartUploader.java
@@ -51,6 +51,7 @@
import org.apache.hadoop.fs.PathHandle;
import org.apache.hadoop.fs.UploadHandle;
import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.util.functional.FutureIO;
import static org.apache.hadoop.fs.Path.mergePaths;
import static org.apache.hadoop.io.IOUtils.cleanupWithLogger;
@@ -98,7 +99,7 @@ public FileSystemMultipartUploader(
public CompletableFuture startUpload(Path filePath)
throws IOException {
checkPath(filePath);
- return FutureIOSupport.eval(() -> {
+ return FutureIO.eval(() -> {
Path collectorPath = createCollectorPath(filePath);
fs.mkdirs(collectorPath, FsPermission.getDirDefault());
@@ -116,7 +117,7 @@ public CompletableFuture putPart(UploadHandle uploadId,
throws IOException {
checkPutArguments(filePath, inputStream, partNumber, uploadId,
lengthInBytes);
- return FutureIOSupport.eval(() -> innerPutPart(filePath,
+ return FutureIO.eval(() -> innerPutPart(filePath,
inputStream, partNumber, uploadId, lengthInBytes));
}
@@ -179,7 +180,7 @@ public CompletableFuture complete(
Map handleMap) throws IOException {
checkPath(filePath);
- return FutureIOSupport.eval(() ->
+ return FutureIO.eval(() ->
innerComplete(uploadId, filePath, handleMap));
}
@@ -251,7 +252,7 @@ public CompletableFuture abort(UploadHandle uploadId,
Path collectorPath = new Path(new String(uploadIdByteArray, 0,
uploadIdByteArray.length, Charsets.UTF_8));
- return FutureIOSupport.eval(() -> {
+ return FutureIO.eval(() -> {
// force a check for a file existing; raises FNFE if not found
fs.getFileStatus(collectorPath);
fs.delete(collectorPath, true);
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/impl/FutureDataInputStreamBuilderImpl.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/impl/FutureDataInputStreamBuilderImpl.java
index 24a8d49747fe6d..833c21ec1a67fd 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/impl/FutureDataInputStreamBuilderImpl.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/impl/FutureDataInputStreamBuilderImpl.java
@@ -19,6 +19,7 @@
package org.apache.hadoop.fs.impl;
import javax.annotation.Nonnull;
+import javax.annotation.Nullable;
import java.io.IOException;
import java.util.concurrent.CompletableFuture;
@@ -47,7 +48,7 @@
* options accordingly, for example:
*
* If the option is not related to the file system, the option will be ignored.
- * If the option is must, but not supported by the file system, a
+ * If the option is must, but not supported/known by the file system, an
* {@link IllegalArgumentException} will be thrown.
*
*/
@@ -125,6 +126,9 @@ protected int getBufferSize() {
/**
* Set the size of the buffer to be used.
+ *
+ * @param bufSize buffer size.
+ * @return FutureDataInputStreamBuilder.
*/
public FutureDataInputStreamBuilder bufferSize(int bufSize) {
bufferSize = bufSize;
@@ -136,6 +140,8 @@ public FutureDataInputStreamBuilder bufferSize(int bufSize) {
* This must be used after the constructor has been invoked to create
* the actual builder: it allows for subclasses to do things after
* construction.
+ *
+ * @return FutureDataInputStreamBuilder.
*/
public FutureDataInputStreamBuilder builder() {
return getThisBuilder();
@@ -147,8 +153,9 @@ public FutureDataInputStreamBuilder getThisBuilder() {
}
@Override
- public FutureDataInputStreamBuilder withFileStatus(FileStatus st) {
- this.status = requireNonNull(st, "status");
+ public FutureDataInputStreamBuilder withFileStatus(
+ @Nullable FileStatus st) {
+ this.status = st;
return this;
}
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/impl/FutureIOSupport.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/impl/FutureIOSupport.java
index 18f5187cb6134d..0a080426c2b247 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/impl/FutureIOSupport.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/impl/FutureIOSupport.java
@@ -20,7 +20,6 @@
import java.io.IOException;
import java.io.InterruptedIOException;
-import java.util.Map;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.CompletionException;
import java.util.concurrent.ExecutionException;
@@ -37,14 +36,16 @@
/**
* Support for future IO and the FS Builder subclasses.
- * If methods in here are needed for applications, promote
- * to {@link FutureIO} for public use -with the original
- * method relaying to it. This is to ensure that external
- * filesystem implementations can safely use these methods
+ * All methods in this class have been superceded by those in
+ * {@link FutureIO}.
+ * The methods here are retained but all marked as deprecated.
+ * This is to ensure that any external
+ * filesystem implementations can still use these methods
* without linkage problems surfacing.
*/
@InterfaceAudience.Private
@InterfaceStability.Unstable
+@Deprecated
public final class FutureIOSupport {
private FutureIOSupport() {
@@ -53,6 +54,7 @@ private FutureIOSupport() {
/**
* Given a future, evaluate it. Raised exceptions are
* extracted and handled.
+ * See {@link FutureIO#awaitFuture(Future, long, TimeUnit)}.
* @param future future to evaluate
* @param type of the result.
* @return the result, if all went well.
@@ -60,7 +62,8 @@ private FutureIOSupport() {
* @throws IOException if something went wrong
* @throws RuntimeException any nested RTE thrown
*/
- public static T awaitFuture(final Future future)
+ @Deprecated
+ public static T awaitFuture(final Future future)
throws InterruptedIOException, IOException, RuntimeException {
return FutureIO.awaitFuture(future);
}
@@ -69,14 +72,18 @@ public static T awaitFuture(final Future future)
/**
* Given a future, evaluate it. Raised exceptions are
* extracted and handled.
+ * See {@link FutureIO#awaitFuture(Future, long, TimeUnit)}.
* @param future future to evaluate
* @param type of the result.
+ * @param timeout timeout.
+ * @param unit unit.
* @return the result, if all went well.
* @throws InterruptedIOException future was interrupted
* @throws IOException if something went wrong
* @throws RuntimeException any nested RTE thrown
* @throws TimeoutException the future timed out.
*/
+ @Deprecated
public static T awaitFuture(final Future future,
final long timeout,
final TimeUnit unit)
@@ -88,10 +95,7 @@ public static T awaitFuture(final Future future,
/**
* From the inner cause of an execution exception, extract the inner cause
* if it is an IOE or RTE.
- * This will always raise an exception, either the inner IOException,
- * an inner RuntimeException, or a new IOException wrapping the raised
- * exception.
- *
+ * See {@link FutureIO#raiseInnerCause(ExecutionException)}.
* @param e exception.
* @param type of return value.
* @return nothing, ever.
@@ -99,6 +103,7 @@ public static T awaitFuture(final Future future,
* any non-Runtime-Exception
* @throws RuntimeException if that is the inner cause.
*/
+ @Deprecated
public static T raiseInnerCause(final ExecutionException e)
throws IOException {
return FutureIO.raiseInnerCause(e);
@@ -107,6 +112,7 @@ public static T raiseInnerCause(final ExecutionException e)
/**
* Extract the cause of a completion failure and rethrow it if an IOE
* or RTE.
+ * See {@link FutureIO#raiseInnerCause(CompletionException)}.
* @param e exception.
* @param type of return value.
* @return nothing, ever.
@@ -114,20 +120,15 @@ public static T raiseInnerCause(final ExecutionException e)
* any non-Runtime-Exception
* @throws RuntimeException if that is the inner cause.
*/
+ @Deprecated
public static T raiseInnerCause(final CompletionException e)
throws IOException {
return FutureIO.raiseInnerCause(e);
}
/**
- * Propagate options to any builder, converting everything with the
- * prefix to an option where, if there were 2+ dot-separated elements,
- * it is converted to a schema.
- * {@code
- * fs.example.s3a.option => s3a:option
- * fs.example.fs.io.policy => s3a.io.policy
- * fs.example.something => something
- * }
+ * Propagate options to any builder.
+ * {@link FutureIO#propagateOptions(FSBuilder, Configuration, String, String)}
* @param builder builder to modify
* @param conf configuration to read
* @param optionalPrefix prefix for optional settings
@@ -136,56 +137,39 @@ public static T raiseInnerCause(final CompletionException e)
* @param type of builder
* @return the builder passed in.
*/
+ @Deprecated
public static >
FSBuilder propagateOptions(
final FSBuilder builder,
final Configuration conf,
final String optionalPrefix,
final String mandatoryPrefix) {
- propagateOptions(builder, conf,
- optionalPrefix, false);
- propagateOptions(builder, conf,
- mandatoryPrefix, true);
- return builder;
+ return FutureIO.propagateOptions(builder,
+ conf, optionalPrefix, mandatoryPrefix);
}
/**
- * Propagate options to any builder, converting everything with the
- * prefix to an option where, if there were 2+ dot-separated elements,
- * it is converted to a schema.
- * {@code
- * fs.example.s3a.option => s3a:option
- * fs.example.fs.io.policy => s3a.io.policy
- * fs.example.something => something
- * }
+ * Propagate options to any builder.
+ * {@link FutureIO#propagateOptions(FSBuilder, Configuration, String, boolean)}
* @param builder builder to modify
* @param conf configuration to read
* @param prefix prefix to scan/strip
* @param mandatory are the options to be mandatory or optional?
*/
+ @Deprecated
public static void propagateOptions(
final FSBuilder builder,
final Configuration conf,
final String prefix,
final boolean mandatory) {
-
- final String p = prefix.endsWith(".") ? prefix : (prefix + ".");
- final Map propsWithPrefix = conf.getPropsWithPrefix(p);
- for (Map.Entry entry : propsWithPrefix.entrySet()) {
- // change the schema off each entry
- String key = entry.getKey();
- String val = entry.getValue();
- if (mandatory) {
- builder.must(key, val);
- } else {
- builder.opt(key, val);
- }
- }
+ FutureIO.propagateOptions(builder, conf, prefix, mandatory);
}
/**
* Evaluate a CallableRaisingIOE in the current thread,
* converting IOEs to RTEs and propagating.
+ * See {@link FutureIO#eval(CallableRaisingIOE)}.
+ *
* @param callable callable to invoke
* @param Return type.
* @return the evaluated result.
@@ -194,17 +178,6 @@ public static void propagateOptions(
*/
public static CompletableFuture eval(
CallableRaisingIOE callable) {
- CompletableFuture result = new CompletableFuture<>();
- try {
- result.complete(callable.apply());
- } catch (UnsupportedOperationException | IllegalArgumentException tx) {
- // fail fast here
- throw tx;
- } catch (Throwable tx) {
- // fail lazily here to ensure callers expect all File IO operations to
- // surface later
- result.completeExceptionally(tx);
- }
- return result;
+ return FutureIO.eval(callable);
}
}
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/impl/MultipartUploaderBuilderImpl.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/impl/MultipartUploaderBuilderImpl.java
index 5584e647849f5b..665bcc6a956602 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/impl/MultipartUploaderBuilderImpl.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/impl/MultipartUploaderBuilderImpl.java
@@ -88,6 +88,9 @@ protected MultipartUploaderBuilderImpl(@Nonnull FileContext fc,
/**
* Constructor.
+ *
+ * @param fileSystem fileSystem.
+ * @param p path.
*/
protected MultipartUploaderBuilderImpl(@Nonnull FileSystem fileSystem,
@Nonnull Path p) {
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/impl/OpenFileParameters.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/impl/OpenFileParameters.java
index 77b4ff52696a35..a19c5faff4d90f 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/impl/OpenFileParameters.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/impl/OpenFileParameters.java
@@ -38,6 +38,9 @@ public class OpenFileParameters {
*/
private Set mandatoryKeys;
+ /** The optional keys. */
+ private Set optionalKeys;
+
/**
* Options set during the build sequence.
*/
@@ -61,6 +64,11 @@ public OpenFileParameters withMandatoryKeys(final Set keys) {
return this;
}
+ public OpenFileParameters withOptionalKeys(final Set keys) {
+ this.optionalKeys = requireNonNull(keys);
+ return this;
+ }
+
public OpenFileParameters withOptions(final Configuration opts) {
this.options = requireNonNull(opts);
return this;
@@ -80,6 +88,10 @@ public Set getMandatoryKeys() {
return mandatoryKeys;
}
+ public Set getOptionalKeys() {
+ return optionalKeys;
+ }
+
public Configuration getOptions() {
return options;
}
@@ -91,4 +103,5 @@ public int getBufferSize() {
public FileStatus getStatus() {
return status;
}
+
}
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/impl/WrappedIOException.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/impl/WrappedIOException.java
index d2c999683c6c62..d5144b5e9c5314 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/impl/WrappedIOException.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/impl/WrappedIOException.java
@@ -20,20 +20,17 @@
import java.io.IOException;
import java.io.UncheckedIOException;
-import java.util.concurrent.ExecutionException;
import org.apache.hadoop.util.Preconditions;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
/**
- * A wrapper for an IOException which
- * {@link FutureIOSupport#raiseInnerCause(ExecutionException)} knows to
- * always extract the exception.
+ * A wrapper for an IOException.
*
* The constructor signature guarantees the cause will be an IOException,
* and as it checks for a null-argument, non-null.
- * @deprecated use the {@code UncheckedIOException}.
+ * @deprecated use the {@code UncheckedIOException} directly.]
*/
@Deprecated
@InterfaceAudience.Private
@@ -51,8 +48,4 @@ public WrappedIOException(final IOException cause) {
super(Preconditions.checkNotNull(cause));
}
- @Override
- public synchronized IOException getCause() {
- return (IOException) super.getCause();
- }
}
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/permission/AclStatus.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/permission/AclStatus.java
index 25b9ba659048a5..260ee7e570c9ba 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/permission/AclStatus.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/permission/AclStatus.java
@@ -185,7 +185,8 @@ public Builder stickyBit(boolean stickyBit) {
/**
* Sets the permission for the file.
- * @param permission
+ * @param permission permission.
+ * @return Builder.
*/
public Builder setPermission(FsPermission permission) {
this.permission = permission;
@@ -224,6 +225,7 @@ private AclStatus(String owner, String group, boolean stickyBit,
/**
* Get the effective permission for the AclEntry
* @param entry AclEntry to get the effective action
+ * @return FsAction.
*/
public FsAction getEffectivePermission(AclEntry entry) {
return getEffectivePermission(entry, permission);
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/permission/FsAction.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/permission/FsAction.java
index 97dcf816c16add..746e0e1e238f43 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/permission/FsAction.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/permission/FsAction.java
@@ -48,7 +48,8 @@ private FsAction(String s) {
/**
* Return true if this action implies that action.
- * @param that
+ * @param that FsAction that.
+ * @return if implies true,not false.
*/
public boolean implies(FsAction that) {
if (that != null) {
@@ -57,15 +58,26 @@ public boolean implies(FsAction that) {
return false;
}
- /** AND operation. */
+ /**
+ * AND operation.
+ * @param that FsAction that.
+ * @return FsAction.
+ */
public FsAction and(FsAction that) {
return vals[ordinal() & that.ordinal()];
}
- /** OR operation. */
+ /**
+ * OR operation.
+ * @param that FsAction that.
+ * @return FsAction.
+ */
public FsAction or(FsAction that) {
return vals[ordinal() | that.ordinal()];
}
- /** NOT operation. */
+ /**
+ * NOT operation.
+ * @return FsAction.
+ */
public FsAction not() {
return vals[7 - ordinal()];
}
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/permission/FsCreateModes.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/permission/FsCreateModes.java
index 2bd6f1f3b91266..ff3b4f6d65a491 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/permission/FsCreateModes.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/permission/FsCreateModes.java
@@ -35,7 +35,10 @@ public final class FsCreateModes extends FsPermission {
/**
* Create from unmasked mode and umask.
*
- * If the mode is already an FsCreateModes object, return it.
+ * @param mode mode.
+ * @param umask umask.
+ * @return If the mode is already
+ * an FsCreateModes object, return it.
*/
public static FsPermission applyUMask(FsPermission mode,
FsPermission umask) {
@@ -47,6 +50,10 @@ public static FsPermission applyUMask(FsPermission mode,
/**
* Create from masked and unmasked modes.
+ *
+ * @param masked masked.
+ * @param unmasked unmasked.
+ * @return FsCreateModes.
*/
public static FsCreateModes create(FsPermission masked,
FsPermission unmasked) {
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/permission/FsPermission.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/permission/FsPermission.java
index 51c113af2702ee..33fed1d3039904 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/permission/FsPermission.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/permission/FsPermission.java
@@ -56,7 +56,11 @@ public class FsPermission implements Writable, Serializable,
/** Maximum acceptable length of a permission string to parse */
public static final int MAX_PERMISSION_LENGTH = 10;
- /** Create an immutable {@link FsPermission} object. */
+ /**
+ * Create an immutable {@link FsPermission} object.
+ * @param permission permission.
+ * @return FsPermission.
+ */
public static FsPermission createImmutable(short permission) {
return new ImmutableFsPermission(permission);
}
@@ -85,7 +89,7 @@ public FsPermission(FsAction u, FsAction g, FsAction o, boolean sb) {
/**
* Construct by the given mode.
- * @param mode
+ * @param mode mode.
* @see #toShort()
*/
public FsPermission(short mode) { fromShort(mode); }
@@ -145,13 +149,19 @@ public FsPermission(String mode) {
this(new RawParser(mode).getPermission());
}
- /** Return user {@link FsAction}. */
+ /**
+ * @return Return user {@link FsAction}.
+ */
public FsAction getUserAction() {return useraction;}
- /** Return group {@link FsAction}. */
+ /**
+ * @return Return group {@link FsAction}.
+ */
public FsAction getGroupAction() {return groupaction;}
- /** Return other {@link FsAction}. */
+ /**
+ * @return Return other {@link FsAction}.
+ */
public FsAction getOtherAction() {return otheraction;}
private void set(FsAction u, FsAction g, FsAction o, boolean sb) {
@@ -180,6 +190,7 @@ public void readFields(DataInput in) throws IOException {
/**
* Get masked permission if exists.
+ * @return masked.
*/
public FsPermission getMasked() {
return null;
@@ -187,6 +198,7 @@ public FsPermission getMasked() {
/**
* Get unmasked permission if exists.
+ * @return unmasked.
*/
public FsPermission getUnmasked() {
return null;
@@ -194,6 +206,10 @@ public FsPermission getUnmasked() {
/**
* Create and initialize a {@link FsPermission} from {@link DataInput}.
+ *
+ * @param in data input.
+ * @throws IOException raised on errors performing I/O.
+ * @return FsPermission.
*/
public static FsPermission read(DataInput in) throws IOException {
FsPermission p = new FsPermission();
@@ -203,6 +219,7 @@ public static FsPermission read(DataInput in) throws IOException {
/**
* Encode the object to a short.
+ * @return object to a short.
*/
public short toShort() {
int s = (stickyBit ? 1 << 9 : 0) |
@@ -301,6 +318,9 @@ public FsPermission applyUMask(FsPermission umask) {
* '-' sets bits in the mask.
*
* Octal umask, the specified bits are set in the file mode creation mask.
+ *
+ * @param conf configuration.
+ * @return FsPermission UMask.
*/
public static FsPermission getUMask(Configuration conf) {
int umask = DEFAULT_UMASK;
@@ -346,7 +366,11 @@ public boolean getAclBit() {
}
/**
- * Returns true if the file is encrypted or directory is in an encryption zone
+ * Returns true if the file is encrypted or directory is in an encryption zone.
+ *
+ * @return if the file is encrypted or directory
+ * is in an encryption zone true, not false.
+ *
* @deprecated Get encryption bit from the
* {@link org.apache.hadoop.fs.FileStatus} object.
*/
@@ -357,6 +381,9 @@ public boolean getEncryptedBit() {
/**
* Returns true if the file or directory is erasure coded.
+ *
+ * @return if the file or directory is
+ * erasure coded true, not false.
* @deprecated Get ec bit from the {@link org.apache.hadoop.fs.FileStatus}
* object.
*/
@@ -365,7 +392,11 @@ public boolean getErasureCodedBit() {
return false;
}
- /** Set the user file creation mask (umask) */
+ /**
+ * Set the user file creation mask (umask)
+ * @param conf configuration.
+ * @param umask umask.
+ */
public static void setUMask(Configuration conf, FsPermission umask) {
conf.set(UMASK_LABEL, String.format("%1$03o", umask.toShort()));
}
@@ -379,6 +410,8 @@ public static void setUMask(Configuration conf, FsPermission umask) {
* {@link FsPermission#getDirDefault()} for directory, and use
* {@link FsPermission#getFileDefault()} for file.
* This method is kept for compatibility.
+ *
+ * @return Default FsPermission.
*/
public static FsPermission getDefault() {
return new FsPermission((short)00777);
@@ -386,6 +419,8 @@ public static FsPermission getDefault() {
/**
* Get the default permission for directory.
+ *
+ * @return DirDefault FsPermission.
*/
public static FsPermission getDirDefault() {
return new FsPermission((short)00777);
@@ -393,6 +428,8 @@ public static FsPermission getDirDefault() {
/**
* Get the default permission for file.
+ *
+ * @return FileDefault FsPermission.
*/
public static FsPermission getFileDefault() {
return new FsPermission((short)00666);
@@ -400,6 +437,8 @@ public static FsPermission getFileDefault() {
/**
* Get the default permission for cache pools.
+ *
+ * @return CachePoolDefault FsPermission.
*/
public static FsPermission getCachePoolDefault() {
return new FsPermission((short)00755);
@@ -408,6 +447,7 @@ public static FsPermission getCachePoolDefault() {
/**
* Create a FsPermission from a Unix symbolic permission string
* @param unixSymbolicPermission e.g. "-rw-rw-rw-"
+ * @return FsPermission.
*/
public static FsPermission valueOf(String unixSymbolicPermission) {
if (unixSymbolicPermission == null) {
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/permission/PermissionStatus.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/permission/PermissionStatus.java
index 3c3693f613baf0..be4beb506a63f8 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/permission/PermissionStatus.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/permission/PermissionStatus.java
@@ -39,7 +39,13 @@ public class PermissionStatus implements Writable {
WritableFactories.setFactory(PermissionStatus.class, FACTORY);
}
- /** Create an immutable {@link PermissionStatus} object. */
+ /**
+ * Create an immutable {@link PermissionStatus} object.
+ * @param user user.
+ * @param group group.
+ * @param permission permission.
+ * @return PermissionStatus.
+ */
public static PermissionStatus createImmutable(
String user, String group, FsPermission permission) {
return new PermissionStatus(user, group, permission) {
@@ -56,20 +62,35 @@ public void readFields(DataInput in) throws IOException {
private PermissionStatus() {}
- /** Constructor */
+ /**
+ * Constructor.
+ *
+ * @param user user.
+ * @param group group.
+ * @param permission permission.
+ */
public PermissionStatus(String user, String group, FsPermission permission) {
username = user;
groupname = group;
this.permission = permission;
}
- /** Return user name */
+ /**
+ * Return user name.
+ * @return user name.
+ */
public String getUserName() {return username;}
- /** Return group name */
+ /**
+ * Return group name.
+ * @return group name.
+ */
public String getGroupName() {return groupname;}
- /** Return permission */
+ /**
+ * Return permission.
+ * @return FsPermission.
+ */
public FsPermission getPermission() {return permission;}
@Override
@@ -86,6 +107,9 @@ public void write(DataOutput out) throws IOException {
/**
* Create and initialize a {@link PermissionStatus} from {@link DataInput}.
+ * @param in data input.
+ * @throws IOException raised on errors performing I/O.
+ * @return PermissionStatus.
*/
public static PermissionStatus read(DataInput in) throws IOException {
PermissionStatus p = new PermissionStatus();
@@ -95,6 +119,11 @@ public static PermissionStatus read(DataInput in) throws IOException {
/**
* Serialize a {@link PermissionStatus} from its base components.
+ * @param out out.
+ * @param username username.
+ * @param groupname groupname.
+ * @param permission FsPermission.
+ * @throws IOException raised on errors performing I/O.
*/
public static void write(DataOutput out,
String username,
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/sftp/SFTPFileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/sftp/SFTPFileSystem.java
index 46b829f14a15c5..3260a0db514797 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/sftp/SFTPFileSystem.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/sftp/SFTPFileSystem.java
@@ -709,11 +709,11 @@ public FileStatus getFileStatus(Path f) throws IOException {
@Override
public void close() throws IOException {
- if (closed.getAndSet(true)) {
- return;
- }
try {
super.close();
+ if (closed.getAndSet(true)) {
+ return;
+ }
} finally {
if (connectionPool != null) {
connectionPool.shutdown();
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Command.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Command.java
index 0bdb47730a9294..7858238ee71fde 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Command.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Command.java
@@ -77,7 +77,11 @@ protected Command() {
err = System.err;
}
- /** Constructor */
+ /**
+ * Constructor.
+ *
+ * @param conf configuration.
+ */
protected Command(Configuration conf) {
super(conf);
}
@@ -109,7 +113,7 @@ protected int getDepth() {
* Execute the command on the input path data. Commands can override to make
* use of the resolved filesystem.
* @param pathData The input path with resolved filesystem
- * @throws IOException
+ * @throws IOException raised on errors performing I/O.
*/
protected void run(PathData pathData) throws IOException {
run(pathData.path);
@@ -136,11 +140,19 @@ public int runAll() {
return exitCode;
}
- /** sets the command factory for later use */
+ /**
+ * sets the command factory for later use.
+ * @param factory factory.
+ */
public void setCommandFactory(CommandFactory factory) {
this.commandFactory = factory;
}
- /** retrieves the command factory */
+
+ /**
+ * retrieves the command factory.
+ *
+ * @return command factory.
+ */
protected CommandFactory getCommandFactory() {
return this.commandFactory;
}
@@ -201,7 +213,7 @@ public int run(String...argv) {
* IllegalArgumentException is thrown, the FsShell object will print the
* short usage of the command.
* @param args the command line arguments
- * @throws IOException
+ * @throws IOException raised on errors performing I/O.
*/
protected void processOptions(LinkedList args) throws IOException {}
@@ -211,7 +223,7 @@ protected void processOptions(LinkedList args) throws IOException {}
* {@link #expandArguments(LinkedList)} and pass the resulting list to
* {@link #processArguments(LinkedList)}
* @param args the list of argument strings
- * @throws IOException
+ * @throws IOException raised on errors performing I/O.
*/
protected void processRawArguments(LinkedList args)
throws IOException {
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CommandWithDestination.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CommandWithDestination.java
index f6f4247489ff36..69a418c1925eb3 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CommandWithDestination.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CommandWithDestination.java
@@ -55,6 +55,9 @@
import static org.apache.hadoop.fs.CreateFlag.CREATE;
import static org.apache.hadoop.fs.CreateFlag.LAZY_PERSIST;
import static org.apache.hadoop.fs.CreateFlag.OVERWRITE;
+import static org.apache.hadoop.fs.Options.OpenFileOptions.FS_OPTION_OPENFILE_READ_POLICY;
+import static org.apache.hadoop.fs.Options.OpenFileOptions.FS_OPTION_OPENFILE_READ_POLICY_WHOLE_FILE;
+import static org.apache.hadoop.util.functional.FutureIO.awaitFuture;
/**
* Provides: argument processing to ensure the destination is valid
@@ -116,6 +119,8 @@ protected void setDirectWrite(boolean flag) {
* owner, group and permission information of the source
* file will be preserved as far as target {@link FileSystem}
* implementation allows.
+ *
+ * @param preserve preserve.
*/
protected void setPreserve(boolean preserve) {
if (preserve) {
@@ -172,6 +177,7 @@ protected void preserve(FileAttribute fileAttribute) {
* The last arg is expected to be a local path, if only one argument is
* given then the destination will be the current directory
* @param args is the list of arguments
+ * @throws IOException raised on errors performing I/O.
*/
protected void getLocalDestination(LinkedList args)
throws IOException {
@@ -348,7 +354,11 @@ protected void copyFileToTarget(PathData src, PathData target)
src.fs.setVerifyChecksum(verifyChecksum);
InputStream in = null;
try {
- in = src.fs.open(src.path);
+ in = awaitFuture(src.fs.openFile(src.path)
+ .withFileStatus(src.stat)
+ .opt(FS_OPTION_OPENFILE_READ_POLICY,
+ FS_OPTION_OPENFILE_READ_POLICY_WHOLE_FILE)
+ .build());
copyStreamToTarget(in, target);
preserveAttributes(src, target, preserveRawXattrs);
} finally {
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CopyCommands.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CopyCommands.java
index b03d7de8a1c725..0643a2e983dcce 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CopyCommands.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CopyCommands.java
@@ -98,7 +98,8 @@ protected void processArguments(LinkedList items)
try {
for (PathData src : srcs) {
if (src.stat.getLen() != 0) {
- try (FSDataInputStream in = src.fs.open(src.path)) {
+ // Always do sequential reads.
+ try (FSDataInputStream in = src.openForSequentialIO()) {
IOUtils.copyBytes(in, out, getConf(), false);
writeDelimiter(out);
}
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Display.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Display.java
index 670fa152f72ede..d3ca013a3f2518 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Display.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Display.java
@@ -105,7 +105,8 @@ private void printToStdout(InputStream in) throws IOException {
}
protected InputStream getInputStream(PathData item) throws IOException {
- return item.fs.open(item.path);
+ // Always do sequential reads;
+ return item.openForSequentialIO();
}
}
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Head.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Head.java
index 2280225b5ae32c..7242f261801d65 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Head.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Head.java
@@ -28,6 +28,8 @@
import java.util.LinkedList;
import java.util.List;
+import static org.apache.hadoop.fs.Options.OpenFileOptions.FS_OPTION_OPENFILE_READ_POLICY_SEQUENTIAL;
+
/**
* Show the first 1KB of the file.
*/
@@ -68,11 +70,9 @@ protected void processPath(PathData item) throws IOException {
}
private void dumpToOffset(PathData item) throws IOException {
- FSDataInputStream in = item.fs.open(item.path);
- try {
+ try (FSDataInputStream in = item.openFile(
+ FS_OPTION_OPENFILE_READ_POLICY_SEQUENTIAL)) {
IOUtils.copyBytes(in, System.out, endingOffset, false);
- } finally {
- in.close();
}
}
}
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/PathData.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/PathData.java
index 1ff8d8f0494a19..da99ac212563eb 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/PathData.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/PathData.java
@@ -29,6 +29,7 @@
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.LocalFileSystem;
@@ -39,6 +40,10 @@
import org.apache.hadoop.fs.PathNotFoundException;
import org.apache.hadoop.fs.RemoteIterator;
+import static org.apache.hadoop.fs.Options.OpenFileOptions.FS_OPTION_OPENFILE_READ_POLICY;
+import static org.apache.hadoop.fs.Options.OpenFileOptions.FS_OPTION_OPENFILE_READ_POLICY_SEQUENTIAL;
+import static org.apache.hadoop.fs.Options.OpenFileOptions.FS_OPTION_OPENFILE_LENGTH;
+import static org.apache.hadoop.util.functional.FutureIO.awaitFuture;
import static org.apache.hadoop.util.functional.RemoteIterators.mappingRemoteIterator;
/**
@@ -601,4 +606,35 @@ public boolean equals(Object o) {
public int hashCode() {
return path.hashCode();
}
+
+
+ /**
+ * Open a file for sequential IO.
+ *
+ * This uses FileSystem.openFile() to request sequential IO;
+ * the file status is also passed in.
+ * Filesystems may use to optimize their IO.
+ *
+ * @return an input stream
+ * @throws IOException failure
+ */
+ protected FSDataInputStream openForSequentialIO()
+ throws IOException {
+ return openFile(FS_OPTION_OPENFILE_READ_POLICY_SEQUENTIAL);
+ }
+
+ /**
+ * Open a file.
+ * @param policy fadvise policy.
+ * @return an input stream
+ * @throws IOException failure
+ */
+ protected FSDataInputStream openFile(final String policy) throws IOException {
+ return awaitFuture(fs.openFile(path)
+ .opt(FS_OPTION_OPENFILE_READ_POLICY,
+ policy)
+ .opt(FS_OPTION_OPENFILE_LENGTH,
+ stat.getLen()) // file length hint for object stores
+ .build());
+ }
}
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Tail.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Tail.java
index 98db1c3b7b605a..26ac3fee4713b1 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Tail.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Tail.java
@@ -30,6 +30,8 @@
import org.apache.hadoop.classification.VisibleForTesting;
+import static org.apache.hadoop.fs.Options.OpenFileOptions.FS_OPTION_OPENFILE_READ_POLICY_SEQUENTIAL;
+
/**
* Get a listing of all files in that match the file patterns.
*/
@@ -107,16 +109,15 @@ private long dumpFromOffset(PathData item, long offset) throws IOException {
if (offset < 0) {
offset = Math.max(fileSize + offset, 0);
}
-
- FSDataInputStream in = item.fs.open(item.path);
- try {
+ // Always do sequential reads.
+ try (FSDataInputStream in = item.openFile(
+ FS_OPTION_OPENFILE_READ_POLICY_SEQUENTIAL)) {
in.seek(offset);
// use conf so the system configured io block size is used
IOUtils.copyBytes(in, System.out, getConf(), false);
offset = in.getPos();
- } finally {
- in.close();
}
return offset;
}
+
}
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/find/BaseExpression.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/find/BaseExpression.java
index 5069d2d34e51c1..cd9bbe2bc884ec 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/find/BaseExpression.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/find/BaseExpression.java
@@ -38,12 +38,18 @@ public abstract class BaseExpression implements Expression, Configurable {
private String[] usage = { "Not yet implemented" };
private String[] help = { "Not yet implemented" };
- /** Sets the usage text for this {@link Expression} */
+ /**
+ * Sets the usage text for this {@link Expression} .
+ * @param usage usage array.
+ */
protected void setUsage(String[] usage) {
this.usage = usage;
}
- /** Sets the help text for this {@link Expression} */
+ /**
+ * Sets the help text for this {@link Expression} .
+ * @param help help.
+ */
protected void setHelp(String[] help) {
this.help = help;
}
@@ -92,7 +98,10 @@ public void finish() throws IOException {
/** Children of this expression. */
private LinkedList children = new LinkedList();
- /** Return the options to be used by this expression. */
+ /**
+ * Return the options to be used by this expression.
+ * @return options.
+ */
protected FindOptions getOptions() {
return (this.options == null) ? new FindOptions() : this.options;
}
@@ -265,6 +274,7 @@ protected void addArgument(String arg) {
* @param depth
* current depth in the process directories
* @return FileStatus
+ * @throws IOException raised on errors performing I/O.
*/
protected FileStatus getFileStatus(PathData item, int depth)
throws IOException {
@@ -285,6 +295,8 @@ protected FileStatus getFileStatus(PathData item, int depth)
* @param item
* PathData
* @return Path
+ *
+ * @throws IOException raised on errors performing I/O.
*/
protected Path getPath(PathData item) throws IOException {
return item.path;
@@ -295,6 +307,7 @@ protected Path getPath(PathData item) throws IOException {
*
* @param item PathData
* @return FileSystem
+ * @throws IOException raised on errors performing I/O.
*/
protected FileSystem getFileSystem(PathData item) throws IOException {
return item.fs;
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/find/Expression.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/find/Expression.java
index ccad631028cc93..353fe685cc9cdc 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/find/Expression.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/find/Expression.java
@@ -30,13 +30,15 @@ public interface Expression {
/**
* Set the options for this expression, called once before processing any
* items.
+ * @param options options.
+ * @throws IOException raised on errors performing I/O.
*/
public void setOptions(FindOptions options) throws IOException;
/**
* Prepares the expression for execution, called once after setting options
* and before processing any options.
- * @throws IOException
+ * @throws IOException raised on errors performing I/O.
*/
public void prepare() throws IOException;
@@ -46,13 +48,14 @@ public interface Expression {
* @param item {@link PathData} item to be processed
* @param depth distance of the item from the command line argument
* @return {@link Result} of applying the expression to the item
+ * @throws IOException raised on errors performing I/O.
*/
public Result apply(PathData item, int depth) throws IOException;
/**
* Finishes the expression, called once after processing all items.
*
- * @throws IOException
+ * @throws IOException raised on errors performing I/O.
*/
public void finish() throws IOException;
@@ -76,15 +79,21 @@ public interface Expression {
/**
* Indicates whether this expression performs an action, i.e. provides output
* back to the user.
+ * @return if is action true, not false.
*/
public boolean isAction();
- /** Identifies the expression as an operator rather than a primary. */
+ /**
+ * Identifies the expression as an operator rather than a primary.
+ * @return if is operator true, not false.
+ */
public boolean isOperator();
/**
* Returns the precedence of this expression
* (only applicable to operators).
+ *
+ * @return precedence.
*/
public int getPrecedence();
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/find/FindOptions.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/find/FindOptions.java
index b0f1be5c35c930..c6051862305905 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/find/FindOptions.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/find/FindOptions.java
@@ -264,6 +264,7 @@ public void setConfiguration(Configuration configuration) {
/**
* Return the {@link Configuration} return configuration {@link Configuration}
+ * @return configuration.
*/
public Configuration getConfiguration() {
return this.configuration;
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/find/Result.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/find/Result.java
index 2ef9cb4a801d66..a242681acd0307 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/find/Result.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/find/Result.java
@@ -35,23 +35,36 @@ private Result(boolean success, boolean recurse) {
this.descend = recurse;
}
- /** Should further directories be descended. */
+ /**
+ * Should further directories be descended.
+ * @return if is pass true,not false.
+ */
public boolean isDescend() {
return this.descend;
}
- /** Should processing continue. */
+ /**
+ * Should processing continue.
+ * @return if is pass true,not false.
+ */
public boolean isPass() {
return this.success;
}
- /** Returns the combination of this and another result. */
+ /**
+ * Returns the combination of this and another result.
+ * @param other other.
+ * @return result.
+ */
public Result combine(Result other) {
return new Result(this.isPass() && other.isPass(), this.isDescend()
&& other.isDescend());
}
- /** Negate this result. */
+ /**
+ * Negate this result.
+ * @return Result.
+ */
public Result negate() {
return new Result(!this.isPass(), this.isDescend());
}
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/statistics/IOStatisticsSnapshot.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/statistics/IOStatisticsSnapshot.java
index 63d37e97c98b8f..88606eb4b30554 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/statistics/IOStatisticsSnapshot.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/statistics/IOStatisticsSnapshot.java
@@ -53,7 +53,7 @@
* deserialized. If for some reason this is required, use
* {@link #requiredSerializationClasses()} to get the list of classes
* used when deserializing instances of this object.
- *
+ *
*
* It is annotated for correct serializations with jackson2.
*
@@ -238,6 +238,8 @@ public static JsonSerialization serializer() {
/**
* Serialize by converting each map to a TreeMap, and saving that
* to the stream.
+ * @param s ObjectOutputStream.
+ * @throws IOException raised on errors performing I/O.
*/
private synchronized void writeObject(ObjectOutputStream s)
throws IOException {
@@ -253,6 +255,10 @@ private synchronized void writeObject(ObjectOutputStream s)
/**
* Deserialize by loading each TreeMap, and building concurrent
* hash maps from them.
+ *
+ * @param s ObjectInputStream.
+ * @throws IOException raised on errors performing I/O.
+ * @throws ClassNotFoundException class not found exception
*/
private void readObject(final ObjectInputStream s)
throws IOException, ClassNotFoundException {
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/statistics/IOStatisticsSupport.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/statistics/IOStatisticsSupport.java
index 75977047c0f2a6..bb4d9a44587a2b 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/statistics/IOStatisticsSupport.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/statistics/IOStatisticsSupport.java
@@ -71,6 +71,7 @@ private IOStatisticsSupport() {
* Returns null if the source isn't of the write type
* or the return value of
* {@link IOStatisticsSource#getIOStatistics()} was null.
+ * @param source source.
* @return an IOStatistics instance or null
*/
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/statistics/MeanStatistic.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/statistics/MeanStatistic.java
index d9ff0c25c6a21c..369db49654382d 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/statistics/MeanStatistic.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/statistics/MeanStatistic.java
@@ -207,6 +207,7 @@ public synchronized double mean() {
/**
* Add another MeanStatistic.
* @param other other value
+ * @return mean statistic.
*/
public synchronized MeanStatistic add(final MeanStatistic other) {
if (other.isEmpty()) {
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/statistics/StoreStatisticNames.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/statistics/StoreStatisticNames.java
index 166007f5c9a42e..c458269c3510d7 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/statistics/StoreStatisticNames.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/statistics/StoreStatisticNames.java
@@ -118,6 +118,9 @@ public final class StoreStatisticNames {
/** {@value}. */
public static final String OP_OPEN = "op_open";
+ /** Call to openFile() {@value}. */
+ public static final String OP_OPENFILE = "op_openfile";
+
/** {@value}. */
public static final String OP_REMOVE_ACL = "op_remove_acl";
@@ -323,6 +326,12 @@ public final class StoreStatisticNames {
public static final String ACTION_EXECUTOR_ACQUIRED =
"action_executor_acquired";
+ /**
+ * A file was opened: {@value}.
+ */
+ public static final String ACTION_FILE_OPENED
+ = "action_file_opened";
+
/**
* An HTTP HEAD request was made: {@value}.
*/
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/statistics/StreamStatisticNames.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/statistics/StreamStatisticNames.java
index 7e9137294c1efb..ca755f08419143 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/statistics/StreamStatisticNames.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/statistics/StreamStatisticNames.java
@@ -76,7 +76,7 @@ public final class StreamStatisticNames {
public static final String STREAM_READ_CLOSED = "stream_read_closed";
/**
- * Total count of times an attempt to close an input stream was made
+ * Total count of times an attempt to close an input stream was made.
* Value: {@value}.
*/
public static final String STREAM_READ_CLOSE_OPERATIONS
@@ -118,6 +118,23 @@ public final class StreamStatisticNames {
public static final String STREAM_READ_OPERATIONS_INCOMPLETE
= "stream_read_operations_incomplete";
+ /**
+ * count/duration of aborting a remote stream during stream IO
+ * IO.
+ * Value: {@value}.
+ */
+ public static final String STREAM_READ_REMOTE_STREAM_ABORTED
+ = "stream_read_remote_stream_aborted";
+
+ /**
+ * count/duration of closing a remote stream,
+ * possibly including draining the stream to recycle
+ * the HTTP connection.
+ * Value: {@value}.
+ */
+ public static final String STREAM_READ_REMOTE_STREAM_DRAINED
+ = "stream_read_remote_stream_drain";
+
/**
* Count of version mismatches encountered while reading an input stream.
* Value: {@value}.
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/statistics/impl/IOStatisticsBinding.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/statistics/impl/IOStatisticsBinding.java
index 70366fc7a30577..6a5d01fb3b0746 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/statistics/impl/IOStatisticsBinding.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/statistics/impl/IOStatisticsBinding.java
@@ -141,6 +141,7 @@ public static String entryToString(
/**
* Convert entry values to the string format used in logging.
*
+ * @param type of values.
* @param name statistic name
* @param value stat value
* @return formatted string
@@ -178,6 +179,8 @@ private static Map copyMap(
/**
* A passthrough copy operation suitable for immutable
* types, including numbers.
+ *
+ * @param type of values.
* @param src source object
* @return the source object
*/
@@ -437,6 +440,7 @@ public static Function trackJavaFunctionDuration(
* @param input input callable.
* @param return type.
* @return the result of the operation.
+ * @throws IOException raised on errors performing I/O.
*/
public static B trackDuration(
DurationTrackerFactory factory,
@@ -521,23 +525,39 @@ public static CallableRaisingIOE trackDurationOfOperation(
// create the tracker outside try-with-resources so
// that failures can be set in the catcher.
DurationTracker tracker = createTracker(factory, statistic);
- try {
- // exec the input function and return its value
- return input.apply();
- } catch (IOException | RuntimeException e) {
- // input function failed: note it
- tracker.failed();
- // and rethrow
- throw e;
- } finally {
- // update the tracker.
- // this is called after the catch() call would have
- // set the failed flag.
- tracker.close();
- }
+ return invokeTrackingDuration(tracker, input);
};
}
+ /**
+ * Given an IOException raising callable/lambda expression,
+ * execute it, updating the tracker on success/failure.
+ * @param tracker duration tracker.
+ * @param input input callable.
+ * @param return type.
+ * @return the result of the invocation
+ * @throws IOException on failure.
+ */
+ public static B invokeTrackingDuration(
+ final DurationTracker tracker,
+ final CallableRaisingIOE input)
+ throws IOException {
+ try {
+ // exec the input function and return its value
+ return input.apply();
+ } catch (IOException | RuntimeException e) {
+ // input function failed: note it
+ tracker.failed();
+ // and rethrow
+ throw e;
+ } finally {
+ // update the tracker.
+ // this is called after the catch() call would have
+ // set the failed flag.
+ tracker.close();
+ }
+ }
+
/**
* Given an IOException raising Consumer,
* return a new one which wraps the inner and tracks
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/store/DataBlocks.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/store/DataBlocks.java
index d9d3850ef4e2e6..c70d0ee91e15ec 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/store/DataBlocks.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/store/DataBlocks.java
@@ -107,6 +107,7 @@ private DataBlocks() {
* @param len number of bytes to be written.
* @throws NullPointerException for a null buffer
* @throws IndexOutOfBoundsException if indices are out of range
+ * @throws IOException raised on errors performing I/O.
*/
public static void validateWriteArgs(byte[] b, int off, int len)
throws IOException {
@@ -287,6 +288,7 @@ protected BlockFactory(String keyToBufferDir, Configuration conf) {
* @param limit limit of the block.
* @param statistics stats to work with
* @return a new block.
+ * @throws IOException raised on errors performing I/O.
*/
public abstract DataBlock create(long index, int limit,
BlockUploadStatistics statistics)
@@ -482,6 +484,8 @@ public void close() throws IOException {
/**
* Inner close logic for subclasses to implement.
+ *
+ * @throws IOException raised on errors performing I/O.
*/
protected void innerClose() throws IOException {
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/store/audit/AuditingFunctions.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/store/audit/AuditingFunctions.java
index acc82766be1909..9ad727f5a2df21 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/store/audit/AuditingFunctions.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/store/audit/AuditingFunctions.java
@@ -86,6 +86,8 @@ public static InvocationRaisingIOE withinAuditSpan(
* activates and deactivates the span around the inner one.
* @param auditSpan audit span
* @param operation operation
+ * @param Generics Type T.
+ * @param Generics Type R.
* @return a new invocation.
*/
public static FunctionRaisingIOE withinAuditSpan(
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ConfigUtil.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ConfigUtil.java
index 09ec5d2933058b..c9ee5e232d944f 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ConfigUtil.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ConfigUtil.java
@@ -48,7 +48,7 @@ public static String getConfigViewFsPrefix() {
/**
* Add a link to the config for the specified mount table
* @param conf - add the link to this conf
- * @param mountTableName
+ * @param mountTableName mountTable.
* @param src - the src path name
* @param target - the target URI link
*/
@@ -71,9 +71,10 @@ public static void addLink(final Configuration conf, final String src,
/**
* Add a LinkMergeSlash to the config for the specified mount table.
- * @param conf
- * @param mountTableName
- * @param target
+ *
+ * @param conf configuration.
+ * @param mountTableName mountTable.
+ * @param target target.
*/
public static void addLinkMergeSlash(Configuration conf,
final String mountTableName, final URI target) {
@@ -83,8 +84,9 @@ public static void addLinkMergeSlash(Configuration conf,
/**
* Add a LinkMergeSlash to the config for the default mount table.
- * @param conf
- * @param target
+ *
+ * @param conf configuration.
+ * @param target targets.
*/
public static void addLinkMergeSlash(Configuration conf, final URI target) {
addLinkMergeSlash(conf, getDefaultMountTableName(conf), target);
@@ -92,9 +94,10 @@ public static void addLinkMergeSlash(Configuration conf, final URI target) {
/**
* Add a LinkFallback to the config for the specified mount table.
- * @param conf
- * @param mountTableName
- * @param target
+ *
+ * @param conf configuration.
+ * @param mountTableName mountTable.
+ * @param target targets.
*/
public static void addLinkFallback(Configuration conf,
final String mountTableName, final URI target) {
@@ -104,8 +107,9 @@ public static void addLinkFallback(Configuration conf,
/**
* Add a LinkFallback to the config for the default mount table.
- * @param conf
- * @param target
+ *
+ * @param conf configuration.
+ * @param target targets.
*/
public static void addLinkFallback(Configuration conf, final URI target) {
addLinkFallback(conf, getDefaultMountTableName(conf), target);
@@ -113,9 +117,10 @@ public static void addLinkFallback(Configuration conf, final URI target) {
/**
* Add a LinkMerge to the config for the specified mount table.
- * @param conf
- * @param mountTableName
- * @param targets
+ *
+ * @param conf configuration.
+ * @param mountTableName mountTable.
+ * @param targets targets.
*/
public static void addLinkMerge(Configuration conf,
final String mountTableName, final URI[] targets) {
@@ -125,8 +130,9 @@ public static void addLinkMerge(Configuration conf,
/**
* Add a LinkMerge to the config for the default mount table.
- * @param conf
- * @param targets
+ *
+ * @param conf configuration.
+ * @param targets targets array.
*/
public static void addLinkMerge(Configuration conf, final URI[] targets) {
addLinkMerge(conf, getDefaultMountTableName(conf), targets);
@@ -134,6 +140,12 @@ public static void addLinkMerge(Configuration conf, final URI[] targets) {
/**
* Add nfly link to configuration for the given mount table.
+ *
+ * @param conf configuration.
+ * @param mountTableName mount table.
+ * @param src src.
+ * @param settings settings.
+ * @param targets targets.
*/
public static void addLinkNfly(Configuration conf, String mountTableName,
String src, String settings, final String targets) {
@@ -144,12 +156,13 @@ public static void addLinkNfly(Configuration conf, String mountTableName,
}
/**
+ * Add nfly link to configuration for the given mount table.
*
- * @param conf
- * @param mountTableName
- * @param src
- * @param settings
- * @param targets
+ * @param conf configuration.
+ * @param mountTableName mount table.
+ * @param src src.
+ * @param settings settings.
+ * @param targets targets.
*/
public static void addLinkNfly(Configuration conf, String mountTableName,
String src, String settings, final URI ... targets) {
@@ -202,6 +215,7 @@ public static void setHomeDirConf(final Configuration conf,
* Add config variable for homedir the specified mount table
* @param conf - add to this conf
* @param homedir - the home dir path starting with slash
+ * @param mountTableName - the mount table.
*/
public static void setHomeDirConf(final Configuration conf,
final String mountTableName, final String homedir) {
@@ -247,4 +261,22 @@ public static String getDefaultMountTableName(final Configuration conf) {
return conf.get(Constants.CONFIG_VIEWFS_DEFAULT_MOUNT_TABLE_NAME_KEY,
Constants.CONFIG_VIEWFS_DEFAULT_MOUNT_TABLE);
}
+
+ /**
+ * Check the bool config whether nested mount point is supported. Default: true
+ * @param conf - from this conf
+ * @return whether nested mount point is supported
+ */
+ public static boolean isNestedMountPointSupported(final Configuration conf) {
+ return conf.getBoolean(Constants.CONFIG_NESTED_MOUNT_POINT_SUPPORTED, true);
+ }
+
+ /**
+ * Set the bool value isNestedMountPointSupported in config.
+ * @param conf - from this conf
+ * @param isNestedMountPointSupported - whether nested mount point is supported
+ */
+ public static void setIsNestedMountPointSupported(final Configuration conf, boolean isNestedMountPointSupported) {
+ conf.setBoolean(Constants.CONFIG_NESTED_MOUNT_POINT_SUPPORTED, isNestedMountPointSupported);
+ }
}
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/Constants.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/Constants.java
index 21f4d99f891c2b..806e69f32c6b7a 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/Constants.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/Constants.java
@@ -35,7 +35,7 @@ public interface Constants {
* Prefix for the config variable for the ViewFs mount-table path.
*/
String CONFIG_VIEWFS_MOUNTTABLE_PATH = CONFIG_VIEWFS_PREFIX + ".path";
-
+
/**
* Prefix for the home dir for the mount table - if not specified
* then the hadoop default value (/user) is used.
@@ -53,12 +53,17 @@ public interface Constants {
*/
public static final String CONFIG_VIEWFS_DEFAULT_MOUNT_TABLE = "default";
+ /**
+ * Config to enable nested mount point in viewfs
+ */
+ String CONFIG_NESTED_MOUNT_POINT_SUPPORTED = CONFIG_VIEWFS_PREFIX + ".nested.mount.point.supported";
+
/**
* Config variable full prefix for the default mount table.
*/
- public static final String CONFIG_VIEWFS_PREFIX_DEFAULT_MOUNT_TABLE =
+ public static final String CONFIG_VIEWFS_PREFIX_DEFAULT_MOUNT_TABLE =
CONFIG_VIEWFS_PREFIX + "." + CONFIG_VIEWFS_DEFAULT_MOUNT_TABLE;
-
+
/**
* Config variable for specifying a simple link
*/
@@ -82,7 +87,7 @@ public interface Constants {
/**
* Config variable for specifying a merge of the root of the mount-table
- * with the root of another file system.
+ * with the root of another file system.
*/
String CONFIG_VIEWFS_LINK_MERGE_SLASH = "linkMergeSlash";
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/FsGetter.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/FsGetter.java
index c72baac25fb756..f723f238e199ee 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/FsGetter.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/FsGetter.java
@@ -34,6 +34,10 @@ public class FsGetter {
/**
* Gets new file system instance of given uri.
+ * @param uri uri.
+ * @param conf configuration.
+ * @throws IOException raised on errors performing I/O.
+ * @return file system.
*/
public FileSystem getNewInstance(URI uri, Configuration conf)
throws IOException {
@@ -42,6 +46,11 @@ public FileSystem getNewInstance(URI uri, Configuration conf)
/**
* Gets file system instance of given uri.
+ *
+ * @param uri uri.
+ * @param conf configuration.
+ * @throws IOException raised on errors performing I/O.
+ * @return FileSystem.
*/
public FileSystem get(URI uri, Configuration conf) throws IOException {
return FileSystem.get(uri, conf);
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/InodeTree.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/InodeTree.java
index ef40f98cc02e04..5360d55e106448 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/InodeTree.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/InodeTree.java
@@ -17,6 +17,10 @@
*/
package org.apache.hadoop.fs.viewfs;
+import java.util.Collection;
+import java.util.Comparator;
+import java.util.Set;
+import java.util.TreeSet;
import java.util.function.Function;
import org.apache.hadoop.util.Preconditions;
import java.io.FileNotFoundException;
@@ -55,13 +59,13 @@
* @param is AbstractFileSystem or FileSystem
*
* The two main methods are
- * {@link #InodeTree(Configuration, String)} // constructor
+ * {@link #InodeTree(Configuration, String, URI, boolean)} // constructor
* {@link #resolve(String, boolean)}
*/
@InterfaceAudience.Private
@InterfaceStability.Unstable
-abstract class InodeTree {
+public abstract class InodeTree {
private static final Logger LOGGER =
LoggerFactory.getLogger(InodeTree.class.getName());
@@ -81,6 +85,8 @@ enum ResultKind {
private List> regexMountPointList =
new ArrayList>();
+ private final boolean isNestedMountPointSupported;
+
public static class MountPoint {
String src;
INodeLink target;
@@ -99,7 +105,7 @@ public String getSource() {
}
/**
- * Returns the target link.
+ * Returns the target INode link.
* @return The target INode link
*/
public INodeLink getTarget() {
@@ -138,6 +144,14 @@ public INode(String pathToNode, UserGroupInformation aUgi) {
boolean isLink() {
return !isInternalDir();
}
+
+ /**
+ * Return the link if isLink.
+ * @return will return null, for non links.
+ */
+ INodeLink getLink() {
+ return null;
+ }
}
/**
@@ -212,6 +226,51 @@ void addLink(final String pathComponent, final INodeLink link)
}
children.put(pathComponent, link);
}
+
+ void addDirLink(final String pathComponent, final INodeDirLink dirLink) {
+ children.put(pathComponent, dirLink);
+ }
+ }
+
+ /**
+ * Internal class to represent an INodeDir which also contains a INodeLink. This is used to support nested mount points
+ * where an INode is internalDir but points to a mount link. The class is a subclass of INodeDir and the semantics are
+ * as follows:
+ * isLink(): true
+ * isInternalDir(): true
+ * @param
+ */
+ static class INodeDirLink extends INodeDir {
+ /**
+ * INodeLink wrapped in the INodeDir
+ */
+ private final INodeLink link;
+
+ INodeDirLink(String pathToNode, UserGroupInformation aUgi, INodeLink link) {
+ super(pathToNode, aUgi);
+ this.link = link;
+ }
+
+ @Override
+ INodeLink getLink() {
+ return link;
+ }
+
+ /**
+ * True because the INodeDirLink also contains a INodeLink
+ */
+ @Override
+ boolean isLink() {
+ return true;
+ }
+
+ /**
+ * True because the INodeDirLink is internal node
+ */
+ @Override
+ boolean isInternalDir() {
+ return true;
+ }
}
/**
@@ -266,8 +325,8 @@ enum LinkType {
* A merge dir link is a merge (junction) of links to dirs:
* example : merge of 2 dirs
- * /users -> hdfs:nn1//users
- * /users -> hdfs:nn2//users
+ * /users -> hdfs:nn1//users
+ * /users -> hdfs:nn2//users
*
* For a merge, each target is checked to be dir when created but if target
* is changed later it is then ignored (a dir with null entries)
@@ -305,6 +364,8 @@ public static class INodeLink extends INode {
/**
* Get the target of the link. If a merge link then it returned
* as "," separated URI list.
+ *
+ * @return the path.
*/
public Path getTargetLink() {
StringBuilder result = new StringBuilder(targetDirLinkList[0].toString());
@@ -320,10 +381,15 @@ boolean isInternalDir() {
return false;
}
+ @Override
+ INodeLink getLink() {
+ return this;
+ }
+
/**
* Get the instance of FileSystem to use, creating one if needed.
* @return An Initialized instance of T
- * @throws IOException
+ * @throws IOException raised on errors performing I/O.
*/
public T getTargetFileSystem() throws IOException {
if (targetFileSystem != null) {
@@ -376,10 +442,17 @@ private void createLink(final String src, final String target,
newDir.setInternalDirFs(getTargetFileSystem(newDir));
nextInode = newDir;
}
- if (nextInode.isLink()) {
- // Error - expected a dir but got a link
- throw new FileAlreadyExistsException("Path " + nextInode.fullPath +
- " already exists as link");
+ if (!nextInode.isInternalDir()) {
+ if (isNestedMountPointSupported) {
+ // nested mount detected, add a new INodeDirLink that wraps existing INodeLink to INodeTree and override existing INodelink
+ INodeDirLink dirLink = new INodeDirLink(nextInode.fullPath, aUgi, (INodeLink) nextInode);
+ curInode.addDirLink(iPath, dirLink);
+ curInode = dirLink;
+ } else {
+ // Error - expected a dir but got a link
+ throw new FileAlreadyExistsException("Path " + nextInode.fullPath +
+ " already exists as link");
+ }
} else {
assert(nextInode.isInternalDir());
curInode = (INodeDir) nextInode;
@@ -429,7 +502,7 @@ private void createLink(final String src, final String target,
/**
* The user of this class must subclass and implement the following
* 3 abstract methods.
- * @throws IOException
+ * @return Function.
*/
protected abstract Function initAndGetTargetFs();
@@ -445,7 +518,7 @@ private INodeDir getRootDir() {
}
private INodeLink getRootLink() {
- Preconditions.checkState(root.isLink());
+ Preconditions.checkState(!root.isInternalDir());
return (INodeLink)root;
}
@@ -458,11 +531,11 @@ private boolean hasFallbackLink() {
* there will be root to root mapping. So, root does not represent as
* internalDir.
*/
- protected boolean isRootInternalDir() {
+ public boolean isRootInternalDir() {
return root.isInternalDir();
}
- protected INodeLink getRootFallbackLink() {
+ public INodeLink getRootFallbackLink() {
Preconditions.checkState(root.isInternalDir());
return rootFallbackLink;
}
@@ -520,14 +593,21 @@ Configuration getConfig() {
}
/**
- * Create Inode Tree from the specified mount-table specified in Config
- * @param config - the mount table keys are prefixed with
- * FsConstants.CONFIG_VIEWFS_PREFIX
- * @param viewName - the name of the mount table - if null use defaultMT name
- * @throws UnsupportedFileSystemException
- * @throws URISyntaxException
- * @throws FileAlreadyExistsException
- * @throws IOException
+ * Create Inode Tree from the specified mount-table specified in Config.
+ *
+ * @param config the mount table keys are prefixed with
+ * FsConstants.CONFIG_VIEWFS_PREFIX.
+ * @param viewName the name of the mount table
+ * if null use defaultMT name.
+ * @param theUri heUri.
+ * @param initingUriAsFallbackOnNoMounts initingUriAsFallbackOnNoMounts.
+ * @throws UnsupportedFileSystemException file system for uri is
+ * not found.
+ * @throws URISyntaxException if the URI does not have an authority
+ * it is badly formed.
+ * @throws FileAlreadyExistsException there is a file at the path specified
+ * or is discovered on one of its ancestors.
+ * @throws IOException raised on errors performing I/O.
*/
protected InodeTree(final Configuration config, final String viewName,
final URI theUri, boolean initingUriAsFallbackOnNoMounts)
@@ -538,6 +618,7 @@ protected InodeTree(final Configuration config, final String viewName,
mountTableName = ConfigUtil.getDefaultMountTableName(config);
}
homedirPrefix = ConfigUtil.getHomeDirValue(config, mountTableName);
+ isNestedMountPointSupported = ConfigUtil.isNestedMountPointSupported(config);
boolean isMergeSlashConfigured = false;
String mergeSlashTarget = null;
@@ -642,7 +723,8 @@ protected InodeTree(final Configuration config, final String viewName,
getRootDir().setInternalDirFs(getTargetFileSystem(getRootDir()));
getRootDir().setRoot(true);
INodeLink fallbackLink = null;
- for (LinkEntry le : linkEntries) {
+
+ for (LinkEntry le : getLinkEntries(linkEntries)) {
switch (le.getLinkType()) {
case SINGLE_FALLBACK:
if (fallbackLink != null) {
@@ -682,6 +764,32 @@ protected InodeTree(final Configuration config, final String viewName,
}
}
+ /**
+ * Get collection of linkEntry. Sort mount point based on alphabetical order of the src paths.
+ * The purpose is to group nested paths(shortest path always comes first) during INodeTree creation.
+ * E.g. /foo is nested with /foo/bar so an INodeDirLink will be created at /foo.
+ * @param linkEntries input linkEntries
+ * @return sorted linkEntries
+ */
+ private Collection getLinkEntries(List linkEntries) {
+ Set sortedLinkEntries = new TreeSet<>(new Comparator() {
+ @Override
+ public int compare(LinkEntry o1, LinkEntry o2) {
+ if (o1 == null) {
+ return -1;
+ }
+ if (o2 == null) {
+ return 1;
+ }
+ String src1 = o1.getSrc();
+ String src2= o2.getSrc();
+ return src1.compareTo(src2);
+ }
+ });
+ sortedLinkEntries.addAll(linkEntries);
+ return sortedLinkEntries;
+ }
+
private void checkMntEntryKeyEqualsTarget(
String mntEntryKey, String targetMntEntryKey) throws IOException {
if (!mntEntryKey.equals(targetMntEntryKey)) {
@@ -742,7 +850,7 @@ private LinkEntry buildLinkRegexEntry(
* If the input pathname leads to an internal mount-table entry then
* the target file system is one that represents the internal inode.
*/
- static class ResolveResult {
+ public static class ResolveResult {
final ResultKind kind;
final T targetFileSystem;
final String resolvedPath;
@@ -773,11 +881,11 @@ boolean isLastInternalDirLink() {
/**
* Resolve the pathname p relative to root InodeDir.
* @param p - input path
- * @param resolveLastComponent
+ * @param resolveLastComponent resolveLastComponent.
* @return ResolveResult which allows further resolution of the remaining path
- * @throws IOException
+ * @throws IOException raised on errors performing I/O.
*/
- ResolveResult resolve(final String p, final boolean resolveLastComponent)
+ public ResolveResult resolve(final String p, final boolean resolveLastComponent)
throws IOException {
ResolveResult resolveResult = null;
String[] path = breakIntoPathComponents(p);
@@ -795,7 +903,7 @@ ResolveResult resolve(final String p, final boolean resolveLastComponent)
* been linked to the root directory of a file system.
* The first non-slash path component should be name of the mount table.
*/
- if (root.isLink()) {
+ if (!root.isInternalDir()) {
Path remainingPath;
StringBuilder remainingPathStr = new StringBuilder();
// ignore first slash
@@ -818,10 +926,17 @@ ResolveResult resolve(final String p, final boolean resolveLastComponent)
}
int i;
+ INodeDirLink lastResolvedDirLink = null;
+ int lastResolvedDirLinkIndex = -1;
// ignore first slash
for (i = 1; i < path.length - (resolveLastComponent ? 0 : 1); i++) {
INode nextInode = curInode.resolveInternal(path[i]);
if (nextInode == null) {
+ // first resolve to dirlink for nested mount point
+ if (isNestedMountPointSupported && lastResolvedDirLink != null) {
+ return new ResolveResult(ResultKind.EXTERNAL_DIR, lastResolvedDirLink.getLink().getTargetFileSystem(),
+ lastResolvedDirLink.fullPath, getRemainingPath(path, i),true);
+ }
if (hasFallbackLink()) {
resolveResult = new ResolveResult(ResultKind.EXTERNAL_DIR,
getRootFallbackLink().getTargetFileSystem(), root.fullPath,
@@ -837,59 +952,67 @@ ResolveResult resolve(final String p, final boolean resolveLastComponent)
}
}
- if (nextInode.isLink()) {
+ if (!nextInode.isInternalDir()) {
final INodeLink link = (INodeLink) nextInode;
- final Path remainingPath;
- if (i >= path.length - 1) {
- remainingPath = SlashPath;
- } else {
- StringBuilder remainingPathStr =
- new StringBuilder("/" + path[i + 1]);
- for (int j = i + 2; j < path.length; ++j) {
- remainingPathStr.append('/').append(path[j]);
- }
- remainingPath = new Path(remainingPathStr.toString());
- }
+ final Path remainingPath = getRemainingPath(path, i + 1);
resolveResult = new ResolveResult(ResultKind.EXTERNAL_DIR,
link.getTargetFileSystem(), nextInode.fullPath, remainingPath,
true);
return resolveResult;
- } else if (nextInode.isInternalDir()) {
+ } else {
curInode = (INodeDir) nextInode;
+ // track last resolved nest mount point.
+ if (isNestedMountPointSupported && nextInode.isLink()) {
+ lastResolvedDirLink = (INodeDirLink) nextInode;
+ lastResolvedDirLinkIndex = i;
+ }
}
}
- // We have resolved to an internal dir in mount table.
Path remainingPath;
- if (resolveLastComponent) {
+ if (isNestedMountPointSupported && lastResolvedDirLink != null) {
+ remainingPath = getRemainingPath(path, lastResolvedDirLinkIndex + 1);
+ resolveResult = new ResolveResult(ResultKind.EXTERNAL_DIR, lastResolvedDirLink.getLink().getTargetFileSystem(),
+ lastResolvedDirLink.fullPath, remainingPath,true);
+ } else {
+ remainingPath = resolveLastComponent ? SlashPath : getRemainingPath(path, i);
+ resolveResult = new ResolveResult(ResultKind.INTERNAL_DIR, curInode.getInternalDirFs(),
+ curInode.fullPath, remainingPath, false);
+ }
+ return resolveResult;
+ }
+
+ /**
+ * Return remaining path from specified index to the end of the path array.
+ * @param path An array of path components split by slash
+ * @param startIndex the specified start index of the path array
+ * @return remaining path.
+ */
+ private Path getRemainingPath(String[] path, int startIndex) {
+ Path remainingPath;
+ if (startIndex >= path.length) {
remainingPath = SlashPath;
} else {
- // note we have taken care of when path is "/" above
- // for internal dirs rem-path does not start with / since the lookup
- // that follows will do a children.get(remaningPath) and will have to
- // strip-out the initial /
- StringBuilder remainingPathStr = new StringBuilder("/" + path[i]);
- for (int j = i + 1; j < path.length; ++j) {
- remainingPathStr.append('/').append(path[j]);
+ StringBuilder remainingPathStr = new StringBuilder();
+ for (int j = startIndex; j < path.length; j++) {
+ remainingPathStr.append("/").append(path[j]);
}
remainingPath = new Path(remainingPathStr.toString());
}
- resolveResult = new ResolveResult(ResultKind.INTERNAL_DIR,
- curInode.getInternalDirFs(), curInode.fullPath, remainingPath, false);
- return resolveResult;
+ return remainingPath;
}
/**
* Walk through all regex mount points to see
* whether the path match any regex expressions.
- * E.g. link: ^/user/(?\\w+) => s3://$user.apache.com/_${user}
+ * E.g. link: ^/user/(?<username>\\w+) => s3://$user.apache.com/_${user}
* srcPath: is /user/hadoop/dir1
* resolveLastComponent: true
* then return value is s3://hadoop.apache.com/_hadoop
*
- * @param srcPath
- * @param resolveLastComponent
- * @return
+ * @param srcPath srcPath.
+ * @param resolveLastComponent resolveLastComponent.
+ * @return ResolveResult.
*/
protected ResolveResult tryResolveInRegexMountpoint(final String srcPath,
final boolean resolveLastComponent) {
@@ -907,7 +1030,7 @@ protected ResolveResult tryResolveInRegexMountpoint(final String srcPath,
* Build resolve result.
* Here's an example
* Mountpoint: fs.viewfs.mounttable.mt
- * .linkRegex.replaceresolveddstpath:_:-#.^/user/(?\w+)
+ * .linkRegex.replaceresolveddstpath:_:-#.^/user/(??<username>\w+)
* Value: /targetTestRoot/$username
* Dir path to test:
* viewfs://mt/user/hadoop_user1/hadoop_dir1
@@ -916,6 +1039,10 @@ protected ResolveResult tryResolveInRegexMountpoint(final String srcPath,
* targetOfResolvedPathStr: /targetTestRoot/hadoop-user1
* remainingPath: /hadoop_dir1
*
+ * @param resultKind resultKind.
+ * @param resolvedPathStr resolvedPathStr.
+ * @param targetOfResolvedPathStr targetOfResolvedPathStr.
+ * @param remainingPath remainingPath.
* @return targetFileSystem or null on exceptions.
*/
protected ResolveResult buildResolveResultForRegexMountPoint(
@@ -957,7 +1084,7 @@ public List