diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/http/AbstractHttpFileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/http/AbstractHttpFileSystem.java
index fa0b2cf6c314e..bfcd9a7db2e89 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/http/AbstractHttpFileSystem.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/http/AbstractHttpFileSystem.java
@@ -19,6 +19,7 @@
package org.apache.hadoop.fs.http;
+import com.google.common.base.Preconditions;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FSDataOutputStream;
@@ -39,13 +40,19 @@
abstract class AbstractHttpFileSystem extends FileSystem {
private static final long DEFAULT_BLOCK_SIZE = 4096;
private static final Path WORKING_DIR = new Path("/");
+ private static final String CONF_HTTP_READ_TIMEOUT = "yarn.nodemanager.localizer.http.read.timeout.ms";
+ private static final String CONF_HTTP_CONNECT_TIMEOUT = "yarn.nodemanager.localizer.http.connect.timeout.ms";
+ private static final Integer DEFAULT_HTTP_READ_TIMEOUT_IN_MILLIS = 5000;
+ private static final Integer DEFAULT_HTTP_CONNECT_TIMEOUT_IN_MILLIS = 5000;
private URI uri;
+ private Configuration conf;
@Override
public void initialize(URI name, Configuration conf) throws IOException {
super.initialize(name, conf);
this.uri = name;
+ this.conf = conf;
}
public abstract String getScheme();
@@ -57,7 +64,14 @@ public URI getUri() {
@Override
public FSDataInputStream open(Path path, int bufferSize) throws IOException {
+ int httpConnectTimeoutInMillis = conf.getInt(CONF_HTTP_CONNECT_TIMEOUT, DEFAULT_HTTP_CONNECT_TIMEOUT_IN_MILLIS);
+ int httpReadTimeoutInMillis = conf.getInt(CONF_HTTP_READ_TIMEOUT, DEFAULT_HTTP_READ_TIMEOUT_IN_MILLIS);
+ Preconditions.checkState(httpConnectTimeoutInMillis > 0, "Http connection timeout has to greater than zero.");
+ Preconditions.checkState(httpReadTimeoutInMillis > 0, "Http read timeout has to greater than zero.");
+
URLConnection conn = path.toUri().toURL().openConnection();
+ conn.setConnectTimeout(httpConnectTimeoutInMillis);
+ conn.setReadTimeout(httpReadTimeoutInMillis);
InputStream in = conn.getInputStream();
return new FSDataInputStream(new HttpDataInputStream(in));
}
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/http/TestHttpFileSystem.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/http/TestHttpFileSystem.java
index 0902c04c79b66..64013f252b25d 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/http/TestHttpFileSystem.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/http/TestHttpFileSystem.java
@@ -26,6 +26,7 @@
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IOUtils;
import org.junit.Test;
+import org.mockito.Mockito;
import java.io.IOException;
import java.io.InputStream;
@@ -64,4 +65,38 @@ public void testHttpFileSystem() throws IOException, URISyntaxException,
assertEquals("/foo", req.getPath());
}
}
+
+ @Test(expected = IllegalStateException.class)
+ public void testHttpFileSystemWithNegativeHttpConnectTimeout() throws Exception {
+ URI mockURI = new URI("dummyURI");
+
+ Configuration conf = new Configuration(false);
+ conf.set("fs.http.impl", HttpFileSystem.class.getCanonicalName());
+ conf.set("yarn.nodemanager.localizer.http.connect.timeout.ms", "-1");
+ conf.set("yarn.nodemanager.localizer.http.read.timeout.ms", "1000");
+
+ Path mockPath = Mockito.mock(Path.class);
+
+ HttpFileSystem httpFileSystem = new HttpFileSystem();
+ httpFileSystem.initialize(mockURI, conf);
+
+ httpFileSystem.open(mockPath, 1000);
+ }
+
+ @Test(expected = IllegalStateException.class)
+ public void testHttpFileSystemWithNegativeHttpReadTimeout() throws Exception {
+ URI mockURI = new URI("dummyURI");
+
+ Configuration conf = new Configuration(false);
+ conf.set("fs.http.impl", HttpFileSystem.class.getCanonicalName());
+ conf.set("yarn.nodemanager.localizer.http.connect.timeout.ms", "1000");
+ conf.set("yarn.nodemanager.localizer.http.read.timeout.ms", "-1");
+
+ Path mockPath = Mockito.mock(Path.class);
+
+ HttpFileSystem httpFileSystem = new HttpFileSystem();
+ httpFileSystem.initialize(mockURI, conf);
+
+ httpFileSystem.open(mockPath, 1000);
+ }
}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
index a00b5d6ba4e70..d19540efc03ea 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
@@ -1183,6 +1183,18 @@
${yarn.nodemanager.hostname}:8040
+
+ Http read timeout in milliseconds to use for the localization requests. Setting this configuration is necessary only if you are using either HttpFileSystem or HttpsFileSystem for localization.
+ yarn.nodemanager.localizer.http.read.timeout.ms
+ 5000
+
+
+
+ Http connect timeout in milliseconds to use for the localization requests. Setting this configuration is necessary only if you are using either HttpFileSystem or HttpsFileSystem for localization.
+ yarn.nodemanager.localizer.http.connect.timeout.ms
+ 5000
+
+
Address where the collector service IPC is.