diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/http/AbstractHttpFileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/http/AbstractHttpFileSystem.java index fa0b2cf6c314e..bfcd9a7db2e89 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/http/AbstractHttpFileSystem.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/http/AbstractHttpFileSystem.java @@ -19,6 +19,7 @@ package org.apache.hadoop.fs.http; +import com.google.common.base.Preconditions; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FSDataInputStream; import org.apache.hadoop.fs.FSDataOutputStream; @@ -39,13 +40,19 @@ abstract class AbstractHttpFileSystem extends FileSystem { private static final long DEFAULT_BLOCK_SIZE = 4096; private static final Path WORKING_DIR = new Path("/"); + private static final String CONF_HTTP_READ_TIMEOUT = "yarn.nodemanager.localizer.http.read.timeout.ms"; + private static final String CONF_HTTP_CONNECT_TIMEOUT = "yarn.nodemanager.localizer.http.connect.timeout.ms"; + private static final Integer DEFAULT_HTTP_READ_TIMEOUT_IN_MILLIS = 5000; + private static final Integer DEFAULT_HTTP_CONNECT_TIMEOUT_IN_MILLIS = 5000; private URI uri; + private Configuration conf; @Override public void initialize(URI name, Configuration conf) throws IOException { super.initialize(name, conf); this.uri = name; + this.conf = conf; } public abstract String getScheme(); @@ -57,7 +64,14 @@ public URI getUri() { @Override public FSDataInputStream open(Path path, int bufferSize) throws IOException { + int httpConnectTimeoutInMillis = conf.getInt(CONF_HTTP_CONNECT_TIMEOUT, DEFAULT_HTTP_CONNECT_TIMEOUT_IN_MILLIS); + int httpReadTimeoutInMillis = conf.getInt(CONF_HTTP_READ_TIMEOUT, DEFAULT_HTTP_READ_TIMEOUT_IN_MILLIS); + Preconditions.checkState(httpConnectTimeoutInMillis > 0, "Http connection timeout has to greater than zero."); + Preconditions.checkState(httpReadTimeoutInMillis > 0, "Http read timeout has to greater than zero."); + URLConnection conn = path.toUri().toURL().openConnection(); + conn.setConnectTimeout(httpConnectTimeoutInMillis); + conn.setReadTimeout(httpReadTimeoutInMillis); InputStream in = conn.getInputStream(); return new FSDataInputStream(new HttpDataInputStream(in)); } diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/http/TestHttpFileSystem.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/http/TestHttpFileSystem.java index 0902c04c79b66..64013f252b25d 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/http/TestHttpFileSystem.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/http/TestHttpFileSystem.java @@ -26,6 +26,7 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.io.IOUtils; import org.junit.Test; +import org.mockito.Mockito; import java.io.IOException; import java.io.InputStream; @@ -64,4 +65,38 @@ public void testHttpFileSystem() throws IOException, URISyntaxException, assertEquals("/foo", req.getPath()); } } + + @Test(expected = IllegalStateException.class) + public void testHttpFileSystemWithNegativeHttpConnectTimeout() throws Exception { + URI mockURI = new URI("dummyURI"); + + Configuration conf = new Configuration(false); + conf.set("fs.http.impl", HttpFileSystem.class.getCanonicalName()); + conf.set("yarn.nodemanager.localizer.http.connect.timeout.ms", "-1"); + conf.set("yarn.nodemanager.localizer.http.read.timeout.ms", "1000"); + + Path mockPath = Mockito.mock(Path.class); + + HttpFileSystem httpFileSystem = new HttpFileSystem(); + httpFileSystem.initialize(mockURI, conf); + + httpFileSystem.open(mockPath, 1000); + } + + @Test(expected = IllegalStateException.class) + public void testHttpFileSystemWithNegativeHttpReadTimeout() throws Exception { + URI mockURI = new URI("dummyURI"); + + Configuration conf = new Configuration(false); + conf.set("fs.http.impl", HttpFileSystem.class.getCanonicalName()); + conf.set("yarn.nodemanager.localizer.http.connect.timeout.ms", "1000"); + conf.set("yarn.nodemanager.localizer.http.read.timeout.ms", "-1"); + + Path mockPath = Mockito.mock(Path.class); + + HttpFileSystem httpFileSystem = new HttpFileSystem(); + httpFileSystem.initialize(mockURI, conf); + + httpFileSystem.open(mockPath, 1000); + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml index a00b5d6ba4e70..d19540efc03ea 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml @@ -1183,6 +1183,18 @@ ${yarn.nodemanager.hostname}:8040 + + Http read timeout in milliseconds to use for the localization requests. Setting this configuration is necessary only if you are using either HttpFileSystem or HttpsFileSystem for localization. + yarn.nodemanager.localizer.http.read.timeout.ms + 5000 + + + + Http connect timeout in milliseconds to use for the localization requests. Setting this configuration is necessary only if you are using either HttpFileSystem or HttpsFileSystem for localization. + yarn.nodemanager.localizer.http.connect.timeout.ms + 5000 + + Address where the collector service IPC is.