diff --git a/.asf.yaml b/.asf.yaml index 91483dfed336c..e1c11790c8bfb 100644 --- a/.asf.yaml +++ b/.asf.yaml @@ -22,4 +22,4 @@ notifications: commits: common-commits@hadoop.apache.org issues: common-issues@hadoop.apache.org pullrequests: common-issues@hadoop.apache.org - jira_options: link label worklog \ No newline at end of file + jira_options: comment link label \ No newline at end of file diff --git a/LICENSE-binary b/LICENSE-binary index 499485263906a..d9762b14c34a5 100644 --- a/LICENSE-binary +++ b/LICENSE-binary @@ -215,15 +215,15 @@ com.aliyun:aliyun-java-sdk-ecs:4.2.0 com.aliyun:aliyun-java-sdk-ram:3.0.0 com.aliyun:aliyun-java-sdk-sts:3.0.0 com.aliyun.oss:aliyun-sdk-oss:3.13.2 -com.amazonaws:aws-java-sdk-bundle:1.11.901 +com.amazonaws:aws-java-sdk-bundle:1.12.262 com.cedarsoftware:java-util:1.9.0 com.cedarsoftware:json-io:2.5.1 -com.fasterxml.jackson.core:jackson-annotations:2.13.2 -com.fasterxml.jackson.core:jackson-core:2.13.2 -com.fasterxml.jackson.core:jackson-databind:2.13.2.2 -com.fasterxml.jackson.jaxrs:jackson-jaxrs-base:2.13.2 -com.fasterxml.jackson.jaxrs:jackson-jaxrs-json-provider:2.13.2 -com.fasterxml.jackson.module:jackson-module-jaxb-annotations:2.13.2 +com.fasterxml.jackson.core:jackson-annotations:2.12.7 +com.fasterxml.jackson.core:jackson-core:2.12.7 +com.fasterxml.jackson.core:jackson-databind:2.12.7 +com.fasterxml.jackson.jaxrs:jackson-jaxrs-base:2.12.7 +com.fasterxml.jackson.jaxrs:jackson-jaxrs-json-provider:2.12.7 +com.fasterxml.jackson.module:jackson-module-jaxb-annotations:2.12.7 com.fasterxml.uuid:java-uuid-generator:3.1.4 com.fasterxml.woodstox:woodstox-core:5.3.0 com.github.davidmoten:rxjava-extras:0.8.0.17 @@ -251,7 +251,7 @@ commons-collections:commons-collections:3.2.2 commons-daemon:commons-daemon:1.0.13 commons-io:commons-io:2.8.0 commons-logging:commons-logging:1.1.3 -commons-net:commons-net:3.6 +commons-net:commons-net:3.8.0 de.ruedigermoeller:fst:2.50 io.grpc:grpc-api:1.26.0 io.grpc:grpc-context:1.26.0 @@ -261,17 +261,36 @@ io.grpc:grpc-protobuf:1.26.0 io.grpc:grpc-protobuf-lite:1.26.0 io.grpc:grpc-stub:1.26.0 io.netty:netty:3.10.6.Final -io.netty:netty-all:4.1.42.Final -io.netty:netty-buffer:4.1.27.Final -io.netty:netty-codec:4.1.27.Final -io.netty:netty-codec-http:4.1.27.Final -io.netty:netty-codec-http2:4.1.27.Final -io.netty:netty-codec-socks:4.1.27.Final -io.netty:netty-common:4.1.27.Final -io.netty:netty-handler:4.1.27.Final -io.netty:netty-handler-proxy:4.1.27.Final -io.netty:netty-resolver:4.1.27.Final -io.netty:netty-transport:4.1.27.Final +io.netty:netty-all:4.1.77.Final +io.netty:netty-buffer:4.1.77.Final +io.netty:netty-codec:4.1.77.Final +io.netty:netty-codec-dns:4.1.77.Final +io.netty:netty-codec-haproxy:4.1.77.Final +io.netty:netty-codec-http:4.1.77.Final +io.netty:netty-codec-http2:4.1.77.Final +io.netty:netty-codec-memcache:4.1.77.Final +io.netty:netty-codec-mqtt:4.1.77.Final +io.netty:netty-codec-redis:4.1.77.Final +io.netty:netty-codec-smtp:4.1.77.Final +io.netty:netty-codec-socks:4.1.77.Final +io.netty:netty-codec-stomp:4.1.77.Final +io.netty:netty-codec-xml:4.1.77.Final +io.netty:netty-common:4.1.77.Final +io.netty:netty-handler:4.1.77.Final +io.netty:netty-handler-proxy:4.1.77.Final +io.netty:netty-resolver:4.1.77.Final +io.netty:netty-resolver-dns:4.1.77.Final +io.netty:netty-transport:4.1.77.Final +io.netty:netty-transport-rxtx:4.1.77.Final +io.netty:netty-transport-sctp:4.1.77.Final +io.netty:netty-transport-udt:4.1.77.Final +io.netty:netty-transport-classes-epoll:4.1.77.Final +io.netty:netty-transport-native-unix-common:4.1.77.Final +io.netty:netty-transport-classes-kqueue:4.1.77.Final +io.netty:netty-resolver-dns-classes-macos:4.1.77.Final +io.netty:netty-transport-native-epoll:4.1.77.Final +io.netty:netty-transport-native-kqueue:4.1.77.Final +io.netty:netty-resolver-dns-native-macos:4.1.77.Final io.opencensus:opencensus-api:0.12.3 io.opencensus:opencensus-contrib-grpc-metrics:0.12.3 io.reactivex:rxjava:1.3.8 @@ -290,7 +309,7 @@ org.apache.commons:commons-configuration2:2.1.1 org.apache.commons:commons-csv:1.0 org.apache.commons:commons-digester:1.8.1 org.apache.commons:commons-lang3:3.12.0 -org.apache.commons:commons-math3:3.1.1 +org.apache.commons:commons-math3:3.6.1 org.apache.commons:commons-text:1.4 org.apache.commons:commons-validator:1.6 org.apache.curator:curator-client:5.2.0 @@ -306,44 +325,44 @@ org.apache.htrace:htrace-core4:4.1.0-incubating org.apache.httpcomponents:httpclient:4.5.6 org.apache.httpcomponents:httpcore:4.4.10 org.apache.kafka:kafka-clients:2.8.1 -org.apache.kerby:kerb-admin:1.0.1 -org.apache.kerby:kerb-client:1.0.1 -org.apache.kerby:kerb-common:1.0.1 -org.apache.kerby:kerb-core:1.0.1 -org.apache.kerby:kerb-crypto:1.0.1 -org.apache.kerby:kerb-identity:1.0.1 -org.apache.kerby:kerb-server:1.0.1 -org.apache.kerby:kerb-simplekdc:1.0.1 -org.apache.kerby:kerb-util:1.0.1 -org.apache.kerby:kerby-asn1:1.0.1 -org.apache.kerby:kerby-config:1.0.1 -org.apache.kerby:kerby-pkix:1.0.1 -org.apache.kerby:kerby-util:1.0.1 -org.apache.kerby:kerby-xdr:1.0.1 -org.apache.kerby:token-provider:1.0.1 +org.apache.kerby:kerb-admin:2.0.2 +org.apache.kerby:kerb-client:2.0.2 +org.apache.kerby:kerb-common:2.0.2 +org.apache.kerby:kerb-core:2.0.2 +org.apache.kerby:kerb-crypto:2.0.2 +org.apache.kerby:kerb-identity:2.0.2 +org.apache.kerby:kerb-server:2.0.2 +org.apache.kerby:kerb-simplekdc:2.0.2 +org.apache.kerby:kerb-util:2.0.2 +org.apache.kerby:kerby-asn1:2.0.2 +org.apache.kerby:kerby-config:2.0.2 +org.apache.kerby:kerby-pkix:2.0.2 +org.apache.kerby:kerby-util:2.0.2 +org.apache.kerby:kerby-xdr:2.0.2 +org.apache.kerby:token-provider:2.0.2 org.apache.solr:solr-solrj:8.8.2 org.apache.yetus:audience-annotations:0.5.0 org.apache.zookeeper:zookeeper:3.6.3 org.codehaus.jettison:jettison:1.1 -org.eclipse.jetty:jetty-annotations:9.4.44.v20210927 -org.eclipse.jetty:jetty-http:9.4.44.v20210927 -org.eclipse.jetty:jetty-io:9.4.44.v20210927 -org.eclipse.jetty:jetty-jndi:9.4.44.v20210927 -org.eclipse.jetty:jetty-plus:9.4.44.v20210927 -org.eclipse.jetty:jetty-security:9.4.44.v20210927 -org.eclipse.jetty:jetty-server:9.4.44.v20210927 -org.eclipse.jetty:jetty-servlet:9.4.44.v20210927 -org.eclipse.jetty:jetty-util:9.4.44.v20210927 -org.eclipse.jetty:jetty-util-ajax:9.4.44.v20210927 -org.eclipse.jetty:jetty-webapp:9.4.44.v20210927 -org.eclipse.jetty:jetty-xml:9.4.44.v20210927 -org.eclipse.jetty.websocket:javax-websocket-client-impl:9.4.44.v20210927 -org.eclipse.jetty.websocket:javax-websocket-server-impl:9.4.44.v20210927 +org.eclipse.jetty:jetty-annotations:9.4.48.v20220622 +org.eclipse.jetty:jetty-http:9.4.48.v20220622 +org.eclipse.jetty:jetty-io:9.4.48.v20220622 +org.eclipse.jetty:jetty-jndi:9.4.48.v20220622 +org.eclipse.jetty:jetty-plus:9.4.48.v20220622 +org.eclipse.jetty:jetty-security:9.4.48.v20220622 +org.eclipse.jetty:jetty-server:9.4.48.v20220622 +org.eclipse.jetty:jetty-servlet:9.4.48.v20220622 +org.eclipse.jetty:jetty-util:9.4.48.v20220622 +org.eclipse.jetty:jetty-util-ajax:9.4.48.v20220622 +org.eclipse.jetty:jetty-webapp:9.4.48.v20220622 +org.eclipse.jetty:jetty-xml:9.4.48.v20220622 +org.eclipse.jetty.websocket:javax-websocket-client-impl:9.4.48.v20220622 +org.eclipse.jetty.websocket:javax-websocket-server-impl:9.4.48.v20220622 org.ehcache:ehcache:3.3.1 org.lz4:lz4-java:1.7.1 org.objenesis:objenesis:2.6 org.xerial.snappy:snappy-java:1.0.5 -org.yaml:snakeyaml:1.16: +org.yaml:snakeyaml:1.31: org.wildfly.openssl:wildfly-openssl:1.0.7.Final @@ -416,7 +435,7 @@ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanage bootstrap v3.3.6 broccoli-asset-rev v2.4.2 broccoli-funnel v1.0.1 -datatables v1.10.8 +datatables v1.10.19 em-helpers v0.5.13 em-table v0.1.6 ember v2.2.0 @@ -491,7 +510,6 @@ javax.annotation:javax.annotation-api:1.3.2 javax.servlet:javax.servlet-api:3.1.0 javax.servlet.jsp:jsp-api:2.1 javax.websocket:javax.websocket-api:1.0 -javax.ws.rs:javax.ws.rs-api:2.1.1 javax.ws.rs:jsr311-api:1.1.1 javax.xml.bind:jaxb-api:2.2.11 diff --git a/hadoop-client-modules/hadoop-client-api/pom.xml b/hadoop-client-modules/hadoop-client-api/pom.xml index 82f2f8a778087..b4b81011eb517 100644 --- a/hadoop-client-modules/hadoop-client-api/pom.xml +++ b/hadoop-client-modules/hadoop-client-api/pom.xml @@ -98,13 +98,6 @@ true true - - - org.apache.hadoop - hadoop-maven-plugins - ${project.version} - - package @@ -254,8 +247,7 @@ - - + NOTICE.txt diff --git a/hadoop-client-modules/hadoop-client-minicluster/pom.xml b/hadoop-client-modules/hadoop-client-minicluster/pom.xml index 4c8900dc2af0d..208345d5f5a53 100644 --- a/hadoop-client-modules/hadoop-client-minicluster/pom.xml +++ b/hadoop-client-modules/hadoop-client-minicluster/pom.xml @@ -671,13 +671,6 @@ org.apache.maven.plugins maven-shade-plugin - - - org.apache.hadoop - hadoop-maven-plugins - ${project.version} - - package @@ -704,7 +697,6 @@ org.bouncycastle:* org.xerial.snappy:* - javax.ws.rs:javax.ws.rs-api @@ -1053,8 +1045,7 @@ - - + diff --git a/hadoop-client-modules/hadoop-client-runtime/pom.xml b/hadoop-client-modules/hadoop-client-runtime/pom.xml index 98756c2439544..b2bd7a4fc43c2 100644 --- a/hadoop-client-modules/hadoop-client-runtime/pom.xml +++ b/hadoop-client-modules/hadoop-client-runtime/pom.xml @@ -128,13 +128,6 @@ org.apache.maven.plugins maven-shade-plugin - - - org.apache.hadoop - hadoop-maven-plugins - ${project.version} - - package @@ -163,7 +156,6 @@ org.bouncycastle:* org.xerial.snappy:* - javax.ws.rs:javax.ws.rs-api @@ -398,8 +390,7 @@ --> - - + diff --git a/hadoop-common-project/hadoop-common/dev-support/jdiff/Apache_Hadoop_Common_3.2.4.xml b/hadoop-common-project/hadoop-common/dev-support/jdiff/Apache_Hadoop_Common_3.2.4.xml new file mode 100644 index 0000000000000..10a4f0d5f16e5 --- /dev/null +++ b/hadoop-common-project/hadoop-common/dev-support/jdiff/Apache_Hadoop_Common_3.2.4.xml @@ -0,0 +1,35426 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + UnsupportedOperationException + + If a key is deprecated in favor of multiple keys, they are all treated as + aliases of each other, and setting any one of them resets all the others + to the new value. + + If you have multiple deprecation entries to add, it is more efficient to + use #addDeprecations(DeprecationDelta[] deltas) instead. + + @param key + @param newKeys + @param customMessage + @deprecated use {@link #addDeprecation(String key, String newKey, + String customMessage)} instead]]> + + + + + + + + UnsupportedOperationException + + If you have multiple deprecation entries to add, it is more efficient to + use #addDeprecations(DeprecationDelta[] deltas) instead. + + @param key + @param newKey + @param customMessage]]> + + + + + + + UnsupportedOperationException + + If a key is deprecated in favor of multiple keys, they are all treated as + aliases of each other, and setting any one of them resets all the others + to the new value. + + If you have multiple deprecation entries to add, it is more efficient to + use #addDeprecations(DeprecationDelta[] deltas) instead. + + @param key Key that is to be deprecated + @param newKeys list of keys that take up the values of deprecated key + @deprecated use {@link #addDeprecation(String key, String newKey)} instead]]> + + + + + + + UnsupportedOperationException + + If you have multiple deprecation entries to add, it is more efficient to + use #addDeprecations(DeprecationDelta[] deltas) instead. + + @param key Key that is to be deprecated + @param newKey key that takes up the value of deprecated key]]> + + + + + + key is deprecated. + + @param key the parameter which is to be checked for deprecation + @return true if the key is deprecated and + false otherwise.]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + final. + + @param name resource to be added, the classpath is examined for a file + with that name.]]> + + + + + + + + + + final. + + @param url url of the resource to be added, the local filesystem is + examined directly to find the resource, without referring to + the classpath.]]> + + + + + + + + + + final. + + @param file file-path of resource to be added, the local filesystem is + examined directly to find the resource, without referring to + the classpath.]]> + + + + + + + + + + final. + + WARNING: The contents of the InputStream will be cached, by this method. + So use this sparingly because it does increase the memory consumption. + + @param in InputStream to deserialize the object from. In will be read from + when a get or set is called next. After it is read the stream will be + closed.]]> + + + + + + + + + + + final. + + @param in InputStream to deserialize the object from. + @param name the name of the resource because InputStream.toString is not + very descriptive some times.]]> + + + + + + + + + + + final. + + @param conf Configuration object from which to load properties]]> + + + + + + + + + + + name property, null if + no such property exists. If the key is deprecated, it returns the value of + the first key which replaces the deprecated key and is not null. + + Values are processed for variable expansion + before being returned. + + @param name the property name, will be trimmed before get value. + @return the value of the name or its replacing property, + or null if no such property exists.]]> + + + + + + + + + + + + + + + name property, but only for + names which have no valid value, usually non-existent or commented + out in XML. + + @param name the property name + @return true if the property name exists without value]]> + + + + + + name property as a trimmed String, + null if no such property exists. + If the key is deprecated, it returns the value of + the first key which replaces the deprecated key and is not null + + Values are processed for variable expansion + before being returned. + + @param name the property name. + @return the value of the name or its replacing property, + or null if no such property exists.]]> + + + + + + + name property as a trimmed String, + defaultValue if no such property exists. + See @{Configuration#getTrimmed} for more details. + + @param name the property name. + @param defaultValue the property default value. + @return the value of the name or defaultValue + if it is not set.]]> + + + + + + name property, without doing + variable expansion.If the key is + deprecated, it returns the value of the first key which replaces + the deprecated key and is not null. + + @param name the property name. + @return the value of the name property or + its replacing property and null if no such property exists.]]> + + + + + + + value of the name property. If + name is deprecated or there is a deprecated name associated to it, + it sets the value to both names. Name will be trimmed before put into + configuration. + + @param name property name. + @param value property value.]]> + + + + + + + + value of the name property. If + name is deprecated, it also sets the value to + the keys that replace the deprecated key. Name will be trimmed before put + into configuration. + + @param name property name. + @param value property value. + @param source the place that this configuration value came from + (For debugging). + @throws IllegalArgumentException when the value or name is null.]]> + + + + + + + + + + + + + + + + + + + + name. If the key is deprecated, + it returns the value of the first key which replaces the deprecated key + and is not null. + If no such property exists, + then defaultValue is returned. + + @param name property name, will be trimmed before get value. + @param defaultValue default value. + @return property value, or defaultValue if the property + doesn't exist.]]> + + + + + + + name property as an int. + + If no such property exists, the provided default value is returned, + or if the specified value is not a valid int, + then an error is thrown. + + @param name property name. + @param defaultValue default value. + @throws NumberFormatException when the value is invalid + @return property value as an int, + or defaultValue.]]> + + + + + + name property as a set of comma-delimited + int values. + + If no such property exists, an empty array is returned. + + @param name property name + @return property value interpreted as an array of comma-delimited + int values]]> + + + + + + + name property to an int. + + @param name property name. + @param value int value of the property.]]> + + + + + + + name property as a long. + If no such property exists, the provided default value is returned, + or if the specified value is not a valid long, + then an error is thrown. + + @param name property name. + @param defaultValue default value. + @throws NumberFormatException when the value is invalid + @return property value as a long, + or defaultValue.]]> + + + + + + + name property as a long or + human readable format. If no such property exists, the provided default + value is returned, or if the specified value is not a valid + long or human readable format, then an error is thrown. You + can use the following suffix (case insensitive): k(kilo), m(mega), g(giga), + t(tera), p(peta), e(exa) + + @param name property name. + @param defaultValue default value. + @throws NumberFormatException when the value is invalid + @return property value as a long, + or defaultValue.]]> + + + + + + + name property to a long. + + @param name property name. + @param value long value of the property.]]> + + + + + + + name property as a float. + If no such property exists, the provided default value is returned, + or if the specified value is not a valid float, + then an error is thrown. + + @param name property name. + @param defaultValue default value. + @throws NumberFormatException when the value is invalid + @return property value as a float, + or defaultValue.]]> + + + + + + + name property to a float. + + @param name property name. + @param value property value.]]> + + + + + + + name property as a double. + If no such property exists, the provided default value is returned, + or if the specified value is not a valid double, + then an error is thrown. + + @param name property name. + @param defaultValue default value. + @throws NumberFormatException when the value is invalid + @return property value as a double, + or defaultValue.]]> + + + + + + + name property to a double. + + @param name property name. + @param value property value.]]> + + + + + + + name property as a boolean. + If no such property is specified, or if the specified value is not a valid + boolean, then defaultValue is returned. + + @param name property name. + @param defaultValue default value. + @return property value as a boolean, + or defaultValue.]]> + + + + + + + name property to a boolean. + + @param name property name. + @param value boolean value of the property.]]> + + + + + + + + + + + + + + name property to the given type. This + is equivalent to set(<name>, value.toString()). + @param name property name + @param value new value]]> + + + + + + + + + + + + + + + name to the given time duration. This + is equivalent to set(<name>, value + <time suffix>). + @param name Property name + @param value Time duration + @param unit Unit of time]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + name property as a Pattern. + If no such property is specified, or if the specified value is not a valid + Pattern, then DefaultValue is returned. + Note that the returned value is NOT trimmed by this method. + + @param name property name + @param defaultValue default value + @return property value as a compiled Pattern, or defaultValue]]> + + + + + + + Pattern. + If the pattern is passed as null, sets the empty pattern which results in + further calls to getPattern(...) returning the default value. + + @param name property name + @param pattern new value]]> + + + + + + + + + + + + + + + + + + + name property as + a collection of Strings. + If no such property is specified then empty collection is returned. +

+ This is an optimized version of {@link #getStrings(String)} + + @param name property name. + @return property value as a collection of Strings.]]> + + + + + + name property as + an array of Strings. + If no such property is specified then null is returned. + + @param name property name. + @return property value as an array of Strings, + or null.]]> + + + + + + + name property as + an array of Strings. + If no such property is specified then default value is returned. + + @param name property name. + @param defaultValue The default value + @return property value as an array of Strings, + or default value.]]> + + + + + + name property as + a collection of Strings, trimmed of the leading and trailing whitespace. + If no such property is specified then empty Collection is returned. + + @param name property name. + @return property value as a collection of Strings, or empty Collection]]> + + + + + + name property as + an array of Strings, trimmed of the leading and trailing whitespace. + If no such property is specified then an empty array is returned. + + @param name property name. + @return property value as an array of trimmed Strings, + or empty array.]]> + + + + + + + name property as + an array of Strings, trimmed of the leading and trailing whitespace. + If no such property is specified then default value is returned. + + @param name property name. + @param defaultValue The default value + @return property value as an array of trimmed Strings, + or default value.]]> + + + + + + + name property as + as comma delimited values. + + @param name property name. + @param values The values]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + hostProperty as a + InetSocketAddress. If hostProperty is + null, addressProperty will be used. This + is useful for cases where we want to differentiate between host + bind address and address clients should use to establish connection. + + @param hostProperty bind host property name. + @param addressProperty address property name. + @param defaultAddressValue the default value + @param defaultPort the default port + @return InetSocketAddress]]> + + + + + + + + name property as a + InetSocketAddress. + @param name property name. + @param defaultAddress the default value + @param defaultPort the default port + @return InetSocketAddress]]> + + + + + + + name property as + a host:port.]]> + + + + + + + + + name property as a host:port. The wildcard + address is replaced with the local host's address. If the host and address + properties are configured the host component of the address will be combined + with the port component of the addr to generate the address. This is to allow + optional control over which host name is used in multi-home bind-host + cases where a host can have multiple names + @param hostProperty the bind-host configuration name + @param addressProperty the service address configuration name + @param defaultAddressValue the service default address configuration value + @param addr InetSocketAddress of the service listener + @return InetSocketAddress for clients to connect]]> + + + + + + + name property as a host:port. The wildcard + address is replaced with the local host's address. + @param name property name. + @param addr InetSocketAddress of a listener to store in the given property + @return InetSocketAddress for clients to connect]]> + + + + + + + + + + + + + + + + + + + + name property + as an array of Class. + The value of the property specifies a list of comma separated class names. + If no such property is specified, then defaultValue is + returned. + + @param name the property name. + @param defaultValue default value. + @return property value as a Class[], + or defaultValue.]]> + + + + + + + name property as a Class. + If no such property is specified, then defaultValue is + returned. + + @param name the class name. + @param defaultValue default value. + @return property value as a Class, + or defaultValue.]]> + + + + + + + + name property as a Class + implementing the interface specified by xface. + + If no such property is specified, then defaultValue is + returned. + + An exception is thrown if the returned class does not implement the named + interface. + + @param name the class name. + @param defaultValue default value. + @param xface the interface implemented by the named class. + @return property value as a Class, + or defaultValue.]]> + + + + + + + name property as a List + of objects implementing the interface specified by xface. + + An exception is thrown if any of the classes does not exist, or if it does + not implement the named interface. + + @param name the property name. + @param xface the interface implemented by the classes named by + name. + @return a List of objects implementing xface.]]> + + + + + + + + name property to the name of a + theClass implementing the given interface xface. + + An exception is thrown if theClass does not implement the + interface xface. + + @param name property name. + @param theClass property value. + @param xface the interface implemented by the named class.]]> + + + + + + + + dirsProp with + the given path. If dirsProp contains multiple directories, + then one is chosen based on path's hash code. If the selected + directory does not exist, an attempt is made to create it. + + @param dirsProp directory in which to locate the file. + @param path file-path. + @return local file under the directory with the given path.]]> + + + + + + + + dirsProp with + the given path. If dirsProp contains multiple directories, + then one is chosen based on path's hash code. If the selected + directory does not exist, an attempt is made to create it. + + @param dirsProp directory in which to locate the file. + @param path file-path. + @return local file under the directory with the given path.]]> + + + + + + + + + + + + name. + + @param name configuration resource name. + @return an input stream attached to the resource.]]> + + + + + + name. + + @param name configuration resource name. + @return a reader attached to the resource.]]> + + + + + + + + + + + + + + + + + + + + + + String + key-value pairs in the configuration. + + @return an iterator over the entries.]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + When property name is not empty and the property exists in the + configuration, this method writes the property and its attributes + to the {@link Writer}. + +

+ +

  • + When property name is null or empty, this method writes all the + configuration properties and their attributes to the {@link Writer}. +
    • +

    + +

  • + When property name is not empty but the property doesn't exist in + the configuration, this method throws an {@link IllegalArgumentException}. +
    • +

    + @param out the writer to write to.]]> + + + + + + + + + + When propertyName is not empty, and the property exists + in the configuration, the format of the output would be, + 

    +  {
+    "property": {
+      "key" : "key1",
+      "value" : "value1",
+      "isFinal" : "key1.isFinal",
+      "resource" : "key1.resource"
+    }
+  }
+
    + + +
  • + When propertyName is null or empty, it behaves same as + {@link #dumpConfiguration(Configuration, Writer)}, the + output would be, + 
    +  { "properties" :
+      [ { key : "key1",
+          value : "value1",
+          isFinal : "key1.isFinal",
+          resource : "key1.resource" },
+        { key : "key2",
+          value : "value2",
+          isFinal : "ke2.isFinal",
+          resource : "key2.resource" }
+       ]
+   }
+
    +
    • + +
  • + When propertyName is not empty, and the property is not + found in the configuration, this method will throw an + {@link IllegalArgumentException}. +
    • +

    + @param config the configuration + @param propertyName property name + @param out the Writer to write to + @throws IOException + @throws IllegalArgumentException when property name is not + empty and the property is not found in configuration]]> + + + + + + + + + { "properties" : + [ { key : "key1", + value : "value1", + isFinal : "key1.isFinal", + resource : "key1.resource" }, + { key : "key2", + value : "value2", + isFinal : "ke2.isFinal", + resource : "key2.resource" } + ] + } + + + It does not output the properties of the configuration object which + is loaded from an input stream. +

    + + @param config the configuration + @param out the Writer to write to + @throws IOException]]> + + + + + + + + + + + + + + + + + + + true to set quiet-mode on, false + to turn it off.]]> + + + + + + + + + + + + + + + + + + + + + with matching keys]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Resources + +

    Configurations are specified by resources. A resource contains a set of + name/value pairs as XML data. Each resource is named by either a + String or by a {@link Path}. If named by a String, + then the classpath is examined for a file with that name. If named by a + Path, then the local filesystem is examined directly, without + referring to the classpath. + +

    Unless explicitly turned off, Hadoop by default specifies two + resources, loaded in-order from the classpath:

      +
    1. + + core-default.xml: Read-only defaults for hadoop.
      2. +
    2. core-site.xml: Site-specific configuration for a given hadoop + installation.
      3. +
    + Applications may add additional resources, which are loaded + subsequent to these resources in the order they are added. + +

    Final Parameters

    + +

    Configuration parameters may be declared final. + Once a resource declares a value final, no subsequently-loaded + resource can alter that value. + For example, one might define a final parameter with: + 

    +  <property>
+    <name>dfs.hosts.include</name>
+    <value>/etc/hadoop/conf/hosts.include</value>
+    <final>true</final>
+  </property>
    + + Administrators typically define parameters as final in + core-site.xml for values that user applications may not alter. + +

    Variable Expansion

    + +

    Value strings are first processed for variable expansion. The + available properties are:

      +
    1. Other properties defined in this Configuration; and, if a name is + undefined here,
      2. +
    2. Environment variables in {@link System#getenv()} if a name starts with + "env.", or
      3. +
    3. Properties in {@link System#getProperties()}.
      4. +
    + +

    For example, if a configuration resource contains the following property + definitions: + 

    +  <property>
+    <name>basedir</name>
+    <value>/user/${user.name}</value>
+  </property>
+  
+  <property>
+    <name>tempdir</name>
+    <value>${basedir}/tmp</value>
+  </property>
+
+  <property>
+    <name>otherdir</name>
+    <value>${env.BASE_DIR}/other</value>
+  </property>
+
    + +

    When conf.get("tempdir") is called, then ${basedir} + will be resolved to another property in this Configuration, while + ${user.name} would then ordinarily be resolved to the value + of the System property with that name. +

    When conf.get("otherdir") is called, then ${env.BASE_DIR} + will be resolved to the value of the ${BASE_DIR} environment variable. + It supports ${env.NAME:-default} and ${env.NAME-default} notations. + The former is resolved to "default" if ${NAME} environment variable is undefined + or its value is empty. + The latter behaves the same way only if ${NAME} is undefined. +

    By default, warnings will be given to any deprecated configuration + parameters and these are suppressible by configuring + log4j.logger.org.apache.hadoop.conf.Configuration.deprecation in + log4j.properties file. + +

    Tags

    + +

    Optionally we can tag related properties together by using tag + attributes. System tags are defined by hadoop.tags.system property. Users + can define there own custom tags in hadoop.tags.custom property. + +

    For example, we can tag existing property as: + 

    +  <property>
+    <name>dfs.replication</name>
+    <value>3</value>
+    <tag>HDFS,REQUIRED</tag>
+  </property>
+
+  <property>
+    <name>dfs.data.transfer.protection</name>
+    <value>3</value>
+    <tag>HDFS,SECURITY</tag>
+  </property>
+
    +

    Properties marked with tags can be retrieved with conf + .getAllPropertiesByTag("HDFS") or conf.getAllPropertiesByTags + (Arrays.asList("YARN","SECURITY")).

    ]]> +     +     + + + + + + + + + + + + + + + + + + + + + + + + + + + +     + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + This implementation generates the key material and calls the + {@link #createKey(String, byte[], Options)} method. + + @param name the base name of the key + @param options the options for the new key. + @return the version name of the first version of the key. + @throws IOException + @throws NoSuchAlgorithmException]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + This implementation generates the key material and calls the + {@link #rollNewVersion(String, byte[])} method. + + @param name the basename of the key + @return the name of the new version of the key + @throws IOException]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + KeyProvider implementations must be thread safe.]]> + + + + + + + + + + + + + + + + + + + + + + NULL if + a provider for the specified URI scheme could not be found. + @throws IOException thrown if the provider failed to initialize.]]> + + + + + + + + + + + + + + + + + + + + + + uri has syntax error]]> + + + + + + + + + + + + + + + + uri is + not found]]> + + + + + + + + + + + + + + + + + + + + + + + uri + determines a configuration property name, + fs.AbstractFileSystem.scheme.impl whose value names the + AbstractFileSystem class. + + The entire URI and conf is passed to the AbstractFileSystem factory method. + + @param uri for the file system to be created. + @param conf which is passed to the file system impl. + + @return file system for the given URI. + + @throws UnsupportedFileSystemException if the file system for + uri is not supported.]]> + + + + + + + + + + + + default port;]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + In some FileSystem implementations such as HDFS metadata + synchronization is essential to guarantee consistency of read requests + particularly in HA setting. + @throws IOException + @throws UnsupportedOperationException]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + describing modifications + @throws IOException if an ACL could not be modified]]> + + + + + + + + describing entries to remove + @throws IOException if an ACL could not be modified]]> + + + + + + + + + + + + + + + + + + + + + + describing modifications, must include entries + for user, group, and others for compatibility with permission bits. + @throws IOException if an ACL could not be modified]]> + + + + + + + which returns each AclStatus + @throws IOException if an ACL could not be read]]> + + + + + + + + + + Refer to the HDFS extended attributes user documentation for details. + + @param path Path to modify + @param name xattr name. + @param value xattr value. + @throws IOException]]> + + + + + + + + + + + Refer to the HDFS extended attributes user documentation for details. + + @param path Path to modify + @param name xattr name. + @param value xattr value. + @param flag xattr set flag + @throws IOException]]> + + + + + + + + + Refer to the HDFS extended attributes user documentation for details. + + @param path Path to get extended attribute + @param name xattr name. + @return byte[] xattr value. + @throws IOException]]> + + + + + + + + Refer to the HDFS extended attributes user documentation for details. + + @param path Path to get extended attributes + @return Map describing the XAttrs of the file or directory + @throws IOException]]> + + + + + + + + + Refer to the HDFS extended attributes user documentation for details. + + @param path Path to get extended attributes + @param names XAttr names. + @return Map describing the XAttrs of the file or directory + @throws IOException]]> + + + + + + + + Refer to the HDFS extended attributes user documentation for details. + + @param path Path to get extended attributes + @return Map describing the XAttrs of the file or directory + @throws IOException]]> + + + + + + + + + Refer to the HDFS extended attributes user documentation for details. + + @param path Path to remove extended attribute + @param name xattr name + @throws IOException]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + BlockLocation(offset: 0, length: BLOCK_SIZE, + hosts: {"host1:9866", "host2:9866, host3:9866"}) + + + And if the file is erasure-coded, each BlockLocation represents a logical + block groups. Value offset is the offset of a block group in the file and + value length is the total length of a block group. Hosts of a BlockLocation + are the datanodes that holding all the data blocks and parity blocks of a + block group. + Suppose we have a RS_3_2 coded file (3 data units and 2 parity units). + A BlockLocation example will be like: + 
    + BlockLocation(offset: 0, length: 3 * BLOCK_SIZE, hosts: {"host1:9866",
+   "host2:9866","host3:9866","host4:9866","host5:9866"})
+
    + + Please refer to + {@link FileSystem#getFileBlockLocations(FileStatus, long, long)} or + {@link FileContext#getFileBlockLocations(Path, long, long)} + for more examples.]]> +     +     + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + After a successful call, buf.position() will be advanced by the number + of bytes read and buf.limit() should be unchanged. +

    + In the case of an exception, the values of buf.position() and buf.limit() + are undefined, and callers should be prepared to recover from this + eventuality. +

    + Many implementations will throw {@link UnsupportedOperationException}, so + callers that are not confident in support for this method from the + underlying filesystem should be prepared to handle that exception. +

    + Implementations should treat 0-length requests as legitimate, and must not + signal an error upon their receipt. + + @param buf + the ByteBuffer to receive the results of the read operation. + @return the number of bytes read, possibly zero, or -1 if + reach end-of-stream + @throws IOException + if there is some error performing the read]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + setReplication of FileSystem + @param src file name + @param replication new replication + @throws IOException + @return true if successful; + false if file does not exist or is a directory]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + core-default.xml]]> + + + + + + + + + + + core-default.xml]]> + + + + + + + + + + + core-default.xml]]> + + + + + + + + + + + core-default.xml]]> + + + + + + + + + + + core-default.xml]]> + + + + + + core-default.xml]]> + + + + + + + + + + + core-default.xml]]> + + + + + + + + + + + core-default.xml]]> + + + + + + core-default.xml]]> + + + + + + core-default.xml]]> + + + + + + core-default.xml]]> + + + + + + + + core-default.xml]]> + + + + + + + + + + + + + + + + + + + + + core-default.xml]]> + + + + + + + + + + + core-default.xml]]> + + + + + + core-default.xml]]> + + + + + + core-default.xml]]> + + + + + + core-default.xml]]> + + + + + + + + + + + core-default.xml]]> + + + + + + + + + + + core-default.xml]]> + + + + + + + + + + + core-default.xml]]> + + + + + + + + + + + + + + + + core-default.xml]]> + + + + + + + + + + + core-default.xml]]> + + + + + + + + + + + core-default.xml]]> + + + + + + + + + + + core-default.xml]]> + + + + + + + + + + + core-default.xml]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + core-default.xml]]> + + + + + + + + + + + core-default.xml]]> + + + + + + + + + + + core-default.xml]]> + + + + + + core-default.xml]]> + + + + + + + + + + + core-default.xml]]> + + + + + + + + + + + core-default.xml]]> + + + + + + + + + + + + + + + + + + + + + + + core-default.xml]]> + + + + + + + + + + + core-default.xml]]> + + + + + + + + + + + core-default.xml]]> + + + + + + + + + + + core-default.xml]]> + + + + + + + + + + + core-default.xml]]> + + + + + + + + + + + core-default.xml]]> + + + + + + + + + + + + + + + + + + + + + core-default.xml]]> + + + + + + + + + + + core-default.xml]]> + + + + + + + + + + + core-default.xml]]> + + + + + + + + + + + core-default.xml]]> + + + + + + + + + + + core-default.xml]]> + + + + + + + + + + + + + + + + + + + + + + core-default.xml]]> + + + + + + + + core-default.xml]]> + + + + + + core-default.xml]]> + + + + + + + + + + + core-default.xml]]> + + + + + + core-default.xml]]> + + + + + + core-default.xml]]> + + + + + + core-default.xml]]> + + + + + + core-default.xml]]> + + + + + + core-default.xml]]> + + + + + + + + core-default.xml]]> + + + + + + core-default.xml]]> + + + + + + core-default.xml]]> + + + + + + core-default.xml]]> + + + + + + core-default.xml]]> + + + + + + + + + + + core-default.xml]]> + + + + + + + + + + + core-default.xml]]> + + + + + + core-default.xml]]> + + + + + + core-default.xml]]> + + + + + + core-default.xml]]> + + + + + + core-default.xml]]> + + + + + + core-default.xml]]> + + + + + + core-default.xml]]> + + + + + + core-default.xml]]> + + + + + + core-default.xml]]> + + + + + + + + core-default.xml]]> + + + + + + + + + + + core-default.xml]]> + + + + + + + + + + + + + + + + + core-default.xml]]> + + + + + + + + core-default.xml]]> + + + + + + core-default.xml]]> + + + + + + core-default.xml]]> + + + + + + + + + + + + + + + + core-default.xml]]> + + + + + + core-default.xml]]> + + + + + + + + + + + core-default.xml]]> + + + + + + + + + + + core-default.xml]]> + + + + + + + + + + + core-default.xml]]> + + + + + + + + + + + core-default.xml]]> + + + + + + + + + + + core-default.xml]]> + + + + + + + + + + + core-default.xml]]> + + + + + + + + + + + + + core-default.xml]]> + + + + + + + + + + + core-default.xml]]> + + + + + + + + + + + core-default.xml]]> + + + + + + + + + + + core-default.xml]]> + + + + + + core-default.xml]]> + + + + + + + + core-default.xml]]> + + + + + + + + core-default.xml]]> + + + + + + + + core-default.xml]]> + + + + + + + + + + + core-default.xml]]> + + + + + + core-default.xml]]> + + + + + + + + core-default.xml]]> + + + + + + core-default.xml]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + core-default.xml]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + EnumSet.of(CreateFlag.CREATE, CreateFlag.APPEND) + +

    + + Use the CreateFlag as follows: +

      +
    1. CREATE - to create a file if it does not exist, + else throw FileAlreadyExists.
      2. +
    2. APPEND - to append to a file if it exists, + else throw FileNotFoundException.
      3. +
    3. OVERWRITE - to truncate a file if it exists, + else throw FileNotFoundException.
      4. +
    4. CREATE|APPEND - to create a file if it does not exist, + else append to an existing file.
      5. +
    5. CREATE|OVERWRITE - to create a file if it does not exist, + else overwrite an existing file.
      6. +
    6. SYNC_BLOCK - to force closed blocks to the disk device. + In addition {@link Syncable#hsync()} should be called after each write, + if true synchronous behavior is required.
      7. +
    7. LAZY_PERSIST - Create the block on transient storage (RAM) if + available.
      8. +
    8. APPEND_NEWBLOCK - Append data to a new block instead of end of the last + partial block.
      9. +
    + + Following combinations are not valid and will result in + {@link HadoopIllegalArgumentException}: +
      +
    1. APPEND|OVERWRITE
      2. +
    2. CREATE|APPEND|OVERWRITE
      3. +
    ]]> +     + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + absOrFqPath is not supported. + @throws IOException If the file system for absOrFqPath could + not be instantiated.]]> + +     + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + defaultFsUri is not supported]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + NewWdir can be one of: +
      +
    • relative path: "foo/bar";
      • +
    • absolute without scheme: "/foo/bar"
      • +
    • fully qualified with scheme: "xx://auth/foo/bar"
      • +
    +
    + Illegal WDs: +
      +
    • relative with scheme: "xx:foo/bar"
      • +
    • non existent directory
      • +
    ]]> +     +     + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + f does not exist + @throws AccessControlException if access denied + @throws IOException If an IO Error occurred + + Exceptions applicable to file systems accessed over RPC: + @throws RpcClientException If an exception occurred in the RPC client + @throws RpcServerException If an exception occurred in the RPC server + @throws UnexpectedServerException If server implementation throws + undeclared exception to RPC server + + RuntimeExceptions: + @throws InvalidPathException If path f is not valid]]> + + + + + + + + + + + + + + + + + + + + +
  • Progress - to report progress on the operation - default null +
  • Permission - umask is applied against permission: default is + FsPermissions:getDefault() + +
  • CreateParent - create missing parent path; default is to not + to create parents +
  • The defaults for the following are SS defaults of the file + server implementing the target path. Not all parameters make sense + for all kinds of file system - eg. localFS ignores Blocksize, + replication, checksum +
      +
    • BufferSize - buffersize used in FSDataOutputStream +
    • Blocksize - block size for file blocks +
    • ReplicationFactor - replication for blocks +
    • ChecksumParam - Checksum parameters. server default is used + if not specified. +
    + + + @return {@link FSDataOutputStream} for created file + + @throws AccessControlException If access is denied + @throws FileAlreadyExistsException If file f already exists + @throws FileNotFoundException If parent of f does not exist + and createParent is false + @throws ParentNotDirectoryException If parent of f is not a + directory. + @throws UnsupportedFileSystemException If file system for f is + not supported + @throws IOException If an I/O error occurred + + Exceptions applicable to file systems accessed over RPC: + @throws RpcClientException If an exception occurred in the RPC client + @throws RpcServerException If an exception occurred in the RPC server + @throws UnexpectedServerException If server implementation throws + undeclared exception to RPC server + + RuntimeExceptions: + @throws InvalidPathException If path f is not valid]]> + + + + + + + + + + + + + + + + + + + + + dir already + exists + @throws FileNotFoundException If parent of dir does not exist + and createParent is false + @throws ParentNotDirectoryException If parent of dir is not a + directory + @throws UnsupportedFileSystemException If file system for dir + is not supported + @throws IOException If an I/O error occurred + + Exceptions applicable to file systems accessed over RPC: + @throws RpcClientException If an exception occurred in the RPC client + @throws UnexpectedServerException If server implementation throws + undeclared exception to RPC server + + RuntimeExceptions: + @throws InvalidPathException If path dir is not valid]]> + + + + + + + + + + + f does not exist + @throws UnsupportedFileSystemException If file system for f is + not supported + @throws IOException If an I/O error occurred + + Exceptions applicable to file systems accessed over RPC: + @throws RpcClientException If an exception occurred in the RPC client + @throws RpcServerException If an exception occurred in the RPC server + @throws UnexpectedServerException If server implementation throws + undeclared exception to RPC server + + RuntimeExceptions: + @throws InvalidPathException If path f is invalid]]> + + + + + + + + + + f does not exist + @throws UnsupportedFileSystemException If file system for f + is not supported + @throws IOException If an I/O error occurred + + Exceptions applicable to file systems accessed over RPC: + @throws RpcClientException If an exception occurred in the RPC client + @throws RpcServerException If an exception occurred in the RPC server + @throws UnexpectedServerException If server implementation throws + undeclared exception to RPC server]]> + + + + + + + + + + + f does not exist + @throws UnsupportedFileSystemException If file system for f is + not supported + @throws IOException If an I/O error occurred + + Exceptions applicable to file systems accessed over RPC: + @throws RpcClientException If an exception occurred in the RPC client + @throws RpcServerException If an exception occurred in the RPC server + @throws UnexpectedServerException If server implementation throws + undeclared exception to RPC server]]> + + + + + + + + + + + +
  • Fails if path is a directory. +
  • Fails if path does not exist. +
  • Fails if path is not closed. +
  • Fails if new size is greater than current size. + + @param f The path to the file to be truncated + @param newLength The size the file is to be truncated to + + @return true if the file has been truncated to the desired + newLength and is immediately available to be reused for + write operations such as append, or + false if a background process of adjusting the length of + the last block has been started, and clients should wait for it to + complete before proceeding with further file updates. + + @throws AccessControlException If access is denied + @throws FileNotFoundException If file f does not exist + @throws UnsupportedFileSystemException If file system for f is + not supported + @throws IOException If an I/O error occurred + + Exceptions applicable to file systems accessed over RPC: + @throws RpcClientException If an exception occurred in the RPC client + @throws RpcServerException If an exception occurred in the RPC server + @throws UnexpectedServerException If server implementation throws + undeclared exception to RPC server]]> + + + + + + + + + + f does not exist + @throws IOException If an I/O error occurred + + Exceptions applicable to file systems accessed over RPC: + @throws RpcClientException If an exception occurred in the RPC client + @throws RpcServerException If an exception occurred in the RPC server + @throws UnexpectedServerException If server implementation throws + undeclared exception to RPC server]]> + + + + + + + + + + + + + + +
  • Fails if src is a file and dst is a directory. +
  • Fails if src is a directory and dst is a file. +
  • Fails if the parent of dst does not exist or is a file. + +

    + If OVERWRITE option is not passed as an argument, rename fails if the dst + already exists. +

    + If OVERWRITE option is passed as an argument, rename overwrites the dst if + it is a file or an empty directory. Rename fails if dst is a non-empty + directory. +

    + Note that atomicity of rename is dependent on the file system + implementation. Please refer to the file system documentation for details +

    + + @param src path to be renamed + @param dst new path after rename + + @throws AccessControlException If access is denied + @throws FileAlreadyExistsException If dst already exists and + options has {@link Options.Rename#OVERWRITE} + option false. + @throws FileNotFoundException If src does not exist + @throws ParentNotDirectoryException If parent of dst is not a + directory + @throws UnsupportedFileSystemException If file system for src + and dst is not supported + @throws IOException If an I/O error occurred + + Exceptions applicable to file systems accessed over RPC: + @throws RpcClientException If an exception occurred in the RPC client + @throws RpcServerException If an exception occurred in the RPC server + @throws UnexpectedServerException If server implementation throws + undeclared exception to RPC server]]> + + + + + + + + + + + f does not exist + @throws UnsupportedFileSystemException If file system for f + is not supported + @throws IOException If an I/O error occurred + + Exceptions applicable to file systems accessed over RPC: + @throws RpcClientException If an exception occurred in the RPC client + @throws RpcServerException If an exception occurred in the RPC server + @throws UnexpectedServerException If server implementation throws + undeclared exception to RPC server]]> + + + + + + + + + + + + f does not exist + @throws UnsupportedFileSystemException If file system for f is + not supported + @throws IOException If an I/O error occurred + + Exceptions applicable to file systems accessed over RPC: + @throws RpcClientException If an exception occurred in the RPC client + @throws RpcServerException If an exception occurred in the RPC server + @throws UnexpectedServerException If server implementation throws + undeclared exception to RPC server + + RuntimeExceptions: + @throws HadoopIllegalArgumentException If username or + groupname is invalid.]]> + + + + + + + + + + + + f does not exist + @throws UnsupportedFileSystemException If file system for f is + not supported + @throws IOException If an I/O error occurred + + Exceptions applicable to file systems accessed over RPC: + @throws RpcClientException If an exception occurred in the RPC client + @throws RpcServerException If an exception occurred in the RPC server + @throws UnexpectedServerException If server implementation throws + undeclared exception to RPC server]]> + + + + + + + + + f does not exist + @throws IOException If an I/O error occurred + + Exceptions applicable to file systems accessed over RPC: + @throws RpcClientException If an exception occurred in the RPC client + @throws RpcServerException If an exception occurred in the RPC server + @throws UnexpectedServerException If server implementation throws + undeclared exception to RPC server]]> + + + + + + + + + + + f does not exist + @throws UnsupportedFileSystemException If file system for f is + not supported + @throws IOException If an I/O error occurred + + Exceptions applicable to file systems accessed over RPC: + @throws RpcClientException If an exception occurred in the RPC client + @throws RpcServerException If an exception occurred in the RPC server + @throws UnexpectedServerException If server implementation throws + undeclared exception to RPC server]]> + + + + + + + + + + f does not exist + @throws UnsupportedFileSystemException If file system for f is + not supported + @throws IOException If an I/O error occurred + + Exceptions applicable to file systems accessed over RPC: + @throws RpcClientException If an exception occurred in the RPC client + @throws RpcServerException If an exception occurred in the RPC server + @throws UnexpectedServerException If server implementation throws + undeclared exception to RPC server]]> + + + + + + + + + + + + + + + + + f does not exist + @throws UnsupportedFileSystemException If file system for f is + not supported + @throws IOException If an I/O error occurred]]> + + + + + + + + + + f does not exist + @throws UnsupportedFileSystemException If file system for f is + not supported + @throws IOException If the given path does not refer to a symlink + or an I/O error occurred]]> + + + + + + + + + + f does not exist + @throws UnsupportedFileSystemException If file system for f is + not supported + @throws IOException If an I/O error occurred + + Exceptions applicable to file systems accessed over RPC: + @throws RpcClientException If an exception occurred in the RPC client + @throws RpcServerException If an exception occurred in the RPC server + @throws UnexpectedServerException If server implementation throws + undeclared exception to RPC server]]> + + + + + + + + + + + + + + + Given a path referring to a symlink of form: + + <---X---> + fs://host/A/B/link + <-----Y-----> + + In this path X is the scheme and authority that identify the file system, + and Y is the path leading up to the final path component "link". If Y is + a symlink itself then let Y' be the target of Y and X' be the scheme and + authority of Y'. Symlink targets may: + + 1. Fully qualified URIs + + fs://hostX/A/B/file Resolved according to the target file system. + + 2. Partially qualified URIs (eg scheme but no host) + + fs:///A/B/file Resolved according to the target file system. Eg resolving + a symlink to hdfs:///A results in an exception because + HDFS URIs must be fully qualified, while a symlink to + file:///A will not since Hadoop's local file systems + require partially qualified URIs. + + 3. Relative paths + + path Resolves to [Y'][path]. Eg if Y resolves to hdfs://host/A and path + is "../B/file" then [Y'][path] is hdfs://host/B/file + + 4. Absolute paths + + path Resolves to [X'][path]. Eg if Y resolves hdfs://host/A/B and path + is "/file" then [X][path] is hdfs://host/file + + + @param target the target of the symbolic link + @param link the path to be created that points to target + @param createParent if true then missing parent dirs are created if + false then parent must exist + + + @throws AccessControlException If access is denied + @throws FileAlreadyExistsException If file linkcode> already exists + @throws FileNotFoundException If target does not exist + @throws ParentNotDirectoryException If parent of link is not a + directory. + @throws UnsupportedFileSystemException If file system for + target or link is not supported + @throws IOException If an I/O error occurred]]> + + + + + + + + + + f does not exist + @throws UnsupportedFileSystemException If file system for f is + not supported + @throws IOException If an I/O error occurred + + Exceptions applicable to file systems accessed over RPC: + @throws RpcClientException If an exception occurred in the RPC client + @throws RpcServerException If an exception occurred in the RPC server + @throws UnexpectedServerException If server implementation throws + undeclared exception to RPC server]]> + + + + + + + + + + + + + + + + + f does not exist + @throws UnsupportedFileSystemException If file system for f is + not supported + @throws IOException If an I/O error occurred + + Exceptions applicable to file systems accessed over RPC: + @throws RpcClientException If an exception occurred in the RPC client + @throws RpcServerException If an exception occurred in the RPC server + @throws UnexpectedServerException If server implementation throws + undeclared exception to RPC server]]> + + + + + + + + f is + not supported + @throws IOException If an I/O error occurred + + Exceptions applicable to file systems accessed over RPC: + @throws RpcClientException If an exception occurred in the RPC client + @throws RpcServerException If an exception occurred in the RPC server + @throws UnexpectedServerException If server implementation throws + undeclared exception to RPC server]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + describing modifications + @throws IOException if an ACL could not be modified]]> + + + + + + + + describing entries to remove + @throws IOException if an ACL could not be modified]]> + + + + + + + + + + + + + + + + + + + + + + describing modifications, must include entries + for user, group, and others for compatibility with permission bits. + @throws IOException if an ACL could not be modified]]> + + + + + + + which returns each AclStatus + @throws IOException if an ACL could not be read]]> + + + + + + + + + + Refer to the HDFS extended attributes user documentation for details. + + @param path Path to modify + @param name xattr name. + @param value xattr value. + @throws IOException]]> + + + + + + + + + + + Refer to the HDFS extended attributes user documentation for details. + + @param path Path to modify + @param name xattr name. + @param value xattr value. + @param flag xattr set flag + @throws IOException]]> + + + + + + + + + Refer to the HDFS extended attributes user documentation for details. + + @param path Path to get extended attribute + @param name xattr name. + @return byte[] xattr value. + @throws IOException]]> + + + + + + + + Refer to the HDFS extended attributes user documentation for details. + + @param path Path to get extended attributes + @return Map describing the XAttrs of the file or directory + @throws IOException]]> + + + + + + + + + Refer to the HDFS extended attributes user documentation for details. + + @param path Path to get extended attributes + @param names XAttr names. + @return Map describing the XAttrs of the file or directory + @throws IOException]]> + + + + + + + + + Refer to the HDFS extended attributes user documentation for details. + + @param path Path to remove extended attribute + @param name xattr name + @throws IOException]]> + + + + + + + + Refer to the HDFS extended attributes user documentation for details. + + @param path Path to get extended attributes + @return List of the XAttr names of the file or directory + @throws IOException]]> + + + + + + + Exceptions applicable to file systems accessed over RPC: + @throws RpcClientException If an exception occurred in the RPC client + @throws RpcServerException If an exception occurred in the RPC server + @throws UnexpectedServerException If server implementation throws + undeclared exception to RPC server]]> + + + + + + + + Exceptions applicable to file systems accessed over RPC: + @throws RpcClientException If an exception occurred in the RPC client + @throws RpcServerException If an exception occurred in the RPC server + @throws UnexpectedServerException If server implementation throws + undeclared exception to RPC server]]> + + + + + + + + + Exceptions applicable to file systems accessed over RPC: + @throws RpcClientException If an exception occurred in the RPC client + @throws RpcServerException If an exception occurred in the RPC server + @throws UnexpectedServerException If server implementation throws + undeclared exception to RPC server]]> + + + + + + + + Exceptions applicable to file systems accessed over RPC: + @throws RpcClientException If an exception occurred in the RPC client + @throws RpcServerException If an exception occurred in the RPC server + @throws UnexpectedServerException If server implementation throws + undeclared exception to RPC server]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Path Names + + The Hadoop file system supports a URI namespace and URI names. This enables + multiple types of file systems to be referenced using fully-qualified URIs. + Two common Hadoop file system implementations are +

      +
    • the local file system: file:///path +
    • the HDFS file system: hdfs://nnAddress:nnPort/path +
    + + The Hadoop file system also supports additional naming schemes besides URIs. + Hadoop has the concept of a default file system, which implies a + default URI scheme and authority. This enables slash-relative names + relative to the default FS, which are more convenient for users and + application writers. The default FS is typically set by the user's + environment, though it can also be manually specified. +

    + + Hadoop also supports working-directory-relative names, which are paths + relative to the current working directory (similar to Unix). The working + directory can be in a different file system than the default FS. +

    + Thus, Hadoop path names can be specified as one of the following: +

      +
    • a fully-qualified URI: scheme://authority/path (e.g. + hdfs://nnAddress:nnPort/foo/bar) +
    • a slash-relative name: path relative to the default file system (e.g. + /foo/bar) +
    • a working-directory-relative name: path relative to the working dir (e.g. + foo/bar) +
    + Relative paths with scheme (scheme:foo/bar) are illegal. + +

    Role of FileContext and Configuration Defaults

    + + The FileContext is the analogue of per-process file-related state in Unix. It + contains two properties: + +
      +
    • the default file system (for resolving slash-relative names) +
    • the umask (for file permissions) +
    + In general, these properties are obtained from the default configuration file + in the user's environment (see {@link Configuration}). + + Further file system properties are specified on the server-side. File system + operations default to using these server-side defaults unless otherwise + specified. +

    + The file system related server-side defaults are: +

      +
    • the home directory (default is "/user/userName") +
    • the initial wd (only for local fs) +
    • replication factor +
    • block size +
    • buffer size +
    • encryptDataTransfer +
    • checksum option. (checksumType and bytesPerChecksum) +
    + +

    Example Usage

    + + Example 1: use the default config read from the $HADOOP_CONFIG/core.xml. + Unspecified values come from core-defaults.xml in the release jar. +
      +
    • myFContext = FileContext.getFileContext(); // uses the default config + // which has your default FS +
    • myFContext.create(path, ...); +
    • myFContext.setWorkingDir(path); +
    • myFContext.open (path, ...); +
    • ... +
    + Example 2: Get a FileContext with a specific URI as the default FS +
      +
    • myFContext = FileContext.getFileContext(URI); +
    • myFContext.create(path, ...); +
    • ... +
    + Example 3: FileContext with local file system as the default +
      +
    • myFContext = FileContext.getLocalFSFileContext(); +
    • myFContext.create(path, ...); +
    • ... +
    + Example 4: Use a specific config, ignoring $HADOOP_CONFIG + Generally you should not need use a config unless you are doing +
      +
    • configX = someConfigSomeOnePassedToYou; +
    • myFContext = getFileContext(configX); // configX is not changed, + // is passed down +
    • myFContext.create(path, ...); +
    • ... +
    ]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + This implementation throws an UnsupportedOperationException. + + @return the protocol scheme for this FileSystem. + @throws UnsupportedOperationException if the operation is unsupported + (default).]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
  • + If the configuration has the property + {@code "fs.$SCHEME.impl.disable.cache"} set to true, + a new instance will be created, initialized with the supplied URI and + configuration, then returned without being cached. +
    • +
  • + If the there is a cached FS instance matching the same URI, it will + be returned. +
    • +
  • + Otherwise: a new FS instance will be created, initialized with the + configuration and URI, cached and returned to the caller. +
    • + + @throws IOException if the FileSystem cannot be instantiated.]]> +     +     + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + if f == null : + result = null + elif f.getLen() <= start: + result = [] + else result = [ locations(FS, b) for b in blocks(FS, p, s, s+l)] + + This call is most helpful with and distributed filesystem + where the hostnames of machines that contain blocks of the given file + can be determined. + + The default implementation returns an array containing one element: + 
    + BlockLocation( { "localhost:9866" },  { "localhost" }, 0, file.getLen())
+
    + + In HDFS, if file is three-replicated, the returned array contains + elements like: + 
    + BlockLocation(offset: 0, length: BLOCK_SIZE,
+   hosts: {"host1:9866", "host2:9866, host3:9866"})
+ BlockLocation(offset: BLOCK_SIZE, length: BLOCK_SIZE,
+   hosts: {"host2:9866", "host3:9866, host4:9866"})
+
    + + And if a file is erasure-coded, the returned BlockLocation are logical + block groups. + + Suppose we have a RS_3_2 coded file (3 data units and 2 parity units). + 1. If the file size is less than one stripe size, say 2 * CELL_SIZE, then + there will be one BlockLocation returned, with 0 offset, actual file size + and 4 hosts (2 data blocks and 2 parity blocks) hosting the actual blocks. + 3. If the file size is less than one group size but greater than one + stripe size, then there will be one BlockLocation returned, with 0 offset, + actual file size with 5 hosts (3 data blocks and 2 parity blocks) hosting + the actual blocks. + 4. If the file size is greater than one group size, 3 * BLOCK_SIZE + 123 + for example, then the result will be like: + 
    + BlockLocation(offset: 0, length: 3 * BLOCK_SIZE, hosts: {"host1:9866",
+   "host2:9866","host3:9866","host4:9866","host5:9866"})
+ BlockLocation(offset: 3 * BLOCK_SIZE, length: 123, hosts: {"host1:9866",
+   "host4:9866", "host5:9866"})
+
    + + @param file FilesStatus to get data from + @param start offset into the given file + @param len length for which to get locations for + @throws IOException IO failure]]> +     +     + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Important: the default implementation is not atomic + @param f path to use for create + @throws IOException IO failure]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
  • Fails if src is a file and dst is a directory.
    • +
  • Fails if src is a directory and dst is a file.
    • +
  • Fails if the parent of dst does not exist or is a file.
    • + +

    + If OVERWRITE option is not passed as an argument, rename fails + if the dst already exists. +

    + If OVERWRITE option is passed as an argument, rename overwrites + the dst if it is a file or an empty directory. Rename fails if dst is + a non-empty directory. +

    + Note that atomicity of rename is dependent on the file system + implementation. Please refer to the file system documentation for + details. This default implementation is non atomic. +

    + This method is deprecated since it is a temporary method added to + support the transition from FileSystem to FileContext for user + applications. + + @param src path to be renamed + @param dst new path after rename + @throws FileNotFoundException src path does not exist, or the parent + path of dst does not exist. + @throws FileAlreadyExistsException dest path exists and is a file + @throws ParentNotDirectoryException if the parent path of dest is not + a directory + @throws IOException on failure]]> + + + + + + + + +

  • Fails if path is a directory.
    • +
  • Fails if path does not exist.
    • +
  • Fails if path is not closed.
    • +
  • Fails if new size is greater than current size.
    • + + @param f The path to the file to be truncated + @param newLength The size the file is to be truncated to + + @return true if the file has been truncated to the desired + newLength and is immediately available to be reused for + write operations such as append, or + false if a background process of adjusting the length of + the last block has been started, and clients should wait for it to + complete before proceeding with further file updates. + @throws IOException IO failure + @throws UnsupportedOperationException if the operation is unsupported + (default).]]> +     +     + + + + + + + + + + + + + + + + + + + + +
  • Clean shutdown of the JVM cannot be guaranteed.
    • +
  • The time to shut down a FileSystem will depends on the number of + files to delete. For filesystems where the cost of checking + for the existence of a file/directory and the actual delete operation + (for example: object stores) is high, the time to shutdown the JVM can be + significantly extended by over-use of this feature.
    • +
  • Connectivity problems with a remote filesystem may delay shutdown + further, and may cause the files to not be deleted.
    • + + @param f the path to delete. + @return true if deleteOnExit is successful, otherwise false. + @throws IOException IO failure]]> +     +     + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Does not guarantee to return the List of files/directories status in a + sorted order. +

    + Will not return null. Expect IOException upon access error. + @param f given path + @return the statuses of the files/directories in the given patch + @throws FileNotFoundException when the path does not exist + @throws IOException see specific implementation]]> + + + + + + + + + + + + + + + + + Does not guarantee to return the List of files/directories status in a + sorted order. + + @param f + a path name + @param filter + the user-supplied path filter + @return an array of FileStatus objects for the files under the given path + after applying the filter + @throws FileNotFoundException when the path does not exist + @throws IOException see specific implementation]]> + + + + + + + + + Does not guarantee to return the List of files/directories status in a + sorted order. + + @param files + a list of paths + @return a list of statuses for the files under the given paths after + applying the filter default Path filter + @throws FileNotFoundException when the path does not exist + @throws IOException see specific implementation]]> + + + + + + + + + + Does not guarantee to return the List of files/directories status in a + sorted order. + + @param files + a list of paths + @param filter + the user-supplied path filter + @return a list of statuses for the files under the given paths after + applying the filter + @throws FileNotFoundException when the path does not exist + @throws IOException see specific implementation]]> + + + + + + + Return all the files that match filePattern and are not checksum + files. Results are sorted by their names. + +

    + A filename pattern is composed of regular characters and + special pattern matching characters, which are: + +

    +
    +
    +

    +

    ? +
    Matches any single character. + +

    +

    * +
    Matches zero or more characters. + +

    +

    [abc] +
    Matches a single character from character set + {a,b,c}. + +

    +

    [a-b] +
    Matches a single character from the character range + {a...b}. Note that character a must be + lexicographically less than or equal to character b. + +

    +

    [^a] +
    Matches a single character that is not from character set or range + {a}. Note that the ^ character must occur + immediately to the right of the opening bracket. + +

    +

    \c +
    Removes (escapes) any special meaning of character c. + +

    +

    {ab,cd} +
    Matches a string from the string set {ab, cd} + +

    +

    {ab,c{de,fh}} +
    Matches a string from the string set {ab, cde, cfh} + +
    +
    +
    + + @param pathPattern a glob specifying a path pattern + + @return an array of paths that match the path pattern + @throws IOException IO failure]]> +     +     + + + + + + + + + + + + + + f does not exist + @throws IOException If an I/O error occurred]]> + + + + + + + + + f does not exist + @throws IOException if any I/O error occurred]]> + + + + + + + + p does not exist + @throws IOException if any I/O error occurred]]> + + + + + + + + + + If the path is a directory, + if recursive is false, returns files in the directory; + if recursive is true, return files in the subtree rooted at the path. + If the path is a file, return the file's status and block locations. + + @param f is the path + @param recursive if the subdirectories need to be traversed recursively + + @return an iterator that traverses statuses of the files + + @throws FileNotFoundException when the path does not exist; + @throws IOException see specific implementation]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + undefined. + @throws IOException IO failure]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + In some FileSystem implementations such as HDFS metadata + synchronization is essential to guarantee consistency of read requests + particularly in HA setting. + @throws IOException + @throws UnsupportedOperationException]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + describing modifications + @throws IOException if an ACL could not be modified + @throws UnsupportedOperationException if the operation is unsupported + (default outcome).]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Refer to the HDFS extended attributes user documentation for details. + + @param path Path to modify + @param name xattr name. + @param value xattr value. + @throws IOException IO failure + @throws UnsupportedOperationException if the operation is unsupported + (default outcome).]]> + + + + + + + + + + + Refer to the HDFS extended attributes user documentation for details. + + @param path Path to modify + @param name xattr name. + @param value xattr value. + @param flag xattr set flag + @throws IOException IO failure + @throws UnsupportedOperationException if the operation is unsupported + (default outcome).]]> + + + + + + + + + Refer to the HDFS extended attributes user documentation for details. + + @param path Path to get extended attribute + @param name xattr name. + @return byte[] xattr value. + @throws IOException IO failure + @throws UnsupportedOperationException if the operation is unsupported + (default outcome).]]> + + + + + + + + Refer to the HDFS extended attributes user documentation for details. + + @param path Path to get extended attributes + @return Map describing the XAttrs of the file or directory + @throws IOException IO failure + @throws UnsupportedOperationException if the operation is unsupported + (default outcome).]]> + + + + + + + + + Refer to the HDFS extended attributes user documentation for details. + + @param path Path to get extended attributes + @param names XAttr names. + @return Map describing the XAttrs of the file or directory + @throws IOException IO failure + @throws UnsupportedOperationException if the operation is unsupported + (default outcome).]]> + + + + + + + + Refer to the HDFS extended attributes user documentation for details. + + @param path Path to get extended attributes + @return List of the XAttr names of the file or directory + @throws IOException IO failure + @throws UnsupportedOperationException if the operation is unsupported + (default outcome).]]> + + + + + + + + + Refer to the HDFS extended attributes user documentation for details. + + @param path Path to remove extended attribute + @param name xattr name + @throws IOException IO failure + @throws UnsupportedOperationException if the operation is unsupported + (default outcome).]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + This is a default method which is intended to be overridden by + subclasses. The default implementation returns an empty storage statistics + object.

    + + @return The StorageStatistics for this FileSystem instance. + Will never be null.]]> +     +     + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + All user code that may potentially use the Hadoop Distributed + File System should be written to use a FileSystem object or its + successor, {@link FileContext}. + +

    + The local implementation is {@link LocalFileSystem} and distributed + implementation is DistributedFileSystem. There are other implementations + for object stores and (outside the Apache Hadoop codebase), + third party filesystems. +

    + Notes +

      +
    1. The behaviour of the filesystem is + + specified in the Hadoop documentation. + However, the normative specification of the behavior of this class is + actually HDFS: if HDFS does not behave the way these Javadocs or + the specification in the Hadoop documentations define, assume that + the documentation is incorrect. +
      2. +
    2. The term {@code FileSystem} refers to an instance of this class.
      3. +
    3. The acronym "FS" is used as an abbreviation of FileSystem.
      4. +
    4. The term {@code filesystem} refers to the distributed/local filesystem + itself, rather than the class used to interact with it.
      5. +
    5. The term "file" refers to a file in the remote filesystem, + rather than instances of {@code java.io.File}.
      6. +
    ]]> +     + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + caller's environment variables to use + for expansion + @return String[] with absolute path to new jar in position 0 and + unexpanded wild card entry path in position 1 + @throws IOException if there is an I/O error while writing the jar file]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + FilterFileSystem contains + some other file system, which it uses as + its basic file system, possibly transforming + the data along the way or providing additional + functionality. The class FilterFileSystem + itself simply overrides all methods of + FileSystem with versions that + pass all requests to the contained file + system. Subclasses of FilterFileSystem + may further override some of these methods + and may also provide additional methods + and fields.]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + -1 + if there is no more data because the end of the stream has been + reached]]> + + + + + + + + + + length bytes have been read. + + @param position position in the input stream to seek + @param buffer buffer into which data is read + @param offset offset into the buffer in which data is written + @param length the number of bytes to read + @throws IOException IO problems + @throws EOFException If the end of stream is reached while reading. + If an exception is thrown an undetermined number + of bytes in the buffer may have been written.]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + // Don't + if (fs instanceof FooFileSystem) { + FooFileSystem fs = (FooFileSystem) fs; + OutputStream out = dfs.createFile(path) + .optionA() + .optionB("value") + .cache() + .build() + } else if (fs instanceof BarFileSystem) { + ... + } + + // Do + OutputStream out = fs.createFile(path) + .permission(perm) + .bufferSize(bufSize) + .opt("foofs:option.a", true) + .opt("foofs:option.b", "value") + .opt("barfs:cache", true) + .must("foofs:cache", true) + .must("barfs:cache-size", 256 * 1024 * 1024) + .build(); + + + If the option is not related to the file system, the option will be ignored. + If the option is must, but not supported by the file system, a + {@link IllegalArgumentException} will be thrown.]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + path is invalid]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + @return file]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + and the scheme is null, and the authority + is null. + + @return whether the path is absolute and the URI has no scheme nor + authority parts]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + true if and only if pathname + should be included]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Warning: Not all filesystems satisfy the thread-safety requirement. + @param position position within file + @param buffer destination buffer + @param offset offset in the buffer + @param length number of bytes to read + @return actual number of bytes read; -1 means "none" + @throws IOException IO problems.]]> + + + + + + + + + + Warning: Not all filesystems satisfy the thread-safety requirement. + @param position position within file + @param buffer destination buffer + @param offset offset in the buffer + @param length number of bytes to read + @throws IOException IO problems. + @throws EOFException the end of the data was reached before + the read operation completed]]> + + + + + + + + Warning: Not all filesystems satisfy the thread-safety requirement. + @param position position within file + @param buffer destination buffer + @throws IOException IO problems. + @throws EOFException the end of the data was reached before + the read operation completed]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + <----15----> <----15----> <----15----> <-------18-------> + QUOTA REMAINING_QUATA SPACE_QUOTA SPACE_QUOTA_REM FILE_NAME]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Note: Returned list is not sorted in any given order, + due to reliance on Java's {@link File#list()} API.)]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + XAttr is byte[], this class is to + covert byte[] to some kind of string representation or convert back. + String representation is convenient for display and input. For example + display in screen as shell response and json response, input as http + or shell parameter.]]> + + + + + + + + + + + + + + + + + + + + + +     + + + + + + + + + + + + + + + + + + + + + + + + + @return ftp]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + A {@link FileSystem} backed by an FTP client provided by Apache Commons Net. +

    ]]> +     +     + +     + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Since these methods are often vendor- or device-specific, operators + may implement this interface in order to achieve fencing. +

    + Fencing is configured by the operator as an ordered list of methods to + attempt. Each method will be tried in turn, and the next in the list + will only be attempted if the previous one fails. See {@link NodeFencer} + for more information. +

    + If an implementation also implements {@link Configurable} then its + setConf method will be called upon instantiation.]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + state (e.g ACTIVE/STANDBY) as well as + some additional information. + + @throws AccessControlException + if access is denied. + @throws IOException + if other errors happen + @see HAServiceStatus]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + hadoop.http.filter.initializers. + +

      +
    • StaticUserWebFilter - An authorization plugin that makes all +users a static configured user. +
    ]]> +     +     + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + public class IntArrayWritable extends ArrayWritable { + public IntArrayWritable() { + super(IntWritable.class); + } + } + ]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + o is a ByteWritable with the same value.]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + the class of the item + @param conf the configuration to store + @param item the object to be stored + @param keyName the name of the key to use + @throws IOException : forwards Exceptions from the underlying + {@link Serialization} classes.]]> + + + + + + + + + the class of the item + @param conf the configuration to use + @param keyName the name of the key to use + @param itemClass the class of the item + @return restored object + @throws IOException : forwards Exceptions from the underlying + {@link Serialization} classes.]]> + + + + + + + + + the class of the item + @param conf the configuration to use + @param items the objects to be stored + @param keyName the name of the key to use + @throws IndexOutOfBoundsException if the items array is empty + @throws IOException : forwards Exceptions from the underlying + {@link Serialization} classes.]]> + + + + + + + + + the class of the item + @param conf the configuration to use + @param keyName the name of the key to use + @param itemClass the class of the item + @return restored object + @throws IOException : forwards Exceptions from the underlying + {@link Serialization} classes.]]> + + + + + DefaultStringifier offers convenience methods to store/load objects to/from + the configuration. + + @param the class of the objects to stringify]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + o is a DoubleWritable with the same value.]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + value argument is null or + its size is zero, the elementType argument must not be null. If + the argument value's size is bigger than zero, the argument + elementType is not be used. + + @param value + @param elementType]]> + + + + + value should not be null + or empty. + + @param value]]> + + + + + + + + + + + + + + value and elementType. If the value argument + is null or its size is zero, the elementType argument must not be + null. If the argument value's size is bigger than zero, the + argument elementType is not be used. + + @param value + @param elementType]]> + + + + + + + + + + + + + + + + + + + o is an EnumSetWritable with the same value, + or both are null.]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + o is a FloatWritable with the same value.]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + When two sequence files, which have same Key type but different Value + types, are mapped out to reduce, multiple Value types is not allowed. + In this case, this class can help you wrap instances with different types. +

    + +

    + Compared with ObjectWritable, this class is much more effective, + because ObjectWritable will append the class declaration as a String + into the output file in every Key-Value pair. +

    + +

    + Generic Writable implements {@link Configurable} interface, so that it will be + configured by the framework. The configuration is passed to the wrapped objects + implementing {@link Configurable} interface before deserialization. +

    + + how to use it:
    + 1. Write your own class, such as GenericObject, which extends GenericWritable.
    + 2. Implements the abstract method getTypes(), defines + the classes which will be wrapped in GenericObject in application. + Attention: this classes defined in getTypes() method, must + implement Writable interface. +

    + + The code looks like this: + 
    + public class GenericObject extends GenericWritable {
+ 
+   private static Class[] CLASSES = {
+               ClassType1.class, 
+               ClassType2.class,
+               ClassType3.class,
+               };
+
+   protected Class[] getTypes() {
+       return CLASSES;
+   }
+
+ }
+
    + + @since Nov 8, 2006]]> +     +     + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + o is a IntWritable with the same value.]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + closes the input and output streams + at the end. + + @param in InputStrem to read from + @param out OutputStream to write to + @param conf the Configuration object]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + ignore any {@link Throwable} or + null pointers. Must only be used for cleanup in exception handlers. + + @param log the log to record problems to at debug level. Can be null. + @param closeables the objects to close + @deprecated use {@link #cleanupWithLogger(Logger, java.io.Closeable...)} + instead]]> + + + + + + + ignore any {@link Throwable} or + null pointers. Must only be used for cleanup in exception handlers. + + @param logger the log to record problems to at debug level. Can be null. + @param closeables the objects to close]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + This is better than File#listDir because it does not ignore IOExceptions. + + @param dir The directory to list. + @param filter If non-null, the filter to use when listing + this directory. + @return The list of files in the directory. + + @throws IOException On I/O error]]> + + + + + + + + Borrowed from Uwe Schindler in LUCENE-5588 + @param fileToSync the file to fsync]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + o is a LongWritable with the same value.]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + A map is a directory containing two files, the data file, + containing all keys and values in the map, and a smaller index + file, containing a fraction of the keys. The fraction is determined by + {@link Writer#getIndexInterval()}. + +

    The index file is read entirely into memory. Thus key implementations + should try to keep themselves small. + +

    Map files are created by adding entries in-order. To maintain a large + database, perform updates by copying the previous version of a database and + merging in a sorted change list, to create a new version of the database in + a new file. Sorting large change lists can be done with {@link + SequenceFile.Sorter}.]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + o is an MD5Hash whose digest contains the + same values.]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + className by first finding + it in the specified conf. If the specified conf is null, + try load it directly.]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + A {@link Comparator} that operates directly on byte representations of + objects. +

    + @param + @see DeserializerComparator]]> +     + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + SequenceFiles are flat files consisting of binary key/value + pairs. + +

    SequenceFile provides {@link SequenceFile.Writer}, + {@link SequenceFile.Reader} and {@link Sorter} classes for writing, + reading and sorting respectively.

    + + There are three SequenceFile Writers based on the + {@link CompressionType} used to compress key/value pairs: +
      +
    1. + Writer : Uncompressed records. +
      2. +
    2. + RecordCompressWriter : Record-compressed files, only compress + values. +
      3. +
    3. + BlockCompressWriter : Block-compressed files, both keys & + values are collected in 'blocks' + separately and compressed. The size of + the 'block' is configurable. +
    + +

    The actual compression algorithm used to compress key and/or values can be + specified by using the appropriate {@link CompressionCodec}.

    + +

    The recommended way is to use the static createWriter methods + provided by the SequenceFile to chose the preferred format.

    + +

    The {@link SequenceFile.Reader} acts as the bridge and can read any of the + above SequenceFile formats.

    + +

    SequenceFile Formats

    + +

    Essentially there are 3 different formats for SequenceFiles + depending on the CompressionType specified. All of them share a + common header described below. + +

    +
      +
    • + version - 3 bytes of magic header SEQ, followed by 1 byte of actual + version number (e.g. SEQ4 or SEQ6) +
      • +
    • + keyClassName -key class +
      • +
    • + valueClassName - value class +
      • +
    • + compression - A boolean which specifies if compression is turned on for + keys/values in this file. +
      • +
    • + blockCompression - A boolean which specifies if block-compression is + turned on for keys/values in this file. +
      • +
    • + compression codec - CompressionCodec class which is used for + compression of keys and/or values (if compression is + enabled). +
      • +
    • + metadata - {@link Metadata} for this file. +
      • +
    • + sync - A sync marker to denote end of the header. +
      • +
    + +
    Uncompressed SequenceFile Format
    +
      +
    • + Header +
      • +
    • + Record +
        +
      • Record length
        • +
      • Key length
        • +
      • Key
        • +
      • Value
        • +
      +
      • +
    • + A sync-marker every few 100 kilobytes or so. +
      • +
    + +
    Record-Compressed SequenceFile Format
    +
      +
    • + Header +
      • +
    • + Record +
        +
      • Record length
        • +
      • Key length
        • +
      • Key
        • +
      • Compressed Value
        • +
      +
      • +
    • + A sync-marker every few 100 kilobytes or so. +
      • +
    + +
    Block-Compressed SequenceFile Format
    +
      +
    • + Header +
      • +
    • + Record Block +
        +
      • Uncompressed number of records in the block
        • +
      • Compressed key-lengths block-size
        • +
      • Compressed key-lengths block
        • +
      • Compressed keys block-size
        • +
      • Compressed keys block
        • +
      • Compressed value-lengths block-size
        • +
      • Compressed value-lengths block
        • +
      • Compressed values block-size
        • +
      • Compressed values block
        • +
      +
      • +
    • + A sync-marker every block. +
      • +
    + +

    The compressed blocks of key lengths and value lengths consist of the + actual lengths of individual keys/values encoded in ZeroCompressedInteger + format.

    + + @see CompressionCodec]]> +     +     + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + o is a ShortWritable with the same value.]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + the class of the objects to stringify]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + position. Note that this + method avoids using the converter or doing String instantiation + @return the Unicode scalar value at position or -1 + if the position is invalid or points to a + trailing byte]]> + + + + + + + + + + what in the backing + buffer, starting as position start. The starting + position is measured in bytes and the return value is in + terms of byte position in the buffer. The backing buffer is + not converted to a string for this operation. + @return byte position of the first occurrence of the search + string in the UTF-8 buffer or -1 if not found]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Note: For performance reasons, this call does not clear the + underlying byte array that is retrievable via {@link #getBytes()}. + In order to free the byte-array memory, call {@link #set(byte[])} + with an empty byte array (For example, new byte[0]).]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + o is a Text with the same contents.]]> + + + + + + + + + + + + + + + + + + + + + + + + + replace is true, then + malformed input is replaced with the + substitution character, which is U+FFFD. Otherwise the + method throws a MalformedInputException.]]> + + + + + + + + + + + + + + + replace is true, then + malformed input is replaced with the + substitution character, which is U+FFFD. Otherwise the + method throws a MalformedInputException. + @return ByteBuffer: bytes stores at ByteBuffer.array() + and length is ByteBuffer.limit()]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + In + addition, it provides methods for string traversal without converting the + byte array to a string.

    Also includes utilities for + serializing/deserialing a string, coding/decoding a string, checking if a + byte array contains valid UTF8 code, calculating the length of an encoded + string.]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + This is useful when a class may evolve, so that instances written by the + old version of the class may still be processed by the new version. To + handle this situation, {@link #readFields(DataInput)} + implementations should catch {@link VersionMismatchException}.]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + o is a VIntWritable with the same value.]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + o is a VLongWritable with the same value.]]> + + + + + + + + + + + + + + + + + + + + + + + + out. + + @param out DataOuput to serialize this object into. + @throws IOException]]> + + + + + + + in. + +

    For efficiency, implementations should attempt to re-use storage in the + existing object where possible.

    + + @param in DataInput to deseriablize this object from. + @throws IOException]]> +     + + + Any key or value type in the Hadoop Map-Reduce + framework implements this interface.

    + +

    Implementations typically implement a static read(DataInput) + method which constructs a new instance, calls {@link #readFields(DataInput)} + and returns the instance.

    + +

    Example:

    + 

    +     public class MyWritable implements Writable {
+       // Some data
+       private int counter;
+       private long timestamp;
+
+       // Default constructor to allow (de)serialization
+       MyWritable() { }
+
+       public void write(DataOutput out) throws IOException {
+         out.writeInt(counter);
+         out.writeLong(timestamp);
+       }
+
+       public void readFields(DataInput in) throws IOException {
+         counter = in.readInt();
+         timestamp = in.readLong();
+       }
+
+       public static MyWritable read(DataInput in) throws IOException {
+         MyWritable w = new MyWritable();
+         w.readFields(in);
+         return w;
+       }
+     }
+

    ]]> +     + + + + + + + + WritableComparables can be compared to each other, typically + via Comparators. Any type which is to be used as a + key in the Hadoop Map-Reduce framework should implement this + interface.

    + +

    Note that hashCode() is frequently used in Hadoop to partition + keys. It's important that your implementation of hashCode() returns the same + result across different instances of the JVM. Note also that the default + hashCode() implementation in Object does not + satisfy this property.

    + +

    Example:

    + 

    +     public class MyWritableComparable implements WritableComparable {
+       // Some data
+       private int counter;
+       private long timestamp;
+       
+       public void write(DataOutput out) throws IOException {
+         out.writeInt(counter);
+         out.writeLong(timestamp);
+       }
+       
+       public void readFields(DataInput in) throws IOException {
+         counter = in.readInt();
+         timestamp = in.readLong();
+       }
+       
+       public int compareTo(MyWritableComparable o) {
+         int thisValue = this.value;
+         int thatValue = o.value;
+         return (thisValue < thatValue ? -1 : (thisValue==thatValue ? 0 : 1));
+       }
+
+       public int hashCode() {
+         final int prime = 31;
+         int result = 1;
+         result = prime * result + counter;
+         result = prime * result + (int) (timestamp ^ (timestamp >>> 32));
+         return result
+       }
+     }
+

    ]]> +     +     + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + The default implementation reads the data into two {@link + WritableComparable}s (using {@link + Writable#readFields(DataInput)}, then calls {@link + #compare(WritableComparable,WritableComparable)}.]]> + + + + + + + The default implementation uses the natural ordering, calling {@link + Comparable#compareTo(Object)}.]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + This base implementation uses the natural ordering. To define alternate + orderings, override {@link #compare(WritableComparable,WritableComparable)}. + +

    One may optimize compare-intensive operations by overriding + {@link #compare(byte[],int,int,byte[],int,int)}. Static utility methods are + provided to assist in optimized implementations of this method.]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Enum type + @param in DataInput to read from + @param enumType Class type of Enum + @return Enum represented by String read from DataInput + @throws IOException]]> + + + + + + + + + + + + + + + + len number of bytes in input streamin + @param in input stream + @param len number of bytes to skip + @throws IOException when skipped less number of bytes]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + CompressionCodec for which to get the + Compressor + @param conf the Configuration object which contains confs for creating or reinit the compressor + @return Compressor for the given + CompressionCodec from the pool or a new one]]> + + + + + + + + + CompressionCodec for which to get the + Decompressor + @return Decompressor for the given + CompressionCodec the pool or a new one]]> + + + + + + Compressor to be returned to the pool]]> + + + + + + Decompressor to be returned to the + pool]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Codec aliases are case insensitive. +

    + The code alias is the short class name (without the package name). + If the short class name ends with 'Codec', then there are two aliases for + the codec, the complete short class name and the short class name without + the 'Codec' ending. For example for the 'GzipCodec' codec class name the + alias are 'gzip' and 'gzipcodec'. + + @param codecName the canonical class name of the codec + @return the codec object]]> + + + + + + + Codec aliases are case insensitive. +

    + The code alias is the short class name (without the package name). + If the short class name ends with 'Codec', then there are two aliases for + the codec, the complete short class name and the short class name without + the 'Codec' ending. For example for the 'GzipCodec' codec class name the + alias are 'gzip' and 'gzipcodec'. + + @param codecName the canonical class name of the codec + @return the codec class]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Implementations are assumed to be buffered. This permits clients to + reposition the underlying input stream then call {@link #resetState()}, + without having to also synchronize client buffers.]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + true indicating that more input data is required. + + @param b Input data + @param off Start offset + @param len Length]]> + + + + + true if the input data buffer is empty and + #setInput() should be called in order to provide more input.]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + true if the end of the compressed + data output stream has been reached.]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + true indicating that more input data is required. + (Both native and non-native versions of various Decompressors require + that the data passed in via b[] remain unmodified until + the caller is explicitly notified--via {@link #needsInput()}--that the + buffer may be safely modified. With this requirement, an extra + buffer-copy can be avoided.) + + @param b Input data + @param off Start offset + @param len Length]]> + + + + + true if the input data buffer is empty and + {@link #setInput(byte[], int, int)} should be called to + provide more input. + + @return true if the input data buffer is empty and + {@link #setInput(byte[], int, int)} should be called in + order to provide more input.]]> + + + + + + + + + + + + + true if a preset dictionary is needed for decompression. + @return true if a preset dictionary is needed for decompression]]> + + + + + true if the end of the decompressed + data output stream has been reached. Indicates a concatenated data stream + when finished() returns true and {@link #getRemaining()} + returns a positive value. finished() will be reset with the + {@link #reset()} method. + @return true if the end of the decompressed + data output stream has been reached.]]> + + + + + + + + + + + + + + true and getRemaining() returns a positive value. If + {@link #finished()} returns true and getRemaining() returns + a zero value, indicates that the end of data stream has been reached and + is not a concatenated data stream. + @return The number of bytes remaining in the compressed data buffer.]]> + + + + + true and {@link #getRemaining()} returns a positive value, + reset() is called before processing of the next data stream in the + concatenated data stream. {@link #finished()} will be reset and will + return false when reset() is called.]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +

  • "none" - No compression. +
  • "lzo" - LZO compression. +
  • "gz" - GZIP compression. + ]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
  • Block Compression. +
  • Named meta data blocks. +
  • Sorted or unsorted keys. +
  • Seek by key or by file offset. + + The memory footprint of a TFile includes the following: +
      +
    • Some constant overhead of reading or writing a compressed block. +
        +
      • Each compressed block requires one compression/decompression codec for + I/O. +
      • Temporary space to buffer the key. +
      • Temporary space to buffer the value (for TFile.Writer only). Values are + chunk encoded, so that we buffer at most one chunk of user data. By default, + the chunk buffer is 1MB. Reading chunked value does not require additional + memory. +
      +
    • TFile index, which is proportional to the total number of Data Blocks. + The total amount of memory needed to hold the index can be estimated as + (56+AvgKeySize)*NumBlocks. +
    • MetaBlock index, which is proportional to the total number of Meta + Blocks.The total amount of memory needed to hold the index for Meta Blocks + can be estimated as (40+AvgMetaBlockName)*NumMetaBlock. +
    +

    + The behavior of TFile can be customized by the following variables through + Configuration: +

      +
    • tfile.io.chunk.size: Value chunk size. Integer (in bytes). Default + to 1MB. Values of the length less than the chunk size is guaranteed to have + known value length in read time (See + {@link TFile.Reader.Scanner.Entry#isValueLengthKnown()}). +
    • tfile.fs.output.buffer.size: Buffer size used for + FSDataOutputStream. Integer (in bytes). Default to 256KB. +
    • tfile.fs.input.buffer.size: Buffer size used for + FSDataInputStream. Integer (in bytes). Default to 256KB. +
    +

    + Suggestions on performance optimization. +

      +
    • Minimum block size. We recommend a setting of minimum block size between + 256KB to 1MB for general usage. Larger block size is preferred if files are + primarily for sequential access. However, it would lead to inefficient random + access (because there are more data to decompress). Smaller blocks are good + for random access, but require more memory to hold the block index, and may + be slower to create (because we must flush the compressor stream at the + conclusion of each data block, which leads to an FS I/O flush). Further, due + to the internal caching in Compression codec, the smallest possible block + size would be around 20KB-30KB. +
    • The current implementation does not offer true multi-threading for + reading. The implementation uses FSDataInputStream seek()+read(), which is + shown to be much faster than positioned-read call in single thread mode. + However, it also means that if multiple threads attempt to access the same + TFile (using multiple scanners) simultaneously, the actual I/O is carried out + sequentially even if they access different DFS blocks. +
    • Compression codec. Use "none" if the data is not very compressable (by + compressable, I mean a compression ratio at least 2:1). Generally, use "lzo" + as the starting point for experimenting. "gz" overs slightly better + compression ratio over "lzo" but requires 4x CPU to compress and 2x CPU to + decompress, comparing to "lzo". +
    • File system buffering, if the underlying FSDataInputStream and + FSDataOutputStream is already adequately buffered; or if applications + reads/writes keys and values in large buffers, we can reduce the sizes of + input/output buffering in TFile layer by setting the configuration parameters + "tfile.fs.input.buffer.size" and "tfile.fs.output.buffer.size". +
    + + Some design rationale behind TFile can be found at Hadoop-3315.]]> + + + + + + + + + + + Utils#writeVLong(out, n). + + @param out + output stream + @param n + The integer to be encoded + @throws IOException + @see Utils#writeVLong(DataOutput, long)]]> + + + + + + + + +
  • if n in [-32, 127): encode in one byte with the actual value. + Otherwise, +
  • if n in [-20*2^8, 20*2^8): encode in two bytes: byte[0] = n/256 - 52; + byte[1]=n&0xff. Otherwise, +
  • if n IN [-16*2^16, 16*2^16): encode in three bytes: byte[0]=n/2^16 - + 88; byte[1]=(n>>8)&0xff; byte[2]=n&0xff. Otherwise, +
  • if n in [-8*2^24, 8*2^24): encode in four bytes: byte[0]=n/2^24 - 112; + byte[1] = (n>>16)&0xff; byte[2] = (n>>8)&0xff; byte[3]=n&0xff. Otherwise: +
  • if n in [-2^31, 2^31): encode in five bytes: byte[0]=-125; byte[1] = + (n>>24)&0xff; byte[2]=(n>>16)&0xff; byte[3]=(n>>8)&0xff; byte[4]=n&0xff; +
  • if n in [-2^39, 2^39): encode in six bytes: byte[0]=-124; byte[1] = + (n>>32)&0xff; byte[2]=(n>>24)&0xff; byte[3]=(n>>16)&0xff; + byte[4]=(n>>8)&0xff; byte[5]=n&0xff +
  • if n in [-2^47, 2^47): encode in seven bytes: byte[0]=-123; byte[1] = + (n>>40)&0xff; byte[2]=(n>>32)&0xff; byte[3]=(n>>24)&0xff; + byte[4]=(n>>16)&0xff; byte[5]=(n>>8)&0xff; byte[6]=n&0xff; +
  • if n in [-2^55, 2^55): encode in eight bytes: byte[0]=-122; byte[1] = + (n>>48)&0xff; byte[2] = (n>>40)&0xff; byte[3]=(n>>32)&0xff; + byte[4]=(n>>24)&0xff; byte[5]=(n>>16)&0xff; byte[6]=(n>>8)&0xff; + byte[7]=n&0xff; +
  • if n in [-2^63, 2^63): encode in nine bytes: byte[0]=-121; byte[1] = + (n>>54)&0xff; byte[2] = (n>>48)&0xff; byte[3] = (n>>40)&0xff; + byte[4]=(n>>32)&0xff; byte[5]=(n>>24)&0xff; byte[6]=(n>>16)&0xff; + byte[7]=(n>>8)&0xff; byte[8]=n&0xff; + + + @param out + output stream + @param n + the integer number + @throws IOException]]> + + + + + + + (int)Utils#readVLong(in). + + @param in + input stream + @return the decoded integer + @throws IOException + + @see Utils#readVLong(DataInput)]]> + + + + + + + +
  • if (FB >= -32), return (long)FB; +
  • if (FB in [-72, -33]), return (FB+52)<<8 + NB[0]&0xff; +
  • if (FB in [-104, -73]), return (FB+88)<<16 + (NB[0]&0xff)<<8 + + NB[1]&0xff; +
  • if (FB in [-120, -105]), return (FB+112)<<24 + (NB[0]&0xff)<<16 + + (NB[1]&0xff)<<8 + NB[2]&0xff; +
  • if (FB in [-128, -121]), return interpret NB[FB+129] as a signed + big-endian integer. + + @param in + input stream + @return the decoded long integer. + @throws IOException]]> + + + + + + + + + + + + + + + + + + + + + + + + Type of the input key. + @param list + The list + @param key + The input key. + @param cmp + Comparator for the key. + @return The index to the desired element if it exists; or list.size() + otherwise.]]> + + + + + + + + + Type of the input key. + @param list + The list + @param key + The input key. + @param cmp + Comparator for the key. + @return The index to the desired element if it exists; or list.size() + otherwise.]]> + + + + + + + + Type of the input key. + @param list + The list + @param key + The input key. + @return The index to the desired element if it exists; or list.size() + otherwise.]]> + + + + + + + + Type of the input key. + @param list + The list + @param key + The input key. + @return The index to the desired element if it exists; or list.size() + otherwise.]]> + + + + + + + + + + + + + + + + + An experimental {@link Serialization} for Java {@link Serializable} classes. +

    + @see JavaSerializationComparator]]> +     +     + + + + + + + + + A {@link RawComparator} that uses a {@link JavaSerialization} + {@link Deserializer} to deserialize objects that are then compared via + their {@link Comparable} interfaces. +

    + @param + @see JavaSerialization]]> +     +     + + + + + + + + + + + + + +This package provides a mechanism for using different serialization frameworks +in Hadoop. The property "io.serializations" defines a list of +{@link org.apache.hadoop.io.serializer.Serialization}s that know how to create +{@link org.apache.hadoop.io.serializer.Serializer}s and +{@link org.apache.hadoop.io.serializer.Deserializer}s. +

    + +

    +To add a new serialization framework write an implementation of +{@link org.apache.hadoop.io.serializer.Serialization} and add its name to the +"io.serializations" property. +

    ]]> +     +     + + + + + + + + + + + + + + + + + + + avro.reflect.pkgs or implement + {@link AvroReflectSerializable} interface.]]> + + + + + + + + + + + + + + + + + + + + + + + + + + +This package provides Avro serialization in Hadoop. This can be used to +serialize/deserialize Avro types in Hadoop. +

    + +

    +Use {@link org.apache.hadoop.io.serializer.avro.AvroSpecificSerialization} for +serialization of classes generated by Avro's 'specific' compiler. +

    + +

    +Use {@link org.apache.hadoop.io.serializer.avro.AvroReflectSerialization} for +other classes. +{@link org.apache.hadoop.io.serializer.avro.AvroReflectSerialization} work for +any class which is either in the package list configured via +{@link org.apache.hadoop.io.serializer.avro.AvroReflectSerialization#AVRO_REFLECT_PACKAGES} +or implement {@link org.apache.hadoop.io.serializer.avro.AvroReflectSerializable} +interface. +

    ]]> +     +     + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Implementations of this interface consume the {@link MetricsRecord} generated + from {@link MetricsSource}. It registers with {@link MetricsSystem} which + periodically pushes the {@link MetricsRecord} to the sink using + {@link #putMetrics(MetricsRecord)} method. If the implementing class also + implements {@link Closeable}, then the MetricsSystem will close the sink when + it is stopped.]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + the actual type of the source object + @param source object to register + @return the source object + @exception MetricsException]]> + + + + + + + + the actual type of the source object + @param source object to register + @param name of the source. Must be unique or null (then extracted from + the annotations of the source object.) + @param desc the description of the source (or null. See above.) + @return the source object + @exception MetricsException]]> + + + + + + + + + + + + + + + + + + + + +
  • {@link MetricsSource} generate and update metrics information.
    • +
  • {@link MetricsSink} consume the metrics information
    • + + + {@link MetricsSource} and {@link MetricsSink} register with the metrics + system. Implementations of {@link MetricsSystem} polls the + {@link MetricsSource}s periodically and pass the {@link MetricsRecord}s to + {@link MetricsSink}.]]> +     +     + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +     + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + (aggregate). + Filter out entries that don't have at least minSamples. + + @return a map of peer DataNode Id to the average latency to that + node seen over the measurement period.]]> + + + + + + + + + + + This class maintains a group of rolling average metrics. It implements the + algorithm of rolling average, i.e. a number of sliding windows are kept to + roll over and evict old subsets of samples. Each window has a subset of + samples in a stream, where sub-sum and sub-total are collected. All sub-sums + and sub-totals in all windows will be aggregated to final-sum and final-total + used to compute final average, which is called rolling average. +

    ]]> +     +     + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +     + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + This class is a metrics sink that uses + {@link org.apache.hadoop.fs.FileSystem} to write the metrics logs. Every + roll interval a new directory will be created under the path specified by the + basepath property. All metrics will be logged to a file in the + current interval's directory in a file named <hostname>.log, where + <hostname> is the name of the host on which the metrics logging + process is running. The base path is set by the + <prefix>.sink.<instance>.basepath property. The + time zone used to create the current interval's directory name is GMT. If + the basepath property isn't specified, it will default to + "/tmp", which is the temp directory on whatever default file + system is configured for the cluster.

    + +

    The <prefix>.sink.<instance>.ignore-error + property controls whether an exception is thrown when an error is encountered + writing a log file. The default value is true. When set to + false, file errors are quietly swallowed.

    + +

    The roll-interval property sets the amount of time before + rolling the directory. The default value is 1 hour. The roll interval may + not be less than 1 minute. The property's value should be given as + number unit, where number is an integer value, and + unit is a valid unit. Valid units are minute, hour, + and day. The units are case insensitive and may be abbreviated or + plural. If no units are specified, hours are assumed. For example, + "2", "2h", "2 hour", and + "2 hours" are all valid ways to specify two hours.

    + +

    The roll-offset-interval-millis property sets the upper + bound on a random time interval (in milliseconds) that is used to delay + before the initial roll. All subsequent rolls will happen an integer + number of roll intervals after the initial roll, hence retaining the original + offset. The purpose of this property is to insert some variance in the roll + times so that large clusters using this sink on every node don't cause a + performance impact on HDFS by rolling simultaneously. The default value is + 30000 (30s). When writing to HDFS, as a rule of thumb, the roll offset in + millis should be no less than the number of sink instances times 5. + +

    The primary use of this class is for logging to HDFS. As it uses + {@link org.apache.hadoop.fs.FileSystem} to access the target file system, + however, it can be used to write to the local file system, Amazon S3, or any + other supported file system. The base path for the sink will determine the + file system used. An unqualified path will write to the default file system + set by the configuration.

    + +

    Not all file systems support the ability to append to files. In file + systems without the ability to append to files, only one writer can write to + a file at a time. To allow for concurrent writes from multiple daemons on a + single host, the source property is used to set unique headers + for the log files. The property should be set to the name of + the source daemon, e.g. namenode. The value of the + source property should typically be the same as the property's + prefix. If this property is not set, the source is taken to be + unknown.

    + +

    Instead of appending to an existing file, by default the sink + will create a new file with a suffix of ".<n>&quet;, where + n is the next lowest integer that isn't already used in a file name, + similar to the Hadoop daemon logs. NOTE: the file with the highest + sequence number is the newest file, unlike the Hadoop daemon logs.

    + +

    For file systems that allow append, the sink supports appending to the + existing file instead. If the allow-append property is set to + true, the sink will instead append to the existing file on file systems that + support appends. By default, the allow-append property is + false.

    + +

    Note that when writing to HDFS with allow-append set to true, + there is a minimum acceptable number of data nodes. If the number of data + nodes drops below that minimum, the append will succeed, but reading the + data will fail with an IOException in the DataStreamer class. The minimum + number of data nodes required for a successful append is generally 2 or + 3.

    + +

    Note also that when writing to HDFS, the file size information is not + updated until the file is closed (at the end of the interval) even though + the data is being written successfully. This is a known HDFS limitation that + exists because of the performance cost of updating the metadata. See + HDFS-5478.

    + +

    When using this sink in a secure (Kerberos) environment, two additional + properties must be set: keytab-key and + principal-key. keytab-key should contain the key by + which the keytab file can be found in the configuration, for example, + yarn.nodemanager.keytab. principal-key should + contain the key by which the principal can be found in the configuration, + for example, yarn.nodemanager.principal.]]> + + + + + + + + + + + + + + + + + + + + + + + + + CollectD StatsD plugin). +
    + To configure this plugin, you will need to add the following + entries to your hadoop-metrics2.properties file: +
    + 

    + *.sink.statsd.class=org.apache.hadoop.metrics2.sink.StatsDSink
+ [prefix].sink.statsd.server.host=
+ [prefix].sink.statsd.server.port=
+ [prefix].sink.statsd.skip.hostname=true|false (optional)
+ [prefix].sink.statsd.service.name=NameNode (name you want for service)
+
    ]]> +     +     + +     + + + + + + + + + + + + + ,name=" + Where the and are the supplied parameters. + + @param serviceName + @param nameName + @param theMbean - the MBean to register + @return the named used to register the MBean]]> + + + + + + + + + ,name=" + Where the and are the supplied parameters. + + @param serviceName + @param nameName + @param properties - Key value pairs to define additional JMX ObjectName + properties. + @param theMbean - the MBean to register + @return the named used to register the MBean]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + hostname or hostname:port. If + the specs string is null, defaults to localhost:defaultPort. + + @param specs server specs (see description) + @param defaultPort the default port if not specified + @return a list of InetSocketAddress objects.]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + This method is used when parts of Hadoop need know whether to apply + single rack vs multi-rack policies, such as during block placement. + Such algorithms behave differently if they are on multi-switch systems. +

    + + @return true if the mapping thinks that it is on a single switch]]> +     +     + + + + + + + + + + + + + + + + + This predicate simply assumes that all mappings not derived from + this class are multi-switch. + @param mapping the mapping to query + @return true if the base class says it is single switch, or the mapping + is not derived from this class.]]> + + + + It is not mandatory to + derive {@link DNSToSwitchMapping} implementations from it, but it is strongly + recommended, as it makes it easy for the Hadoop developers to add new methods + to this base class that are automatically picked up by all implementations. +

    + + This class does not extend the Configured + base class, and should not be changed to do so, as it causes problems + for subclasses. The constructor of the Configured calls + the {@link #setConf(Configuration)} method, which will call into the + subclasses before they have been fully constructed.]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + If a name cannot be resolved to a rack, the implementation + should return {@link NetworkTopology#DEFAULT_RACK}. This + is what the bundled implementations do, though it is not a formal requirement + + @param names the list of hosts to resolve (can be empty) + @return list of resolved network paths. + If names is empty, the returned list is also empty]]> + + + + + + + + + + + + + + + + + + + + + + + + Calling {@link #setConf(Configuration)} will trigger a + re-evaluation of the configuration settings and so be used to + set up the mapping script.]]> + + + + + + + + + + + + + + + + + + + + + This will get called in the superclass constructor, so a check is needed + to ensure that the raw mapping is defined before trying to relaying a null + configuration. + @param conf]]> + + + + + + + + + + It contains a static class RawScriptBasedMapping that performs + the work: reading the configuration parameters, executing any defined + script, handling errors and such like. The outer + class extends {@link CachedDNSToSwitchMapping} to cache the delegated + queries. +

    + This DNS mapper's {@link #isSingleSwitch()} predicate returns + true if and only if a script is defined.]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Simple {@link DNSToSwitchMapping} implementation that reads a 2 column text + file. The columns are separated by whitespace. The first column is a DNS or + IP address and the second column specifies the rack where the address maps. +

    +

    + This class uses the configuration parameter {@code + net.topology.table.file.name} to locate the mapping file. +

    +

    + Calls to {@link #resolve(List)} will look up the address as defined in the + mapping file. If no entry corresponding to the address is found, the value + {@code /default-rack} is returned. +

    ]]> +     +     + + + + +     + + + + + + + + + + + + + + + + + + (cause==null ? null : cause.toString()) (which + typically contains the class and detail message of cause). + @param cause the cause (which is saved for later retrieval by the + {@link #getCause()} method). (A null value is + permitted, and indicates that the cause is nonexistent or + unknown.)]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + mapping + and mapping]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + /host@realm. + @param principalName principal name of format as described above + @return host name if the the string conforms to the above format, else null]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + "jack" + + @param userName + @return userName without login method]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + the return type of the run method + @param action the method to execute + @return the value from the run method]]> + + + + + + + + the return type of the run method + @param action the method to execute + @return the value from the run method + @throws IOException if the action throws an IOException + @throws Error if the action throws an Error + @throws RuntimeException if the action throws a RuntimeException + @throws InterruptedException if the action throws an InterruptedException + @throws UndeclaredThrowableException if the action throws something else]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + CredentialProvider implementations must be thread safe.]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + (cause==null ? null : cause.toString()) (which + typically contains the class and detail message of cause). + @param cause the cause (which is saved for later retrieval by the + {@link #getCause()} method). (A null value is + permitted, and indicates that the cause is nonexistent or + unknown.)]]> + + + + + + + + + + + + + + does not provide the stack trace for security purposes.]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + A User-Agent String is considered to be a browser if it matches + any of the regex patterns from browser-useragent-regex; the default + behavior is to consider everything a browser that matches the following: + "^Mozilla.*,^Opera.*". Subclasses can optionally override + this method to use different behavior. + + @param userAgent The User-Agent String, or null if there isn't one + @return true if the User-Agent String refers to a browser, false if not]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + The type of the token identifier]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + T extends TokenIdentifier]]> + + + + + + + + + + DelegationTokenAuthenticatedURL. +

    + An instance of the default {@link DelegationTokenAuthenticator} will be + used.]]> + + + + + DelegationTokenAuthenticatedURL. + + @param authenticator the {@link DelegationTokenAuthenticator} instance to + use, if null the default one will be used.]]> + + + + + DelegationTokenAuthenticatedURL using the default + {@link DelegationTokenAuthenticator} class. + + @param connConfigurator a connection configurator.]]> + + + + + DelegationTokenAuthenticatedURL. + + @param authenticator the {@link DelegationTokenAuthenticator} instance to + use, if null the default one will be used. + @param connConfigurator a connection configurator.]]> + + + + + + + + + + + + The default class is {@link KerberosDelegationTokenAuthenticator} + + @return the delegation token authenticator class to use as default.]]> + + + + + + + This method is provided to enable WebHDFS backwards compatibility. + + @param useQueryString TRUE if the token is transmitted in the + URL query string, FALSE if the delegation token is transmitted + using the {@link DelegationTokenAuthenticator#DELEGATION_TOKEN_HEADER} HTTP + header.]]> + + + + + TRUE if the token is transmitted in the URL query + string, FALSE if the delegation token is transmitted using the + {@link DelegationTokenAuthenticator#DELEGATION_TOKEN_HEADER} HTTP header.]]> + + + + + + + + + + + + + + + + + + Authenticator. + + @param url the URL to connect to. Only HTTP/S URLs are supported. + @param token the authentication token being used for the user. + @return an authenticated {@link HttpURLConnection}. + @throws IOException if an IO error occurred. + @throws AuthenticationException if an authentication exception occurred.]]> + + + + + + + + + + Authenticator. If the doAs parameter is not NULL, + the request will be done on behalf of the specified doAs user. + + @param url the URL to connect to. Only HTTP/S URLs are supported. + @param token the authentication token being used for the user. + @param doAs user to do the the request on behalf of, if NULL the request is + as self. + @return an authenticated {@link HttpURLConnection}. + @throws IOException if an IO error occurred. + @throws AuthenticationException if an authentication exception occurred.]]> + + + + + + + + + + Authenticator + for authentication. + + @param url the URL to get the delegation token from. Only HTTP/S URLs are + supported. + @param token the authentication token being used for the user where the + Delegation token will be stored. + @param renewer the renewer user. + @return a delegation token. + @throws IOException if an IO error occurred. + @throws AuthenticationException if an authentication exception occurred.]]> + + + + + + + + + + + Authenticator + for authentication. + + @param url the URL to get the delegation token from. Only HTTP/S URLs are + supported. + @param token the authentication token being used for the user where the + Delegation token will be stored. + @param renewer the renewer user. + @param doAsUser the user to do as, which will be the token owner. + @return a delegation token. + @throws IOException if an IO error occurred. + @throws AuthenticationException if an authentication exception occurred.]]> + + + + + + + + + Authenticator for authentication. + + @param url the URL to renew the delegation token from. Only HTTP/S URLs are + supported. + @param token the authentication token with the Delegation Token to renew. + @throws IOException if an IO error occurred. + @throws AuthenticationException if an authentication exception occurred.]]> + + + + + + + + + + Authenticator for authentication. + + @param url the URL to renew the delegation token from. Only HTTP/S URLs are + supported. + @param token the authentication token with the Delegation Token to renew. + @param doAsUser the user to do as, which will be the token owner. + @throws IOException if an IO error occurred. + @throws AuthenticationException if an authentication exception occurred.]]> + + + + + + + + Authenticator. + + @param url the URL to cancel the delegation token from. Only HTTP/S URLs + are supported. + @param token the authentication token with the Delegation Token to cancel. + @throws IOException if an IO error occurred.]]> + + + + + + + + + Authenticator. + + @param url the URL to cancel the delegation token from. Only HTTP/S URLs + are supported. + @param token the authentication token with the Delegation Token to cancel. + @param doAsUser the user to do as, which will be the token owner. + @throws IOException if an IO error occurred.]]> + + + + DelegationTokenAuthenticatedURL is a + {@link AuthenticatedURL} sub-class with built-in Hadoop Delegation Token + functionality. +

    + The authentication mechanisms supported by default are Hadoop Simple + authentication (also known as pseudo authentication) and Kerberos SPNEGO + authentication. +

    + Additional authentication mechanisms can be supported via {@link + DelegationTokenAuthenticator} implementations. +

    + The default {@link DelegationTokenAuthenticator} is the {@link + KerberosDelegationTokenAuthenticator} class which supports + automatic fallback from Kerberos SPNEGO to Hadoop Simple authentication via + the {@link PseudoDelegationTokenAuthenticator} class. +

    + AuthenticatedURL instances are not thread-safe.]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Authenticator + for authentication. + + @param url the URL to get the delegation token from. Only HTTP/S URLs are + supported. + @param token the authentication token being used for the user where the + Delegation token will be stored. + @param renewer the renewer user. + @throws IOException if an IO error occurred. + @throws AuthenticationException if an authentication exception occurred.]]> + + + + + + + + + + + Authenticator + for authentication. + + @param url the URL to get the delegation token from. Only HTTP/S URLs are + supported. + @param token the authentication token being used for the user where the + Delegation token will be stored. + @param renewer the renewer user. + @param doAsUser the user to do as, which will be the token owner. + @throws IOException if an IO error occurred. + @throws AuthenticationException if an authentication exception occurred.]]> + + + + + + + + + + Authenticator for authentication. + + @param url the URL to renew the delegation token from. Only HTTP/S URLs are + supported. + @param token the authentication token with the Delegation Token to renew. + @throws IOException if an IO error occurred. + @throws AuthenticationException if an authentication exception occurred.]]> + + + + + + + + + + + Authenticator for authentication. + + @param url the URL to renew the delegation token from. Only HTTP/S URLs are + supported. + @param token the authentication token with the Delegation Token to renew. + @param doAsUser the user to do as, which will be the token owner. + @throws IOException if an IO error occurred. + @throws AuthenticationException if an authentication exception occurred.]]> + + + + + + + + + Authenticator. + + @param url the URL to cancel the delegation token from. Only HTTP/S URLs + are supported. + @param token the authentication token with the Delegation Token to cancel. + @throws IOException if an IO error occurred.]]> + + + + + + + + + + Authenticator. + + @param url the URL to cancel the delegation token from. Only HTTP/S URLs + are supported. + @param token the authentication token with the Delegation Token to cancel. + @param doAsUser the user to do as, which will be the token owner. + @throws IOException if an IO error occurred.]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + KerberosDelegationTokenAuthenticator provides support for + Kerberos SPNEGO authentication mechanism and support for Hadoop Delegation + Token operations. +

    + It falls back to the {@link PseudoDelegationTokenAuthenticator} if the HTTP + endpoint does not trigger a SPNEGO authentication]]> + + + + + + + + + PseudoDelegationTokenAuthenticator provides support for + Hadoop's pseudo authentication mechanism that accepts + the user name specified as a query string parameter and support for Hadoop + Delegation Token operations. +

    + This mimics the model of Hadoop Simple authentication trusting the + {@link UserGroupInformation#getCurrentUser()} value.]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + live. + @return a (snapshotted) map of blocker name->description values]]> + + + + + + + + + + + + + Do nothing if the service is null or not + in a state in which it can be/needs to be stopped. +

    + The service state is checked before the operation begins. + This process is not thread safe. + @param service a service or null]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +

  • Any long-lived operation here will prevent the service state + change from completing in a timely manner.
    • +
  • If another thread is somehow invoked from the listener, and + that thread invokes the methods of the service (including + subclass-specific methods), there is a risk of a deadlock.
    • + + + + @param service the service that has changed.]]> +     + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +     + + + + + + + + + + + + + + + + The base implementation logs all arguments at the debug level, + then returns the passed in config unchanged.]]> + + + + + + + The action is to signal success by returning the exit code 0.]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + This method is called before {@link #init(Configuration)}; + Any non-null configuration that is returned from this operation + becomes the one that is passed on to that {@link #init(Configuration)} + operation. +

    + This permits implementations to change the configuration before + the init operation. As the ServiceLauncher only creates + an instance of the base {@link Configuration} class, it is + recommended to instantiate any subclass (such as YarnConfiguration) + that injects new resources. +

    + @param config the initial configuration build up by the + service launcher. + @param args list of arguments passed to the command line + after any launcher-specific commands have been stripped. + @return the configuration to init the service with. + Recommended: pass down the config parameter with any changes + @throws Exception any problem]]> + + + + + + + The return value becomes the exit code of the launched process. +

    + If an exception is raised, the policy is: +

      +
    1. Any subset of {@link org.apache.hadoop.util.ExitUtil.ExitException}: + the exception is passed up unmodified. +
      2. +
    2. Any exception which implements + {@link org.apache.hadoop.util.ExitCodeProvider}: + A new {@link ServiceLaunchException} is created with the exit code + and message of the thrown exception; the thrown exception becomes the + cause.
      3. +
    3. Any other exception: a new {@link ServiceLaunchException} is created + with the exit code {@link LauncherExitCodes#EXIT_EXCEPTION_THROWN} and + the message of the original exception (which becomes the cause).
      4. +
    + @return the exit code + @throws org.apache.hadoop.util.ExitUtil.ExitException an exception passed + up as the exit code and error text. + @throws Exception any exception to report. If it provides an exit code + this is used in a wrapping exception.]]> +     +     + + + The command line options will be passed down before the + {@link Service#init(Configuration)} operation is invoked via an + invocation of {@link LaunchableService#bindArgs(Configuration, List)} + After the service has been successfully started via {@link Service#start()} + the {@link LaunchableService#execute()} method is called to execute the + service. When this method returns, the service launcher will exit, using + the return code from the method as its exit option.]]> + +     + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Approximate HTTP equivalent: {@code 400 Bad Request}]]> + + + + + + approximate HTTP equivalent: Approximate HTTP equivalent: {@code 401 Unauthorized}]]> + + + + + + + + + + + Approximate HTTP equivalent: Approximate HTTP equivalent: {@code 403: Forbidden}]]> + + + + + + Approximate HTTP equivalent: {@code 404: Not Found}]]> + + + + + + Approximate HTTP equivalent: {@code 405: Not allowed}]]> + + + + + + Approximate HTTP equivalent: {@code 406: Not Acceptable}]]> + + + + + + Approximate HTTP equivalent: {@code 408: Request Timeout}]]> + + + + + + Approximate HTTP equivalent: {@code 409: Conflict}]]> + + + + + + Approximate HTTP equivalent: {@code 500 Internal Server Error}]]> + + + + + + Approximate HTTP equivalent: {@code 501: Not Implemented}]]> + + + + + + Approximate HTTP equivalent: {@code 503 Service Unavailable}]]> + + + + + + If raised, this is expected to be raised server-side and likely due + to client/server version incompatibilities. +

    + Approximate HTTP equivalent: {@code 505: Version Not Supported}]]> + + + + + + + + + + + + + + + Codes with a YARN prefix are YARN-related. +

    + Many of the exit codes are designed to resemble HTTP error codes, + squashed into a single byte. e.g 44 , "not found" is the equivalent + of 404. The various 2XX HTTP error codes aren't followed; + the Unix standard of "0" for success is used. + 

    +    0-10: general command issues
+   30-39: equivalent to the 3XX responses, where those responses are
+          considered errors by the application.
+   40-49: client-side/CLI/config problems
+   50-59: service-side problems.
+   60+  : application specific error codes
+
    ]]> +     +     + + + + + + + + + + + + + + + + + + + This uses {@link String#format(String, Object...)} + to build the formatted exception in the ENGLISH locale. +

    + If the last argument is a throwable, it becomes the cause of the exception. + It will also be used as a parameter for the format. + @param exitCode exit code + @param format format for message to use in exception + @param args list of arguments]]> + + + + + When caught by the ServiceLauncher, it will convert that + into a process exit code. + + The {@link #ServiceLaunchException(int, String, Object...)} constructor + generates formatted exceptions.]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Clients and/or applications can use the provided Progressable + to explicitly report progress to the Hadoop framework. This is especially + important for operations which take significant amount of time since, + in-lieu of the reported progress, the framework has to assume that an error + has occurred and time-out the operation.

    ]]> +     + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Class is to be obtained + @return the correctly typed Class of the given object.]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + kill -0 command or equivalent]]> + + + + + + + + + + + + + + + + + + + ".cmd" on Windows, or ".sh" otherwise. + + @param parent File parent directory + @param basename String script file basename + @return File referencing the script in the directory]]> + + + + + + ".cmd" on Windows, or ".sh" otherwise. + + @param basename String script file basename + @return String script file name]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + IOException. + @return the path to {@link #WINUTILS_EXE} + @throws RuntimeException if the path is not resolvable]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Shell. + @return the thread that ran runCommand() that spawned this shell + or null if no thread is waiting for this shell to complete]]> + + + + + + + + + + + + Shell interface. + @param cmd shell command to execute. + @return the output of the executed command.]]> + + + + + + + + + Shell interface. + @param env the map of environment key=value + @param cmd shell command to execute. + @param timeout time in milliseconds after which script should be marked timeout + @return the output of the executed command. + @throws IOException on any problem.]]> + + + + + + + + Shell interface. + @param env the map of environment key=value + @param cmd shell command to execute. + @return the output of the executed command. + @throws IOException on any problem.]]> + + + + + Shell processes. + Iterates through a map of all currently running Shell + processes and destroys them one by one. This method is thread safe]]> + + + + + Shell objects.]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + CreateProcess synchronization object.]]> + + + + + os.name property.]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Important: caller must check for this value being null. + The lack of such checks has led to many support issues being raised. +

    + @deprecated use one of the exception-raising getter methods, + specifically {@link #getWinUtilsPath()} or {@link #getWinUtilsFile()}]]> + + + + + + + + + + + + + + Shell can be used to run shell commands like du or + df. It also offers facilities to gate commands by + time-intervals.]]> + + + + + + + + ShutdownHookManager singleton. + + @return ShutdownHookManager singleton.]]> + + + + + + + Runnable + @param priority priority of the shutdownHook.]]> + + + + + + + + + Runnable + @param priority priority of the shutdownHook + @param timeout timeout of the shutdownHook + @param unit unit of the timeout TimeUnit]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + ShutdownHookManager enables running shutdownHook + in a deterministic order, higher priority first. +

    + The JVM runs ShutdownHooks in a non-deterministic order or in parallel. + This class registers a single JVM shutdownHook and run all the + shutdownHooks registered to it (to this class) in order based on their + priority. + + Unless a hook was registered with a shutdown explicitly set through + {@link #addShutdownHook(Runnable, int, long, TimeUnit)}, + the shutdown time allocated to it is set by the configuration option + {@link CommonConfigurationKeysPublic#SERVICE_SHUTDOWN_TIMEOUT} in + {@code core-site.xml}, with a default value of + {@link CommonConfigurationKeysPublic#SERVICE_SHUTDOWN_TIMEOUT_DEFAULT} + seconds.]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Tool, is the standard for any Map-Reduce tool/application. + The tool/application should delegate the handling of + + standard command-line options to {@link ToolRunner#run(Tool, String[])} + and only handle its custom arguments.

    + +

    Here is how a typical Tool is implemented:

    + 

    +     public class MyApp extends Configured implements Tool {
+     
+       public int run(String[] args) throws Exception {
+         // Configuration processed by ToolRunner
+         Configuration conf = getConf();
+         
+         // Create a JobConf using the processed conf
+         JobConf job = new JobConf(conf, MyApp.class);
+         
+         // Process custom command-line options
+         Path in = new Path(args[1]);
+         Path out = new Path(args[2]);
+         
+         // Specify various job-specific parameters     
+         job.setJobName("my-app");
+         job.setInputPath(in);
+         job.setOutputPath(out);
+         job.setMapperClass(MyMapper.class);
+         job.setReducerClass(MyReducer.class);
+
+         // Submit the job, then poll for progress until the job is complete
+         RunningJob runningJob = JobClient.runJob(job);
+         if (runningJob.isSuccessful()) {
+           return 0;
+         } else {
+           return 1;
+         }
+       }
+       
+       public static void main(String[] args) throws Exception {
+         // Let ToolRunner handle generic command-line options 
+         int res = ToolRunner.run(new Configuration(), new MyApp(), args);
+         
+         System.exit(res);
+       }
+     }
+

    + + @see GenericOptionsParser + @see ToolRunner]]> +     + + + + + + + + + + + + + Tool by {@link Tool#run(String[])}, after + parsing with the given generic arguments. Uses the given + Configuration, or builds one if null. + + Sets the Tool's configuration with the possibly modified + version of the conf. + + @param conf Configuration for the Tool. + @param tool Tool to run. + @param args command-line arguments to the tool. + @return exit code of the {@link Tool#run(String[])} method.]]> + + + + + + + + Tool with its Configuration. + + Equivalent to run(tool.getConf(), tool, args). + + @param tool Tool to run. + @param args command-line arguments to the tool. + @return exit code of the {@link Tool#run(String[])} method.]]> + + + + + + + + + + + + + + + + + ToolRunner can be used to run classes implementing + Tool interface. It works in conjunction with + {@link GenericOptionsParser} to parse the + + generic hadoop command line arguments and modifies the + Configuration of the Tool. The + application-specific options are passed along without being modified. +

    + + @see Tool + @see GenericOptionsParser]]> +     +     + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +     + + + + + + + + + + + this filter. + @param nbHash The number of hash function to consider. + @param hashType type of the hashing function (see + {@link org.apache.hadoop.util.hash.Hash}).]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Bloom filter, as defined by Bloom in 1970. +

    + The Bloom filter is a data structure that was introduced in 1970 and that has been adopted by + the networking research community in the past decade thanks to the bandwidth efficiencies that it + offers for the transmission of set membership information between networked hosts. A sender encodes + the information into a bit vector, the Bloom filter, that is more compact than a conventional + representation. Computation and space costs for construction are linear in the number of elements. + The receiver uses the filter to test whether various elements are members of the set. Though the + filter will occasionally return a false positive, it will never return a false negative. When creating + the filter, the sender can choose its desired point in a trade-off between the false positive rate and the size. + +

    + Originally created by + European Commission One-Lab Project 034819. + + @see Filter The general behavior of a filter + + @see Space/Time Trade-Offs in Hash Coding with Allowable Errors]]> + + + + + + + + + + + + + this filter. + @param nbHash The number of hash function to consider. + @param hashType type of the hashing function (see + {@link org.apache.hadoop.util.hash.Hash}).]]> + + + + + + + + + this counting Bloom filter. +

    + Invariant: nothing happens if the specified key does not belong to this counter Bloom filter. + @param key The key to remove.]]> + + + + + + + + + + + + key -> count map. +

    NOTE: due to the bucket size of this filter, inserting the same + key more than 15 times will cause an overflow at all filter positions + associated with this key, and it will significantly increase the error + rate for this and other keys. For this reason the filter can only be + used to store small count values 0 <= N << 15. + @param key key to be tested + @return 0 if the key is not present. Otherwise, a positive value v will + be returned such that v == count with probability equal to the + error rate of this filter, and v > count otherwise. + Additionally, if the filter experienced an underflow as a result of + {@link #delete(Key)} operation, the return value may be lower than the + count with the probability of the false negative rate of such + filter.]]> + + + + + + + + + + + + + + + + + + + + + + counting Bloom filter, as defined by Fan et al. in a ToN + 2000 paper. +

    + A counting Bloom filter is an improvement to standard a Bloom filter as it + allows dynamic additions and deletions of set membership information. This + is achieved through the use of a counting vector instead of a bit vector. +

    + Originally created by + European Commission One-Lab Project 034819. + + @see Filter The general behavior of a filter + + @see Summary cache: a scalable wide-area web cache sharing protocol]]> + + + + + + + + + + + + + + Builds an empty Dynamic Bloom filter. + @param vectorSize The number of bits in the vector. + @param nbHash The number of hash function to consider. + @param hashType type of the hashing function (see + {@link org.apache.hadoop.util.hash.Hash}). + @param nr The threshold for the maximum number of keys to record in a + dynamic Bloom filter row.]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + dynamic Bloom filter, as defined in the INFOCOM 2006 paper. +

    + A dynamic Bloom filter (DBF) makes use of a s * m bit matrix but + each of the s rows is a standard Bloom filter. The creation + process of a DBF is iterative. At the start, the DBF is a 1 * m + bit matrix, i.e., it is composed of a single standard Bloom filter. + It assumes that nr elements are recorded in the + initial bit vector, where nr <= n (n is + the cardinality of the set A to record in the filter). +

    + As the size of A grows during the execution of the application, + several keys must be inserted in the DBF. When inserting a key into the DBF, + one must first get an active Bloom filter in the matrix. A Bloom filter is + active when the number of recorded keys, nr, is + strictly less than the current cardinality of A, n. + If an active Bloom filter is found, the key is inserted and + nr is incremented by one. On the other hand, if there + is no active Bloom filter, a new one is created (i.e., a new row is added to + the matrix) according to the current size of A and the element + is added in this new Bloom filter and the nr value of + this new Bloom filter is set to one. A given key is said to belong to the + DBF if the k positions are set to one in one of the matrix rows. +

    + Originally created by + European Commission One-Lab Project 034819. + + @see Filter The general behavior of a filter + @see BloomFilter A Bloom filter + + @see Theory and Network Applications of Dynamic Bloom Filters]]> + + + + + + + + + Builds a hash function that must obey to a given maximum number of returned values and a highest value. + @param maxValue The maximum highest returned value. + @param nbHash The number of resulting hashed values. + @param hashType type of the hashing function (see {@link Hash}).]]> + + + + + this hash function. A NOOP]]> + + + + + + + + + + + + + + + + + + + The idea is to randomly select a bit to reset.]]> + + + + + + The idea is to select the bit to reset that will generate the minimum + number of false negative.]]> + + + + + + The idea is to select the bit to reset that will remove the maximum number + of false positive.]]> + + + + + + The idea is to select the bit to reset that will, at the same time, remove + the maximum number of false positve while minimizing the amount of false + negative generated.]]> + + + + + Originally created by + European Commission One-Lab Project 034819.]]> + + + + + + + + + + + + + + this filter. + @param nbHash The number of hash function to consider. + @param hashType type of the hashing function (see + {@link org.apache.hadoop.util.hash.Hash}).]]> + + + + + + + + + this retouched Bloom filter. +

    + Invariant: if the false positive is null, nothing happens. + @param key The false positive key to add.]]> + + + + + + this retouched Bloom filter. + @param coll The collection of false positive.]]> + + + + + + this retouched Bloom filter. + @param keys The list of false positive.]]> + + + + + + this retouched Bloom filter. + @param keys The array of false positive.]]> + + + + + + + this retouched Bloom filter. + @param scheme The selective clearing scheme to apply.]]> + + + + + + + + + + + + retouched Bloom filter, as defined in the CoNEXT 2006 paper. +

    + It allows the removal of selected false positives at the cost of introducing + random false negatives, and with the benefit of eliminating some random false + positives at the same time. + +

    + Originally created by + European Commission One-Lab Project 034819. + + @see Filter The general behavior of a filter + @see BloomFilter A Bloom filter + @see RemoveScheme The different selective clearing algorithms + + @see Retouched Bloom Filters: Allowing Networked Applications to Trade Off Selected False Positives Against False Negatives]]> + + + + + + + + + + diff --git a/hadoop-common-project/hadoop-common/dev-support/jdiff/Apache_Hadoop_Common_3.3.4.xml b/hadoop-common-project/hadoop-common/dev-support/jdiff/Apache_Hadoop_Common_3.3.4.xml new file mode 100644 index 0000000000000..62a0e09f121af --- /dev/null +++ b/hadoop-common-project/hadoop-common/dev-support/jdiff/Apache_Hadoop_Common_3.3.4.xml @@ -0,0 +1,39037 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + UnsupportedOperationException + + If a key is deprecated in favor of multiple keys, they are all treated as + aliases of each other, and setting any one of them resets all the others + to the new value. + + If you have multiple deprecation entries to add, it is more efficient to + use #addDeprecations(DeprecationDelta[] deltas) instead. + + @param key to be deprecated + @param newKeys list of keys that take up the values of deprecated key + @param customMessage depcrication message + @deprecated use {@link #addDeprecation(String key, String newKey, + String customMessage)} instead]]> + + + + + + + + UnsupportedOperationException + + If you have multiple deprecation entries to add, it is more efficient to + use #addDeprecations(DeprecationDelta[] deltas) instead. + + @param key to be deprecated + @param newKey key that take up the values of deprecated key + @param customMessage deprecation message]]> + + + + + + + UnsupportedOperationException + + If a key is deprecated in favor of multiple keys, they are all treated as + aliases of each other, and setting any one of them resets all the others + to the new value. + + If you have multiple deprecation entries to add, it is more efficient to + use #addDeprecations(DeprecationDelta[] deltas) instead. + + @param key Key that is to be deprecated + @param newKeys list of keys that take up the values of deprecated key + @deprecated use {@link #addDeprecation(String key, String newKey)} instead]]> + + + + + + + UnsupportedOperationException + + If you have multiple deprecation entries to add, it is more efficient to + use #addDeprecations(DeprecationDelta[] deltas) instead. + + @param key Key that is to be deprecated + @param newKey key that takes up the value of deprecated key]]> + + + + + + key is deprecated. + + @param key the parameter which is to be checked for deprecation + @return true if the key is deprecated and + false otherwise.]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + final. + + @param name resource to be added, the classpath is examined for a file + with that name.]]> + + + + + + + + + + final. + + @param url url of the resource to be added, the local filesystem is + examined directly to find the resource, without referring to + the classpath.]]> + + + + + + + + + + final. + + @param file file-path of resource to be added, the local filesystem is + examined directly to find the resource, without referring to + the classpath.]]> + + + + + + + + + + final. + + WARNING: The contents of the InputStream will be cached, by this method. + So use this sparingly because it does increase the memory consumption. + + @param in InputStream to deserialize the object from. In will be read from + when a get or set is called next. After it is read the stream will be + closed.]]> + + + + + + + + + + + final. + + @param in InputStream to deserialize the object from. + @param name the name of the resource because InputStream.toString is not + very descriptive some times.]]> + + + + + + + + + + + final. + + @param conf Configuration object from which to load properties]]> + + + + + + + + + + + name property, null if + no such property exists. If the key is deprecated, it returns the value of + the first key which replaces the deprecated key and is not null. + + Values are processed for variable expansion + before being returned. + + As a side effect get loads the properties from the sources if called for + the first time as a lazy init. + + @param name the property name, will be trimmed before get value. + @return the value of the name or its replacing property, + or null if no such property exists.]]> + + + + + + + + + + + + + + + name property, but only for + names which have no valid value, usually non-existent or commented + out in XML. + + @param name the property name + @return true if the property name exists without value]]> + + + + + + name property as a trimmed String, + null if no such property exists. + If the key is deprecated, it returns the value of + the first key which replaces the deprecated key and is not null + + Values are processed for variable expansion + before being returned. + + @param name the property name. + @return the value of the name or its replacing property, + or null if no such property exists.]]> + + + + + + + name property as a trimmed String, + defaultValue if no such property exists. + See @{Configuration#getTrimmed} for more details. + + @param name the property name. + @param defaultValue the property default value. + @return the value of the name or defaultValue + if it is not set.]]> + + + + + + name property, without doing + variable expansion.If the key is + deprecated, it returns the value of the first key which replaces + the deprecated key and is not null. + + @param name the property name. + @return the value of the name property or + its replacing property and null if no such property exists.]]> + + + + + + + value of the name property. If + name is deprecated or there is a deprecated name associated to it, + it sets the value to both names. Name will be trimmed before put into + configuration. + + @param name property name. + @param value property value.]]> + + + + + + + + value of the name property. If + name is deprecated, it also sets the value to + the keys that replace the deprecated key. Name will be trimmed before put + into configuration. + + @param name property name. + @param value property value. + @param source the place that this configuration value came from + (For debugging). + @throws IllegalArgumentException when the value or name is null.]]> + + + + + + + + + + + + + + + + + + + + name. If the key is deprecated, + it returns the value of the first key which replaces the deprecated key + and is not null. + If no such property exists, + then defaultValue is returned. + + @param name property name, will be trimmed before get value. + @param defaultValue default value. + @return property value, or defaultValue if the property + doesn't exist.]]> + + + + + + + name property as an int. + + If no such property exists, the provided default value is returned, + or if the specified value is not a valid int, + then an error is thrown. + + @param name property name. + @param defaultValue default value. + @throws NumberFormatException when the value is invalid + @return property value as an int, + or defaultValue.]]> + + + + + + name property as a set of comma-delimited + int values. + + If no such property exists, an empty array is returned. + + @param name property name + @return property value interpreted as an array of comma-delimited + int values]]> + + + + + + + name property to an int. + + @param name property name. + @param value int value of the property.]]> + + + + + + + name property as a long. + If no such property exists, the provided default value is returned, + or if the specified value is not a valid long, + then an error is thrown. + + @param name property name. + @param defaultValue default value. + @throws NumberFormatException when the value is invalid + @return property value as a long, + or defaultValue.]]> + + + + + + + name property as a long or + human readable format. If no such property exists, the provided default + value is returned, or if the specified value is not a valid + long or human readable format, then an error is thrown. You + can use the following suffix (case insensitive): k(kilo), m(mega), g(giga), + t(tera), p(peta), e(exa) + + @param name property name. + @param defaultValue default value. + @throws NumberFormatException when the value is invalid + @return property value as a long, + or defaultValue.]]> + + + + + + + name property to a long. + + @param name property name. + @param value long value of the property.]]> + + + + + + + name property as a float. + If no such property exists, the provided default value is returned, + or if the specified value is not a valid float, + then an error is thrown. + + @param name property name. + @param defaultValue default value. + @throws NumberFormatException when the value is invalid + @return property value as a float, + or defaultValue.]]> + + + + + + + name property to a float. + + @param name property name. + @param value property value.]]> + + + + + + + name property as a double. + If no such property exists, the provided default value is returned, + or if the specified value is not a valid double, + then an error is thrown. + + @param name property name. + @param defaultValue default value. + @throws NumberFormatException when the value is invalid + @return property value as a double, + or defaultValue.]]> + + + + + + + name property to a double. + + @param name property name. + @param value property value.]]> + + + + + + + name property as a boolean. + If no such property is specified, or if the specified value is not a valid + boolean, then defaultValue is returned. + + @param name property name. + @param defaultValue default value. + @return property value as a boolean, + or defaultValue.]]> + + + + + + + name property to a boolean. + + @param name property name. + @param value boolean value of the property.]]> + + + + + + + + + + + + + + name property to the given type. This + is equivalent to set(<name>, value.toString()). + @param name property name + @param value new value + @param enumeration type]]> + + + + + + + enumeration type + @throws IllegalArgumentException If mapping is illegal for the type + provided + @return enumeration type]]> + + + + + + + + name to the given time duration. This + is equivalent to set(<name>, value + <time suffix>). + @param name Property name + @param value Time duration + @param unit Unit of time]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + name property as a Pattern. + If no such property is specified, or if the specified value is not a valid + Pattern, then DefaultValue is returned. + Note that the returned value is NOT trimmed by this method. + + @param name property name + @param defaultValue default value + @return property value as a compiled Pattern, or defaultValue]]> + + + + + + + Pattern. + If the pattern is passed as null, sets the empty pattern which results in + further calls to getPattern(...) returning the default value. + + @param name property name + @param pattern new value]]> + + + + + + + + + + + + + + + + + + + name property as + a collection of Strings. + If no such property is specified then empty collection is returned. +

    + This is an optimized version of {@link #getStrings(String)} + + @param name property name. + @return property value as a collection of Strings.]]> + + + + + + name property as + an array of Strings. + If no such property is specified then null is returned. + + @param name property name. + @return property value as an array of Strings, + or null.]]> + + + + + + + name property as + an array of Strings. + If no such property is specified then default value is returned. + + @param name property name. + @param defaultValue The default value + @return property value as an array of Strings, + or default value.]]> + + + + + + name property as + a collection of Strings, trimmed of the leading and trailing whitespace. + If no such property is specified then empty Collection is returned. + + @param name property name. + @return property value as a collection of Strings, or empty Collection]]> + + + + + + name property as + an array of Strings, trimmed of the leading and trailing whitespace. + If no such property is specified then an empty array is returned. + + @param name property name. + @return property value as an array of trimmed Strings, + or empty array.]]> + + + + + + + name property as + an array of Strings, trimmed of the leading and trailing whitespace. + If no such property is specified then default value is returned. + + @param name property name. + @param defaultValue The default value + @return property value as an array of trimmed Strings, + or default value.]]> + + + + + + + name property as + as comma delimited values. + + @param name property name. + @param values The values]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + hostProperty as a + InetSocketAddress. If hostProperty is + null, addressProperty will be used. This + is useful for cases where we want to differentiate between host + bind address and address clients should use to establish connection. + + @param hostProperty bind host property name. + @param addressProperty address property name. + @param defaultAddressValue the default value + @param defaultPort the default port + @return InetSocketAddress]]> + + + + + + + + name property as a + InetSocketAddress. + @param name property name. + @param defaultAddress the default value + @param defaultPort the default port + @return InetSocketAddress]]> + + + + + + + name property as + a host:port.]]> + + + + + + + + + name property as a host:port. The wildcard + address is replaced with the local host's address. If the host and address + properties are configured the host component of the address will be combined + with the port component of the addr to generate the address. This is to allow + optional control over which host name is used in multi-home bind-host + cases where a host can have multiple names + @param hostProperty the bind-host configuration name + @param addressProperty the service address configuration name + @param defaultAddressValue the service default address configuration value + @param addr InetSocketAddress of the service listener + @return InetSocketAddress for clients to connect]]> + + + + + + + name property as a host:port. The wildcard + address is replaced with the local host's address. + @param name property name. + @param addr InetSocketAddress of a listener to store in the given property + @return InetSocketAddress for clients to connect]]> + + + + + + + + + + + + + + + + + + + + name property + as an array of Class. + The value of the property specifies a list of comma separated class names. + If no such property is specified, then defaultValue is + returned. + + @param name the property name. + @param defaultValue default value. + @return property value as a Class[], + or defaultValue.]]> + + + + + + + name property as a Class. + If no such property is specified, then defaultValue is + returned. + + @param name the conf key name. + @param defaultValue default value. + @return property value as a Class, + or defaultValue.]]> + + + + + + + + name property as a Class + implementing the interface specified by xface. + + If no such property is specified, then defaultValue is + returned. + + An exception is thrown if the returned class does not implement the named + interface. + + @param name the conf key name. + @param defaultValue default value. + @param xface the interface implemented by the named class. + @return property value as a Class, + or defaultValue.]]> + + + + + + + name property as a List + of objects implementing the interface specified by xface. + + An exception is thrown if any of the classes does not exist, or if it does + not implement the named interface. + + @param name the property name. + @param xface the interface implemented by the classes named by + name. + @return a List of objects implementing xface.]]> + + + + + + + + name property to the name of a + theClass implementing the given interface xface. + + An exception is thrown if theClass does not implement the + interface xface. + + @param name property name. + @param theClass property value. + @param xface the interface implemented by the named class.]]> + + + + + + + + dirsProp with + the given path. If dirsProp contains multiple directories, + then one is chosen based on path's hash code. If the selected + directory does not exist, an attempt is made to create it. + + @param dirsProp directory in which to locate the file. + @param path file-path. + @return local file under the directory with the given path.]]> + + + + + + + + dirsProp with + the given path. If dirsProp contains multiple directories, + then one is chosen based on path's hash code. If the selected + directory does not exist, an attempt is made to create it. + + @param dirsProp directory in which to locate the file. + @param path file-path. + @return local file under the directory with the given path.]]> + + + + + + + + + + + + name. + + @param name configuration resource name. + @return an input stream attached to the resource.]]> + + + + + + name. + + @param name configuration resource name. + @return a reader attached to the resource.]]> + + + + + + + + + + + + + + + + + + + + + + String + key-value pairs in the configuration. + + @return an iterator over the entries.]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +

  • + When property name is not empty and the property exists in the + configuration, this method writes the property and its attributes + to the {@link Writer}. +
    • + +
  • + When property name is null or empty, this method writes all the + configuration properties and their attributes to the {@link Writer}. +
    • + +
  • + When property name is not empty but the property doesn't exist in + the configuration, this method throws an {@link IllegalArgumentException}. +
    • + + @param out the writer to write to.]]> +     + + + + + + + + +
  • + When propertyName is not empty, and the property exists + in the configuration, the format of the output would be, + 
    +  {
+    "property": {
+      "key" : "key1",
+      "value" : "value1",
+      "isFinal" : "key1.isFinal",
+      "resource" : "key1.resource"
+    }
+  }
+
    +
    • + +
  • + When propertyName is null or empty, it behaves same as + {@link #dumpConfiguration(Configuration, Writer)}, the + output would be, + 
    +  { "properties" :
+      [ { key : "key1",
+          value : "value1",
+          isFinal : "key1.isFinal",
+          resource : "key1.resource" },
+        { key : "key2",
+          value : "value2",
+          isFinal : "ke2.isFinal",
+          resource : "key2.resource" }
+       ]
+   }
+
    +
    • + +
  • + When propertyName is not empty, and the property is not + found in the configuration, this method will throw an + {@link IllegalArgumentException}. +
    • + +

    + @param config the configuration + @param propertyName property name + @param out the Writer to write to + @throws IOException + @throws IllegalArgumentException when property name is not + empty and the property is not found in configuration]]> + + + + + + + + + { "properties" : + [ { key : "key1", + value : "value1", + isFinal : "key1.isFinal", + resource : "key1.resource" }, + { key : "key2", + value : "value2", + isFinal : "ke2.isFinal", + resource : "key2.resource" } + ] + } + + + It does not output the properties of the configuration object which + is loaded from an input stream. +

    + + @param config the configuration + @param out the Writer to write to + @throws IOException]]> + + + + + + + + + + + + + + + + + + + true to set quiet-mode on, false + to turn it off.]]> + + + + + + + + + + + + + + + + + + + + + } with matching keys]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Resources + +

    Configurations are specified by resources. A resource contains a set of + name/value pairs as XML data. Each resource is named by either a + String or by a {@link Path}. If named by a String, + then the classpath is examined for a file with that name. If named by a + Path, then the local filesystem is examined directly, without + referring to the classpath. + +

    Unless explicitly turned off, Hadoop by default specifies two + resources, loaded in-order from the classpath:

      +
    1. + + core-default.xml: Read-only defaults for hadoop.
      2. +
    2. core-site.xml: Site-specific configuration for a given hadoop + installation.
      3. +
    + Applications may add additional resources, which are loaded + subsequent to these resources in the order they are added. + +

    Final Parameters

    + +

    Configuration parameters may be declared final. + Once a resource declares a value final, no subsequently-loaded + resource can alter that value. + For example, one might define a final parameter with: + 

    
+  <property>
+    <name>dfs.hosts.include</name>
+    <value>/etc/hadoop/conf/hosts.include</value>
+    <final>true</final>
+  </property>
    + + Administrators typically define parameters as final in + core-site.xml for values that user applications may not alter. + +

    Variable Expansion

    + +

    Value strings are first processed for variable expansion. The + available properties are:

      +
    1. Other properties defined in this Configuration; and, if a name is + undefined here,
      2. +
    2. Environment variables in {@link System#getenv()} if a name starts with + "env.", or
      3. +
    3. Properties in {@link System#getProperties()}.
      4. +
    + +

    For example, if a configuration resource contains the following property + definitions: + 

    
+  <property>
+    <name>basedir</name>
+    <value>/user/${user.name}</value>
+  </property>
+  
+  <property>
+    <name>tempdir</name>
+    <value>${basedir}/tmp</value>
+  </property>
+
+  <property>
+    <name>otherdir</name>
+    <value>${env.BASE_DIR}/other</value>
+  </property>
+
    + +

    When conf.get("tempdir") is called, then ${basedir} + will be resolved to another property in this Configuration, while + ${user.name} would then ordinarily be resolved to the value + of the System property with that name. +

    When conf.get("otherdir") is called, then ${env.BASE_DIR} + will be resolved to the value of the ${BASE_DIR} environment variable. + It supports ${env.NAME:-default} and ${env.NAME-default} notations. + The former is resolved to "default" if ${NAME} environment variable is undefined + or its value is empty. + The latter behaves the same way only if ${NAME} is undefined. +

    By default, warnings will be given to any deprecated configuration + parameters and these are suppressible by configuring + log4j.logger.org.apache.hadoop.conf.Configuration.deprecation in + log4j.properties file. + +

    Tags

    + +

    Optionally we can tag related properties together by using tag + attributes. System tags are defined by hadoop.tags.system property. Users + can define there own custom tags in hadoop.tags.custom property. + +

    For example, we can tag existing property as: + 

    
+  <property>
+    <name>dfs.replication</name>
+    <value>3</value>
+    <tag>HDFS,REQUIRED</tag>
+  </property>
+
+  <property>
+    <name>dfs.data.transfer.protection</name>
+    <value>3</value>
+    <tag>HDFS,SECURITY</tag>
+  </property>
+
    +

    Properties marked with tags can be retrieved with conf + .getAllPropertiesByTag("HDFS") or conf.getAllPropertiesByTags + (Arrays.asList("YARN","SECURITY")).

    ]]> +     +     + + + + + + + + + + + + + + + + + + + + + + + + + + + +     + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + This implementation generates the key material and calls the + {@link #createKey(String, byte[], Options)} method. + + @param name the base name of the key + @param options the options for the new key. + @return the version name of the first version of the key. + @throws IOException + @throws NoSuchAlgorithmException]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + This implementation generates the key material and calls the + {@link #rollNewVersion(String, byte[])} method. + + @param name the basename of the key + @return the name of the new version of the key + @throws IOException]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + KeyProvider implementations must be thread safe.]]> + + + + + + + + + + + + + + + + + + + + + + NULL if + a provider for the specified URI scheme could not be found. + @throws IOException thrown if the provider failed to initialize.]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + uri has syntax error]]> + + + + + + + + + + + + + + + + uri is + not found]]> + + + + + + + + + + + + + + + + + + + + + + + uri + determines a configuration property name, + fs.AbstractFileSystem.scheme.impl whose value names the + AbstractFileSystem class. + + The entire URI and conf is passed to the AbstractFileSystem factory method. + + @param uri for the file system to be created. + @param conf which is passed to the file system impl. + + @return file system for the given URI. + + @throws UnsupportedFileSystemException if the file system for + uri is not supported.]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + In some FileSystem implementations such as HDFS metadata + synchronization is essential to guarantee consistency of read requests + particularly in HA setting. + @throws IOException + @throws UnsupportedOperationException]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + } describing modifications + @throws IOException if an ACL could not be modified]]> + + + + + + + + } describing entries to remove + @throws IOException if an ACL could not be modified]]> + + + + + + + + + + + + + + + + + + + + + + } describing modifications, must + include entries for user, group, and others for compatibility with + permission bits. + @throws IOException if an ACL could not be modified]]> + + + + + + + } which returns each AclStatus + @throws IOException if an ACL could not be read]]> + + + + + + + + + + Refer to the HDFS extended attributes user documentation for details. + + @param path Path to modify + @param name xattr name. + @param value xattr value. + @throws IOException]]> + + + + + + + + + + + Refer to the HDFS extended attributes user documentation for details. + + @param path Path to modify + @param name xattr name. + @param value xattr value. + @param flag xattr set flag + @throws IOException]]> + + + + + + + + + Refer to the HDFS extended attributes user documentation for details. + + @param path Path to get extended attribute + @param name xattr name. + @return byte[] xattr value. + @throws IOException]]> + + + + + + + + Refer to the HDFS extended attributes user documentation for details. + + @param path Path to get extended attributes + + @return {@literal Map} describing the XAttrs of the file + or directory + @throws IOException]]> + + + + + + + + + Refer to the HDFS extended attributes user documentation for details. + + @param path Path to get extended attributes + @param names XAttr names. + @return {@literal Map} describing the XAttrs of the file + or directory + @throws IOException]]> + + + + + + + + Refer to the HDFS extended attributes user documentation for details. + + @param path Path to get extended attributes + @return {@literal Map} describing the XAttrs of the file + or directory + @throws IOException]]> + + + + + + + + + Refer to the HDFS extended attributes user documentation for details. + + @param path Path to remove extended attribute + @param name xattr name + @throws IOException]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + BlockLocation(offset: 0, length: BLOCK_SIZE, + hosts: {"host1:9866", "host2:9866, host3:9866"}) + + + And if the file is erasure-coded, each BlockLocation represents a logical + block groups. Value offset is the offset of a block group in the file and + value length is the total length of a block group. Hosts of a BlockLocation + are the datanodes that holding all the data blocks and parity blocks of a + block group. + Suppose we have a RS_3_2 coded file (3 data units and 2 parity units). + A BlockLocation example will be like: + 
    + BlockLocation(offset: 0, length: 3 * BLOCK_SIZE, hosts: {"host1:9866",
+   "host2:9866","host3:9866","host4:9866","host5:9866"})
+
    + + Please refer to + {@link FileSystem#getFileBlockLocations(FileStatus, long, long)} or + {@link FileContext#getFileBlockLocations(Path, long, long)} + for more examples.]]> +     +     + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + After a successful call, {@code buf.position()} will be advanced by the + number of bytes read and {@code buf.limit()} will be unchanged. +

    + In the case of an exception, the state of the buffer (the contents of the + buffer, the {@code buf.position()}, the {@code buf.limit()}, etc.) is + undefined, and callers should be prepared to recover from this + eventuality. +

    + Callers should use {@link StreamCapabilities#hasCapability(String)} with + {@link StreamCapabilities#PREADBYTEBUFFER} to check if the underlying + stream supports this interface, otherwise they might get a + {@link UnsupportedOperationException}. +

    + Implementations should treat 0-length requests as legitimate, and must not + signal an error upon their receipt. +

    + This does not change the current offset of a file, and is thread-safe. + + @param position position within file + @param buf the ByteBuffer to receive the results of the read operation. + @return the number of bytes read, possibly zero, or -1 if reached + end-of-stream + @throws IOException if there is some error performing the read]]> + + + + + + + + + This operation provides similar semantics to + {@link #read(long, ByteBuffer)}, the difference is that this method is + guaranteed to read data until the {@link ByteBuffer} is full, or until + the end of the data stream is reached. + + @param position position within file + @param buf the ByteBuffer to receive the results of the read operation. + @throws IOException if there is some error performing the read + @throws EOFException the end of the data was reached before + the read operation completed + @see #read(long, ByteBuffer)]]> + + + + + + + + + + + + + + + After a successful call, {@code buf.position()} will be advanced by the + number of bytes read and {@code buf.limit()} will be unchanged. +

    + In the case of an exception, the state of the buffer (the contents of the + buffer, the {@code buf.position()}, the {@code buf.limit()}, etc.) is + undefined, and callers should be prepared to recover from this + eventuality. +

    + Callers should use {@link StreamCapabilities#hasCapability(String)} with + {@link StreamCapabilities#READBYTEBUFFER} to check if the underlying + stream supports this interface, otherwise they might get a + {@link UnsupportedOperationException}. +

    + Implementations should treat 0-length requests as legitimate, and must not + signal an error upon their receipt. + + @param buf + the ByteBuffer to receive the results of the read operation. + @return the number of bytes read, possibly zero, or -1 if + reach end-of-stream + @throws IOException + if there is some error performing the read]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + setReplication of FileSystem + @param src file name + @param replication new replication + @throws IOException + @return true if successful; + false if file does not exist or is a directory]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + core-default.xml]]> + + + + + + + + + + + core-default.xml]]> + + + + + + + + + + + core-default.xml]]> + + + + + + + + + + + core-default.xml]]> + + + + + + + + + + + core-default.xml]]> + + + + + + core-default.xml]]> + + + + + + + + + + + core-default.xml]]> + + + + + + + + + + + core-default.xml]]> + + + + + + core-default.xml]]> + + + + + + core-default.xml]]> + + + + + + core-default.xml]]> + + + + + + + + core-default.xml]]> + + + + + + + + + + + + + + + + + + + + + core-default.xml]]> + + + + + + + + + +

    + A higher number here does not necessarily improve performance, especially + for object stores, where multiple threads may be attempting to create an FS + instance for the same URI. +

    + Default value: {@value}.]]> +     + + + +

    + Default value: {@value}.]]> +     +     + + + + core-default.xml]]> + + + + + + core-default.xml]]> + + + + + + core-default.xml]]> + + + + + + core-default.xml]]> + + + + + + + + + + + core-default.xml]]> + + + + + + + + + + + core-default.xml]]> + + + + + + + + + + + core-default.xml]]> + + + + + + + + + + + + + + + + core-default.xml]]> + + + + + + + + + + + core-default.xml]]> + + + + + + + + + + + core-default.xml]]> + + + + + + + + + + + core-default.xml]]> + + + + + + + + + + + core-default.xml]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + core-default.xml]]> + + + + + + + + + + + core-default.xml]]> + + + + + + + + + + + core-default.xml]]> + + + + + + core-default.xml]]> + + + + + + + + + + + core-default.xml]]> + + + + + + + + + + + core-default.xml]]> + + + + + + + + + + + + + + + + + + + + + + + core-default.xml]]> + + + + + + + + + + + core-default.xml]]> + + + + + + + + + + + core-default.xml]]> + + + + + + + + + + + core-default.xml]]> + + + + + + + + + + + core-default.xml]]> + + + + + + + + + + + core-default.xml]]> + + + + + + + + + + + + + + + + + + + + + core-default.xml]]> + + + + + + + + + + + core-default.xml]]> + + + + + + + + + + + core-default.xml]]> + + + + + + + + + + + core-default.xml]]> + + + + + + + + + + + core-default.xml]]> + + + + + + + + + + + core-default.xml]]> + + + + + + + + + + + + + + + + + + + + + + core-default.xml]]> + + + + + + + + core-default.xml]]> + + + + + + core-default.xml]]> + + + + + + + + + + + core-default.xml]]> + + + + + + core-default.xml]]> + + + + + + core-default.xml]]> + + + + + + core-default.xml]]> + + + + + + core-default.xml]]> + + + + + + core-default.xml]]> + + + + + + + + core-default.xml]]> + + + + + + core-default.xml]]> + + + + + + core-default.xml]]> + + + + + + core-default.xml]]> + + + + + + core-default.xml]]> + + + + + + + + + + + core-default.xml]]> + + + + + + + + + + + core-default.xml]]> + + + + + + core-default.xml]]> + + + + + + core-default.xml]]> + + + + + + core-default.xml]]> + + + + + + core-default.xml]]> + + + + + + core-default.xml]]> + + + + + + core-default.xml]]> + + + + + + core-default.xml]]> + + + + + + core-default.xml]]> + + + + + + + + + + core-default.xml]]> + + + + + + + + + + + core-default.xml]]> + + + + + + + + + + + core-default.xml]]> + + + + + + + + + + + + + + + + + core-default.xml]]> + + + + + + + + core-default.xml]]> + + + + + + core-default.xml]]> + + + + + + core-default.xml]]> + + + + + + + + + + + + + + + + core-default.xml]]> + + + + + + core-default.xml]]> + + + + + + + + + + + core-default.xml]]> + + + + + + + + + + + core-default.xml]]> + + + + + + + + + + + core-default.xml]]> + + + + + + + + + + + core-default.xml]]> + + + + + + + + + + + core-default.xml]]> + + + + + + + + + + + core-default.xml]]> + + + + + + + + + + + + + core-default.xml]]> + + + + + + + + + + + core-default.xml]]> + + + + + + + + + + + core-default.xml]]> + + + + + + + + + + + core-default.xml]]> + + + + + + core-default.xml]]> + + + + + + + + core-default.xml]]> + + + + + + + + core-default.xml]]> + + + + + + + + core-default.xml]]> + + + + + + + + + + + core-default.xml]]> + + + + + + core-default.xml]]> + + + + + + + + core-default.xml]]> + + + + + + core-default.xml]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + core-default.xml]]> + + + + + + + + core-default.xml]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + EnumSet.of(CreateFlag.CREATE, CreateFlag.APPEND) + +

    + + Use the CreateFlag as follows: +

      +
    1. CREATE - to create a file if it does not exist, + else throw FileAlreadyExists.
      2. +
    2. APPEND - to append to a file if it exists, + else throw FileNotFoundException.
      3. +
    3. OVERWRITE - to truncate a file if it exists, + else throw FileNotFoundException.
      4. +
    4. CREATE|APPEND - to create a file if it does not exist, + else append to an existing file.
      5. +
    5. CREATE|OVERWRITE - to create a file if it does not exist, + else overwrite an existing file.
      6. +
    6. SYNC_BLOCK - to force closed blocks to the disk device. + In addition {@link Syncable#hsync()} should be called after each write, + if true synchronous behavior is required.
      7. +
    7. LAZY_PERSIST - Create the block on transient storage (RAM) if + available.
      8. +
    8. APPEND_NEWBLOCK - Append data to a new block instead of end of the last + partial block.
      9. +
    + + Following combinations are not valid and will result in + {@link HadoopIllegalArgumentException}: +
      +
    1. APPEND|OVERWRITE
      2. +
    2. CREATE|APPEND|OVERWRITE
      3. +
    ]]> +     +     + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + absOrFqPath is not supported. + @throws IOException If the file system for absOrFqPath could + not be instantiated.]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + defaultFsUri is not supported]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + NewWdir can be one of: +
      +
    • relative path: "foo/bar";
      • +
    • absolute without scheme: "/foo/bar"
      • +
    • fully qualified with scheme: "xx://auth/foo/bar"
      • +
    +
    + Illegal WDs: +
      +
    • relative with scheme: "xx:foo/bar"
      • +
    • non existent directory
      • +
    ]]> +     +     + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + f does not exist + @throws AccessControlException if access denied + @throws IOException If an IO Error occurred + + Exceptions applicable to file systems accessed over RPC: + @throws RpcClientException If an exception occurred in the RPC client + @throws RpcServerException If an exception occurred in the RPC server + @throws UnexpectedServerException If server implementation throws + undeclared exception to RPC server + + RuntimeExceptions: + @throws InvalidPathException If path f is not valid]]> + + + + + + + + + + + + + + + + + + + + +
  • Progress - to report progress on the operation - default null +
  • Permission - umask is applied against permission: default is + FsPermissions:getDefault() + +
  • CreateParent - create missing parent path; default is to not + to create parents +
  • The defaults for the following are SS defaults of the file + server implementing the target path. Not all parameters make sense + for all kinds of file system - eg. localFS ignores Blocksize, + replication, checksum +
      +
    • BufferSize - buffersize used in FSDataOutputStream +
    • Blocksize - block size for file blocks +
    • ReplicationFactor - replication for blocks +
    • ChecksumParam - Checksum parameters. server default is used + if not specified. +
    + + + @return {@link FSDataOutputStream} for created file + + @throws AccessControlException If access is denied + @throws FileAlreadyExistsException If file f already exists + @throws FileNotFoundException If parent of f does not exist + and createParent is false + @throws ParentNotDirectoryException If parent of f is not a + directory. + @throws UnsupportedFileSystemException If file system for f is + not supported + @throws IOException If an I/O error occurred + + Exceptions applicable to file systems accessed over RPC: + @throws RpcClientException If an exception occurred in the RPC client + @throws RpcServerException If an exception occurred in the RPC server + @throws UnexpectedServerException If server implementation throws + undeclared exception to RPC server + + RuntimeExceptions: + @throws InvalidPathException If path f is not valid]]> + + + + + + + + + + + + + + + + + + + + + dir already + exists + @throws FileNotFoundException If parent of dir does not exist + and createParent is false + @throws ParentNotDirectoryException If parent of dir is not a + directory + @throws UnsupportedFileSystemException If file system for dir + is not supported + @throws IOException If an I/O error occurred + + Exceptions applicable to file systems accessed over RPC: + @throws RpcClientException If an exception occurred in the RPC client + @throws UnexpectedServerException If server implementation throws + undeclared exception to RPC server + + RuntimeExceptions: + @throws InvalidPathException If path dir is not valid]]> + + + + + + + + + + + f does not exist + @throws UnsupportedFileSystemException If file system for f is + not supported + @throws IOException If an I/O error occurred + + Exceptions applicable to file systems accessed over RPC: + @throws RpcClientException If an exception occurred in the RPC client + @throws RpcServerException If an exception occurred in the RPC server + @throws UnexpectedServerException If server implementation throws + undeclared exception to RPC server + + RuntimeExceptions: + @throws InvalidPathException If path f is invalid]]> + + + + + + + + + + f does not exist + @throws UnsupportedFileSystemException If file system for f + is not supported + @throws IOException If an I/O error occurred + + Exceptions applicable to file systems accessed over RPC: + @throws RpcClientException If an exception occurred in the RPC client + @throws RpcServerException If an exception occurred in the RPC server + @throws UnexpectedServerException If server implementation throws + undeclared exception to RPC server]]> + + + + + + + + + + + f does not exist + @throws UnsupportedFileSystemException If file system for f is + not supported + @throws IOException If an I/O error occurred + + Exceptions applicable to file systems accessed over RPC: + @throws RpcClientException If an exception occurred in the RPC client + @throws RpcServerException If an exception occurred in the RPC server + @throws UnexpectedServerException If server implementation throws + undeclared exception to RPC server]]> + + + + + + + + + + + +
  • Fails if path is a directory. +
  • Fails if path does not exist. +
  • Fails if path is not closed. +
  • Fails if new size is greater than current size. + + @param f The path to the file to be truncated + @param newLength The size the file is to be truncated to + + @return true if the file has been truncated to the desired + newLength and is immediately available to be reused for + write operations such as append, or + false if a background process of adjusting the length of + the last block has been started, and clients should wait for it to + complete before proceeding with further file updates. + + @throws AccessControlException If access is denied + @throws FileNotFoundException If file f does not exist + @throws UnsupportedFileSystemException If file system for f is + not supported + @throws IOException If an I/O error occurred + + Exceptions applicable to file systems accessed over RPC: + @throws RpcClientException If an exception occurred in the RPC client + @throws RpcServerException If an exception occurred in the RPC server + @throws UnexpectedServerException If server implementation throws + undeclared exception to RPC server]]> + + + + + + + + + + f does not exist + @throws IOException If an I/O error occurred + + Exceptions applicable to file systems accessed over RPC: + @throws RpcClientException If an exception occurred in the RPC client + @throws RpcServerException If an exception occurred in the RPC server + @throws UnexpectedServerException If server implementation throws + undeclared exception to RPC server]]> + + + + + + + + + + + + + + +
  • Fails if src is a file and dst is a directory. +
  • Fails if src is a directory and dst is a file. +
  • Fails if the parent of dst does not exist or is a file. + +

    + If OVERWRITE option is not passed as an argument, rename fails if the dst + already exists. +

    + If OVERWRITE option is passed as an argument, rename overwrites the dst if + it is a file or an empty directory. Rename fails if dst is a non-empty + directory. +

    + Note that atomicity of rename is dependent on the file system + implementation. Please refer to the file system documentation for details +

    + + @param src path to be renamed + @param dst new path after rename + + @throws AccessControlException If access is denied + @throws FileAlreadyExistsException If dst already exists and + options has {@link Options.Rename#OVERWRITE} + option false. + @throws FileNotFoundException If src does not exist + @throws ParentNotDirectoryException If parent of dst is not a + directory + @throws UnsupportedFileSystemException If file system for src + and dst is not supported + @throws IOException If an I/O error occurred + + Exceptions applicable to file systems accessed over RPC: + @throws RpcClientException If an exception occurred in the RPC client + @throws RpcServerException If an exception occurred in the RPC server + @throws UnexpectedServerException If server implementation throws + undeclared exception to RPC server]]> + + + + + + + + + + + f does not exist + @throws UnsupportedFileSystemException If file system for f + is not supported + @throws IOException If an I/O error occurred + + Exceptions applicable to file systems accessed over RPC: + @throws RpcClientException If an exception occurred in the RPC client + @throws RpcServerException If an exception occurred in the RPC server + @throws UnexpectedServerException If server implementation throws + undeclared exception to RPC server]]> + + + + + + + + + + + + f does not exist + @throws UnsupportedFileSystemException If file system for f is + not supported + @throws IOException If an I/O error occurred + + Exceptions applicable to file systems accessed over RPC: + @throws RpcClientException If an exception occurred in the RPC client + @throws RpcServerException If an exception occurred in the RPC server + @throws UnexpectedServerException If server implementation throws + undeclared exception to RPC server + + RuntimeExceptions: + @throws HadoopIllegalArgumentException If username or + groupname is invalid.]]> + + + + + + + + + + + + f does not exist + @throws UnsupportedFileSystemException If file system for f is + not supported + @throws IOException If an I/O error occurred + + Exceptions applicable to file systems accessed over RPC: + @throws RpcClientException If an exception occurred in the RPC client + @throws RpcServerException If an exception occurred in the RPC server + @throws UnexpectedServerException If server implementation throws + undeclared exception to RPC server]]> + + + + + + + + + f does not exist + @throws IOException If an I/O error occurred + + Exceptions applicable to file systems accessed over RPC: + @throws RpcClientException If an exception occurred in the RPC client + @throws RpcServerException If an exception occurred in the RPC server + @throws UnexpectedServerException If server implementation throws + undeclared exception to RPC server]]> + + + + + + + + + + + f does not exist + @throws UnsupportedFileSystemException If file system for f is + not supported + @throws IOException If an I/O error occurred + + Exceptions applicable to file systems accessed over RPC: + @throws RpcClientException If an exception occurred in the RPC client + @throws RpcServerException If an exception occurred in the RPC server + @throws UnexpectedServerException If server implementation throws + undeclared exception to RPC server]]> + + + + + + + + + + f does not exist + @throws UnsupportedFileSystemException If file system for f is + not supported + @throws IOException If an I/O error occurred + + Exceptions applicable to file systems accessed over RPC: + @throws RpcClientException If an exception occurred in the RPC client + @throws RpcServerException If an exception occurred in the RPC server + @throws UnexpectedServerException If server implementation throws + undeclared exception to RPC server]]> + + + + + + + + + + + + + + + + + f does not exist + @throws UnsupportedFileSystemException If file system for f is + not supported + @throws IOException If an I/O error occurred]]> + + + + + + + + + + f does not exist + @throws UnsupportedFileSystemException If file system for f is + not supported + @throws IOException If the given path does not refer to a symlink + or an I/O error occurred]]> + + + + + + + + + + f does not exist + @throws UnsupportedFileSystemException If file system for f is + not supported + @throws IOException If an I/O error occurred + + Exceptions applicable to file systems accessed over RPC: + @throws RpcClientException If an exception occurred in the RPC client + @throws RpcServerException If an exception occurred in the RPC server + @throws UnexpectedServerException If server implementation throws + undeclared exception to RPC server]]> + + + + + + + + + + + + + + + Given a path referring to a symlink of form: + + {@literal <---}X{@literal --->} + fs://host/A/B/link + {@literal <-----}Y{@literal ----->} + + In this path X is the scheme and authority that identify the file system, + and Y is the path leading up to the final path component "link". If Y is + a symlink itself then let Y' be the target of Y and X' be the scheme and + authority of Y'. Symlink targets may: + + 1. Fully qualified URIs + + fs://hostX/A/B/file Resolved according to the target file system. + + 2. Partially qualified URIs (eg scheme but no host) + + fs:///A/B/file Resolved according to the target file system. Eg resolving + a symlink to hdfs:///A results in an exception because + HDFS URIs must be fully qualified, while a symlink to + file:///A will not since Hadoop's local file systems + require partially qualified URIs. + + 3. Relative paths + + path Resolves to [Y'][path]. Eg if Y resolves to hdfs://host/A and path + is "../B/file" then [Y'][path] is hdfs://host/B/file + + 4. Absolute paths + + path Resolves to [X'][path]. Eg if Y resolves hdfs://host/A/B and path + is "/file" then [X][path] is hdfs://host/file + + + @param target the target of the symbolic link + @param link the path to be created that points to target + @param createParent if true then missing parent dirs are created if + false then parent must exist + + + @throws AccessControlException If access is denied + @throws FileAlreadyExistsException If file link already exists + @throws FileNotFoundException If target does not exist + @throws ParentNotDirectoryException If parent of link is not a + directory. + @throws UnsupportedFileSystemException If file system for + target or link is not supported + @throws IOException If an I/O error occurred]]> + + + + + + + + + + f does not exist + @throws UnsupportedFileSystemException If file system for f is + not supported + @throws IOException If an I/O error occurred + + Exceptions applicable to file systems accessed over RPC: + @throws RpcClientException If an exception occurred in the RPC client + @throws RpcServerException If an exception occurred in the RPC server + @throws UnexpectedServerException If server implementation throws + undeclared exception to RPC server]]> + + + + + + + + + + + + + + + + + f does not exist + @throws UnsupportedFileSystemException If file system for f is + not supported + @throws IOException If an I/O error occurred + + Exceptions applicable to file systems accessed over RPC: + @throws RpcClientException If an exception occurred in the RPC client + @throws RpcServerException If an exception occurred in the RPC server + @throws UnexpectedServerException If server implementation throws + undeclared exception to RPC server]]> + + + + + + + + f is + not supported + @throws IOException If an I/O error occurred + + Exceptions applicable to file systems accessed over RPC: + @throws RpcClientException If an exception occurred in the RPC client + @throws RpcServerException If an exception occurred in the RPC server + @throws UnexpectedServerException If server implementation throws + undeclared exception to RPC server]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + } describing + modifications + @throws IOException if an ACL could not be modified]]> + + + + + + + + } describing entries + to remove + @throws IOException if an ACL could not be modified]]> + + + + + + + + + + + + + + + + + + + + + + } describing + modifications, must include entries for user, group, and others for + compatibility with permission bits. + @throws IOException if an ACL could not be modified]]> + + + + + + + } which returns + each AclStatus + @throws IOException if an ACL could not be read]]> + + + + + + + + + + Refer to the HDFS extended attributes user documentation for details. + + @param path Path to modify + @param name xattr name. + @param value xattr value. + @throws IOException]]> + + + + + + + + + + + Refer to the HDFS extended attributes user documentation for details. + + @param path Path to modify + @param name xattr name. + @param value xattr value. + @param flag xattr set flag + @throws IOException]]> + + + + + + + + + Refer to the HDFS extended attributes user documentation for details. + + @param path Path to get extended attribute + @param name xattr name. + @return byte[] xattr value. + @throws IOException]]> + + + + + + + + Refer to the HDFS extended attributes user documentation for details. + + @param path Path to get extended attributes + @return Map{@literal <}String, byte[]{@literal >} describing the XAttrs + of the file or directory + @throws IOException]]> + + + + + + + + + Refer to the HDFS extended attributes user documentation for details. + + @param path Path to get extended attributes + @param names XAttr names. + @return Map{@literal <}String, byte[]{@literal >} describing the XAttrs + of the file or directory + @throws IOException]]> + + + + + + + + + Refer to the HDFS extended attributes user documentation for details. + + @param path Path to remove extended attribute + @param name xattr name + @throws IOException]]> + + + + + + + + Refer to the HDFS extended attributes user documentation for details. + + @param path Path to get extended attributes + @return List{@literal <}String{@literal >} of the XAttr names of the + file or directory + @throws IOException]]> + + + + + + + Exceptions applicable to file systems accessed over RPC: + @throws RpcClientException If an exception occurred in the RPC client + @throws RpcServerException If an exception occurred in the RPC server + @throws UnexpectedServerException If server implementation throws + undeclared exception to RPC server]]> + + + + + + + + Exceptions applicable to file systems accessed over RPC: + @throws RpcClientException If an exception occurred in the RPC client + @throws RpcServerException If an exception occurred in the RPC server + @throws UnexpectedServerException If server implementation throws + undeclared exception to RPC server]]> + + + + + + + + + Exceptions applicable to file systems accessed over RPC: + @throws RpcClientException If an exception occurred in the RPC client + @throws RpcServerException If an exception occurred in the RPC server + @throws UnexpectedServerException If server implementation throws + undeclared exception to RPC server]]> + + + + + + + + Exceptions applicable to file systems accessed over RPC: + @throws RpcClientException If an exception occurred in the RPC client + @throws RpcServerException If an exception occurred in the RPC server + @throws UnexpectedServerException If server implementation throws + undeclared exception to RPC server]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Path Names + + The Hadoop file system supports a URI namespace and URI names. This enables + multiple types of file systems to be referenced using fully-qualified URIs. + Two common Hadoop file system implementations are +

      +
    • the local file system: file:///path +
    • the HDFS file system: hdfs://nnAddress:nnPort/path +
    + + The Hadoop file system also supports additional naming schemes besides URIs. + Hadoop has the concept of a default file system, which implies a + default URI scheme and authority. This enables slash-relative names + relative to the default FS, which are more convenient for users and + application writers. The default FS is typically set by the user's + environment, though it can also be manually specified. +

    + + Hadoop also supports working-directory-relative names, which are paths + relative to the current working directory (similar to Unix). The working + directory can be in a different file system than the default FS. +

    + Thus, Hadoop path names can be specified as one of the following: +

      +
    • a fully-qualified URI: scheme://authority/path (e.g. + hdfs://nnAddress:nnPort/foo/bar) +
    • a slash-relative name: path relative to the default file system (e.g. + /foo/bar) +
    • a working-directory-relative name: path relative to the working dir (e.g. + foo/bar) +
    + Relative paths with scheme (scheme:foo/bar) are illegal. + +

    Role of FileContext and Configuration Defaults

    + + The FileContext is the analogue of per-process file-related state in Unix. It + contains two properties: + +
      +
    • the default file system (for resolving slash-relative names) +
    • the umask (for file permissions) +
    + In general, these properties are obtained from the default configuration file + in the user's environment (see {@link Configuration}). + + Further file system properties are specified on the server-side. File system + operations default to using these server-side defaults unless otherwise + specified. +

    + The file system related server-side defaults are: +

      +
    • the home directory (default is "/user/userName") +
    • the initial wd (only for local fs) +
    • replication factor +
    • block size +
    • buffer size +
    • encryptDataTransfer +
    • checksum option. (checksumType and bytesPerChecksum) +
    + +

    Example Usage

    + + Example 1: use the default config read from the $HADOOP_CONFIG/core.xml. + Unspecified values come from core-defaults.xml in the release jar. +
      +
    • myFContext = FileContext.getFileContext(); // uses the default config + // which has your default FS +
    • myFContext.create(path, ...); +
    • myFContext.setWorkingDir(path); +
    • myFContext.open (path, ...); +
    • ... +
    + Example 2: Get a FileContext with a specific URI as the default FS +
      +
    • myFContext = FileContext.getFileContext(URI); +
    • myFContext.create(path, ...); +
    • ... +
    + Example 3: FileContext with local file system as the default +
      +
    • myFContext = FileContext.getLocalFSFileContext(); +
    • myFContext.create(path, ...); +
    • ... +
    + Example 4: Use a specific config, ignoring $HADOOP_CONFIG + Generally you should not need use a config unless you are doing +
      +
    • configX = someConfigSomeOnePassedToYou; +
    • myFContext = getFileContext(configX); // configX is not changed, + // is passed down +
    • myFContext.create(path, ...); +
    • ... +
    ]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + This implementation throws an UnsupportedOperationException. + + @return the protocol scheme for this FileSystem. + @throws UnsupportedOperationException if the operation is unsupported + (default).]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
  • + If the configuration has the property + {@code "fs.$SCHEME.impl.disable.cache"} set to true, + a new instance will be created, initialized with the supplied URI and + configuration, then returned without being cached. +
    • +
  • + If the there is a cached FS instance matching the same URI, it will + be returned. +
    • +
  • + Otherwise: a new FS instance will be created, initialized with the + configuration and URI, cached and returned to the caller. +
    • + + @throws IOException if the FileSystem cannot be instantiated.]]> +     +     + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + if f == null : + result = null + elif f.getLen() {@literal <=} start: + result = [] + else result = [ locations(FS, b) for b in blocks(FS, p, s, s+l)] + + This call is most helpful with and distributed filesystem + where the hostnames of machines that contain blocks of the given file + can be determined. + + The default implementation returns an array containing one element: + 
    + BlockLocation( { "localhost:9866" },  { "localhost" }, 0, file.getLen())
+
    + + In HDFS, if file is three-replicated, the returned array contains + elements like: + 
    + BlockLocation(offset: 0, length: BLOCK_SIZE,
+   hosts: {"host1:9866", "host2:9866, host3:9866"})
+ BlockLocation(offset: BLOCK_SIZE, length: BLOCK_SIZE,
+   hosts: {"host2:9866", "host3:9866, host4:9866"})
+
    + + And if a file is erasure-coded, the returned BlockLocation are logical + block groups. + + Suppose we have a RS_3_2 coded file (3 data units and 2 parity units). + 1. If the file size is less than one stripe size, say 2 * CELL_SIZE, then + there will be one BlockLocation returned, with 0 offset, actual file size + and 4 hosts (2 data blocks and 2 parity blocks) hosting the actual blocks. + 3. If the file size is less than one group size but greater than one + stripe size, then there will be one BlockLocation returned, with 0 offset, + actual file size with 5 hosts (3 data blocks and 2 parity blocks) hosting + the actual blocks. + 4. If the file size is greater than one group size, 3 * BLOCK_SIZE + 123 + for example, then the result will be like: + 
    + BlockLocation(offset: 0, length: 3 * BLOCK_SIZE, hosts: {"host1:9866",
+   "host2:9866","host3:9866","host4:9866","host5:9866"})
+ BlockLocation(offset: 3 * BLOCK_SIZE, length: 123, hosts: {"host1:9866",
+   "host4:9866", "host5:9866"})
+
    + + @param file FilesStatus to get data from + @param start offset into the given file + @param len length for which to get locations for + @throws IOException IO failure]]> +     +     + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Important: the default implementation is not atomic + @param f path to use for create + @throws IOException IO failure]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
  • Fails if src is a file and dst is a directory.
    • +
  • Fails if src is a directory and dst is a file.
    • +
  • Fails if the parent of dst does not exist or is a file.
    • + +

    + If OVERWRITE option is not passed as an argument, rename fails + if the dst already exists. +

    + If OVERWRITE option is passed as an argument, rename overwrites + the dst if it is a file or an empty directory. Rename fails if dst is + a non-empty directory. +

    + Note that atomicity of rename is dependent on the file system + implementation. Please refer to the file system documentation for + details. This default implementation is non atomic. +

    + This method is deprecated since it is a temporary method added to + support the transition from FileSystem to FileContext for user + applications. + + @param src path to be renamed + @param dst new path after rename + @throws FileNotFoundException src path does not exist, or the parent + path of dst does not exist. + @throws FileAlreadyExistsException dest path exists and is a file + @throws ParentNotDirectoryException if the parent path of dest is not + a directory + @throws IOException on failure]]> + + + + + + + + +

  • Fails if path is a directory.
    • +
  • Fails if path does not exist.
    • +
  • Fails if path is not closed.
    • +
  • Fails if new size is greater than current size.
    • + + @param f The path to the file to be truncated + @param newLength The size the file is to be truncated to + + @return true if the file has been truncated to the desired + newLength and is immediately available to be reused for + write operations such as append, or + false if a background process of adjusting the length of + the last block has been started, and clients should wait for it to + complete before proceeding with further file updates. + @throws IOException IO failure + @throws UnsupportedOperationException if the operation is unsupported + (default).]]> +     +     + + + + + + + + + + + + + + + + + + + + +
  • Clean shutdown of the JVM cannot be guaranteed.
    • +
  • The time to shut down a FileSystem will depends on the number of + files to delete. For filesystems where the cost of checking + for the existence of a file/directory and the actual delete operation + (for example: object stores) is high, the time to shutdown the JVM can be + significantly extended by over-use of this feature.
    • +
  • Connectivity problems with a remote filesystem may delay shutdown + further, and may cause the files to not be deleted.
    • + + @param f the path to delete. + @return true if deleteOnExit is successful, otherwise false. + @throws IOException IO failure]]> +     +     + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Does not guarantee to return the List of files/directories status in a + sorted order. +

    + Will not return null. Expect IOException upon access error. + @param f given path + @return the statuses of the files/directories in the given patch + @throws FileNotFoundException when the path does not exist + @throws IOException see specific implementation]]> + + + + + + + + + + + + + + + + + Does not guarantee to return the List of files/directories status in a + sorted order. + + @param f + a path name + @param filter + the user-supplied path filter + @return an array of FileStatus objects for the files under the given path + after applying the filter + @throws FileNotFoundException when the path does not exist + @throws IOException see specific implementation]]> + + + + + + + + + Does not guarantee to return the List of files/directories status in a + sorted order. + + @param files + a list of paths + @return a list of statuses for the files under the given paths after + applying the filter default Path filter + @throws FileNotFoundException when the path does not exist + @throws IOException see specific implementation]]> + + + + + + + + + + Does not guarantee to return the List of files/directories status in a + sorted order. + + @param files + a list of paths + @param filter + the user-supplied path filter + @return a list of statuses for the files under the given paths after + applying the filter + @throws FileNotFoundException when the path does not exist + @throws IOException see specific implementation]]> + + + + + + + Return all the files that match filePattern and are not checksum + files. Results are sorted by their names. + +

    + A filename pattern is composed of regular characters and + special pattern matching characters, which are: + +

    +
    +
    +
    ? +
    Matches any single character. + +

    +

    * +
    Matches zero or more characters. + +

    +

    [abc] +
    Matches a single character from character set + {a,b,c}. + +

    +

    [a-b] +
    Matches a single character from the character range + {a...b}. Note that character a must be + lexicographically less than or equal to character b. + +

    +

    [^a] +
    Matches a single character that is not from character set or range + {a}. Note that the ^ character must occur + immediately to the right of the opening bracket. + +

    +

    \c +
    Removes (escapes) any special meaning of character c. + +

    +

    {ab,cd} +
    Matches a string from the string set {ab, cd} + +

    +

    {ab,c{de,fh}} +
    Matches a string from the string set {ab, cde, cfh} + +
    +
    +
    + + @param pathPattern a glob specifying a path pattern + + @return an array of paths that match the path pattern + @throws IOException IO failure]]> +     +     + + + + + + + + + + + + + + f does not exist + @throws IOException If an I/O error occurred]]> + + + + + + + + + f does not exist + @throws IOException if any I/O error occurred]]> + + + + + + + + p does not exist + @throws IOException if any I/O error occurred]]> + + + + + + + + + + If the path is a directory, + if recursive is false, returns files in the directory; + if recursive is true, return files in the subtree rooted at the path. + If the path is a file, return the file's status and block locations. + + @param f is the path + @param recursive if the subdirectories need to be traversed recursively + + @return an iterator that traverses statuses of the files + + @throws FileNotFoundException when the path does not exist; + @throws IOException see specific implementation]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + undefined. + @throws IOException IO failure]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + In some FileSystem implementations such as HDFS metadata + synchronization is essential to guarantee consistency of read requests + particularly in HA setting. + @throws IOException + @throws UnsupportedOperationException]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Refer to the HDFS extended attributes user documentation for details. + + @param path Path to modify + @param name xattr name. + @param value xattr value. + @throws IOException IO failure + @throws UnsupportedOperationException if the operation is unsupported + (default outcome).]]> + + + + + + + + + + + Refer to the HDFS extended attributes user documentation for details. + + @param path Path to modify + @param name xattr name. + @param value xattr value. + @param flag xattr set flag + @throws IOException IO failure + @throws UnsupportedOperationException if the operation is unsupported + (default outcome).]]> + + + + + + + + + Refer to the HDFS extended attributes user documentation for details. + + @param path Path to get extended attribute + @param name xattr name. + @return byte[] xattr value. + @throws IOException IO failure + @throws UnsupportedOperationException if the operation is unsupported + (default outcome).]]> + + + + + + + + Refer to the HDFS extended attributes user documentation for details. + + @param path Path to get extended attributes + @return Map describing the XAttrs of the file or directory + @throws IOException IO failure + @throws UnsupportedOperationException if the operation is unsupported + (default outcome).]]> + + + + + + + + + Refer to the HDFS extended attributes user documentation for details. + + @param path Path to get extended attributes + @param names XAttr names. + @return Map describing the XAttrs of the file or directory + @throws IOException IO failure + @throws UnsupportedOperationException if the operation is unsupported + (default outcome).]]> + + + + + + + + Refer to the HDFS extended attributes user documentation for details. + + @param path Path to get extended attributes + @return List{@literal } of the XAttr names of the file or directory + @throws IOException IO failure + @throws UnsupportedOperationException if the operation is unsupported + (default outcome).]]> + + + + + + + + + Refer to the HDFS extended attributes user documentation for details. + + @param path Path to remove extended attribute + @param name xattr name + @throws IOException IO failure + @throws UnsupportedOperationException if the operation is unsupported + (default outcome).]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + This is a default method which is intended to be overridden by + subclasses. The default implementation returns an empty storage statistics + object.

    + + @return The StorageStatistics for this FileSystem instance. + Will never be null.]]> +     +     + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + All user code that may potentially use the Hadoop Distributed + File System should be written to use a FileSystem object or its + successor, {@link FileContext}. + +

    + The local implementation is {@link LocalFileSystem} and distributed + implementation is DistributedFileSystem. There are other implementations + for object stores and (outside the Apache Hadoop codebase), + third party filesystems. +

    + Notes +

      +
    1. The behaviour of the filesystem is + + specified in the Hadoop documentation. + However, the normative specification of the behavior of this class is + actually HDFS: if HDFS does not behave the way these Javadocs or + the specification in the Hadoop documentations define, assume that + the documentation is incorrect. +
      2. +
    2. The term {@code FileSystem} refers to an instance of this class.
      3. +
    3. The acronym "FS" is used as an abbreviation of FileSystem.
      4. +
    4. The term {@code filesystem} refers to the distributed/local filesystem + itself, rather than the class used to interact with it.
      5. +
    5. The term "file" refers to a file in the remote filesystem, + rather than instances of {@code java.io.File}.
      6. +
    + + This is a carefully evolving class. + New methods may be marked as Unstable or Evolving for their initial release, + as a warning that they are new and may change based on the + experience of use in applications. +

    + Important note for developers +

    + If you are making changes here to the public API or protected methods, + you must review the following subclasses and make sure that + they are filtering/passing through new methods as appropriate. +

    + + {@link FilterFileSystem}: methods are passed through. If not, + then {@code TestFilterFileSystem.MustNotImplement} must be + updated with the unsupported interface. + Furthermore, if the new API's support is probed for via + {@link #hasPathCapability(Path, String)} then + {@link FilterFileSystem#hasPathCapability(Path, String)} + must return false, always. +

    + {@link ChecksumFileSystem}: checksums are created and + verified. +

    + {@code TestHarFileSystem} will need its {@code MustNotImplement} + interface updated. +

    + + There are some external places your changes will break things. + Do co-ordinate changes here. +

    + + HBase: HBoss +

    + Hive: HiveShim23 + {@code shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java}]]> +     +     + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + } caller's + environment variables to use for expansion + @return String[] with absolute path to new jar in position 0 and + unexpanded wild card entry path in position 1 + @throws IOException if there is an I/O error while writing the jar file]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + FilterFileSystem contains + some other file system, which it uses as + its basic file system, possibly transforming + the data along the way or providing additional + functionality. The class FilterFileSystem + itself simply overrides all methods of + FileSystem with versions that + pass all requests to the contained file + system. Subclasses of FilterFileSystem + may further override some of these methods + and may also provide additional methods + and fields.]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Return type on the {@link #build()} call. + @param type of builder itself.]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + -1 + if there is no more data because the end of the stream has been + reached]]> + + + + + + + + + + length bytes have been read. + + @param position position in the input stream to seek + @param buffer buffer into which data is read + @param offset offset into the buffer in which data is written + @param length the number of bytes to read + @throws IOException IO problems + @throws EOFException If the end of stream is reached while reading. + If an exception is thrown an undetermined number + of bytes in the buffer may have been written.]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + // Don't + if (fs instanceof FooFileSystem) { + FooFileSystem fs = (FooFileSystem) fs; + OutputStream out = dfs.createFile(path) + .optionA() + .optionB("value") + .cache() + .build() + } else if (fs instanceof BarFileSystem) { + ... + } + + // Do + OutputStream out = fs.createFile(path) + .permission(perm) + .bufferSize(bufSize) + .opt("foofs:option.a", true) + .opt("foofs:option.b", "value") + .opt("barfs:cache", true) + .must("foofs:cache", true) + .must("barfs:cache-size", 256 * 1024 * 1024) + .build(); + + + If the option is not related to the file system, the option will be ignored. + If the option is must, but not supported by the file system, a + {@link IllegalArgumentException} will be thrown.]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + path is invalid]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + @return file]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +

    + The interface extends {@link IOStatisticsSource} so that there is no + need to cast an instance to see if is a source of statistics. + However, implementations MAY return null for their actual statistics.]]> +     +     + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + ListingBatch behaves similar to a Future, in that getting the result via + {@link #get()} will throw an Exception if there was a failure.]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + and the scheme is null, and the authority + is null. + + @return whether the path is absolute and the URI has no scheme nor + authority parts]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + true if and only if pathname + should be included]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Warning: Not all filesystems satisfy the thread-safety requirement. + @param position position within file + @param buffer destination buffer + @param offset offset in the buffer + @param length number of bytes to read + @return actual number of bytes read; -1 means "none" + @throws IOException IO problems.]]> + + + + + + + + + + Warning: Not all filesystems satisfy the thread-safety requirement. + @param position position within file + @param buffer destination buffer + @param offset offset in the buffer + @param length number of bytes to read + @throws IOException IO problems. + @throws EOFException the end of the data was reached before + the read operation completed]]> + + + + + + + + Warning: Not all filesystems satisfy the thread-safety requirement. + @param position position within file + @param buffer destination buffer + @throws IOException IO problems. + @throws EOFException the end of the data was reached before + the read operation completed]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Note    : Returned list is not sorted in any given order, + due to reliance on Java's {@link File#list()} API.)]]> + +     + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + XAttr is byte[], this class is to + covert byte[] to some kind of string representation or convert back. + String representation is convenient for display and input. For example + display in screen as shell response and json response, input as http + or shell parameter.]]> + + + + + + + + + + + + + + + + + + + + + +     + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + @return ftp]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + A {@link FileSystem} backed by an FTP client provided by Apache Commons Net. +

    ]]> +     +     + +     + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + This is for reporting and testing.]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + These are low-cost per-instance statistics provided by any Hadoop + I/O class instance. +

    + Consult the filesystem specification document for the requirements + of an implementation of this interface.]]> + + + + + + + + + + + + + + + + + + + + + + + Exceptions are caught and downgraded to debug logging. + @param source source of statistics. + @return a string for logging.]]> + + + + + + + + + + + + + + + + + + + Whenever this object's toString() method is called, it evaluates the + statistics. +

    + This is designed to affordable to use in log statements. + @param source source of statistics -may be null. + @return an object whose toString() operation returns the current values.]]> + + + + + + + Whenever this object's toString() method is called, it evaluates the + statistics. +

    + This is for use in log statements where for the cost of creation + of this entry is low; it is affordable to use in log statements. + @param statistics statistics to stringify -may be null. + @return an object whose toString() operation returns the current values.]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + It is serializable so that frameworks which can use java serialization + to propagate data (Spark, Flink...) can send the statistics + back. For this reason, TreeMaps are explicitly used as field types, + even though IDEs can recommend use of Map instead. + For security reasons, untrusted java object streams should never be + deserialized. If for some reason this is required, use + {@link #requiredSerializationClasses()} to get the list of classes + used when deserializing instances of this object. +

    +

    + It is annotated for correct serializations with jackson2. +

    ]]> +     + + + + + + + + + This is not an atomic option. +

    + The instance can be serialized, and its + {@code toString()} method lists all the values. + @param statistics statistics + @return a snapshot of the current values.]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + It can be used to accrue values so as to dynamically update + the mean. If so, know that there is no synchronization + on the methods. +

    +

    + If a statistic has 0 samples then it is considered to be empty. +

    +

    + All 'empty' statistics are equivalent, independent of the sum value. +

    +

    + For non-empty statistics, sum and sample values must match + for equality. +

    +

    + It is serializable and annotated for correct serializations with jackson2. +

    +

    + Thread safety. The operations to add/copy sample data, are thread safe. +

    +
      +
    1. {@link #add(MeanStatistic)}
      2. +
    2. {@link #addSample(long)}
      3. +
    3. {@link #clear()}
      4. +
    4. {@link #setSamplesAndSum(long, long)}
      5. +
    5. {@link #set(MeanStatistic)}
      6. +
    6. {@link #setSamples(long)} and {@link #setSum(long)}
      7. +
    +

    + So is the {@link #mean()} method. This ensures that when + used to aggregated statistics, the aggregate value and sample + count are set and evaluated consistently. +

    +

    + Other methods marked as synchronized because Findbugs overreacts + to the idea that some operations to update sum and sample count + are synchronized, but that things like equals are not. +

    ]]> +     +     + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + names)}: {@value}.]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + When adding new common statistic name constants, please make them unique. + By convention: +

    +
      +
    • the name of the constants are uppercase, words separated by + underscores.
      • +
    • the value of the constants are lowercase of the constant names.
      • +
    ]]> +     +     + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + When adding new common statistic name constants, please make them unique. + By convention, they are implicitly unique: +
      +
    • + The name of the constants are uppercase, words separated by + underscores. +
      • +
    • + The value of the constants are lowercase of the constant names. +
      • +
    ]]> +     +     + +     + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Since these methods are often vendor- or device-specific, operators + may implement this interface in order to achieve fencing. +

    + Fencing is configured by the operator as an ordered list of methods to + attempt. Each method will be tried in turn, and the next in the list + will only be attempted if the previous one fails. See {@link NodeFencer} + for more information. +

    + If an implementation also implements {@link Configurable} then its + setConf method will be called upon instantiation.]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + state (e.g ACTIVE/STANDBY) as well as + some additional information. + + @throws AccessControlException + if access is denied. + @throws IOException + if other errors happen + @see HAServiceStatus]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + hadoop.http.filter.initializers. + +

      +
    • StaticUserWebFilter - An authorization plugin that makes all +users a static configured user. +
    ]]> +     +     + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + public class IntArrayWritable extends ArrayWritable { + public IntArrayWritable() { + super(IntWritable.class); + } + } + ]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + o is a ByteWritable with the same value.]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + the class of the item + @param conf the configuration to store + @param item the object to be stored + @param keyName the name of the key to use + @throws IOException : forwards Exceptions from the underlying + {@link Serialization} classes.]]> + + + + + + + + + the class of the item + @param conf the configuration to use + @param keyName the name of the key to use + @param itemClass the class of the item + @return restored object + @throws IOException : forwards Exceptions from the underlying + {@link Serialization} classes.]]> + + + + + + + + + the class of the item + @param conf the configuration to use + @param items the objects to be stored + @param keyName the name of the key to use + @throws IndexOutOfBoundsException if the items array is empty + @throws IOException : forwards Exceptions from the underlying + {@link Serialization} classes.]]> + + + + + + + + + the class of the item + @param conf the configuration to use + @param keyName the name of the key to use + @param itemClass the class of the item + @return restored object + @throws IOException : forwards Exceptions from the underlying + {@link Serialization} classes.]]> + + + + + DefaultStringifier offers convenience methods to store/load objects to/from + the configuration. + + @param the class of the objects to stringify]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + o is a DoubleWritable with the same value.]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + value argument is null or + its size is zero, the elementType argument must not be null. If + the argument value's size is bigger than zero, the argument + elementType is not be used. + + @param value + @param elementType]]> + + + + + value should not be null + or empty. + + @param value]]> + + + + + + + + + + + + + + value and elementType. If the value argument + is null or its size is zero, the elementType argument must not be + null. If the argument value's size is bigger than zero, the + argument elementType is not be used. + + @param value + @param elementType]]> + + + + + + + + + + + + + + + + + + + o is an EnumSetWritable with the same value, + or both are null.]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + o is a FloatWritable with the same value.]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + When two sequence files, which have same Key type but different Value + types, are mapped out to reduce, multiple Value types is not allowed. + In this case, this class can help you wrap instances with different types. +

    + +

    + Compared with ObjectWritable, this class is much more effective, + because ObjectWritable will append the class declaration as a String + into the output file in every Key-Value pair. +

    + +

    + Generic Writable implements {@link Configurable} interface, so that it will be + configured by the framework. The configuration is passed to the wrapped objects + implementing {@link Configurable} interface before deserialization. +

    + + how to use it:
    + 1. Write your own class, such as GenericObject, which extends GenericWritable.
    + 2. Implements the abstract method getTypes(), defines + the classes which will be wrapped in GenericObject in application. + Attention: this classes defined in getTypes() method, must + implement Writable interface. +

    + + The code looks like this: + 
    + public class GenericObject extends GenericWritable {
+ 
+   private static Class[] CLASSES = {
+               ClassType1.class, 
+               ClassType2.class,
+               ClassType3.class,
+               };
+
+   protected Class[] getTypes() {
+       return CLASSES;
+   }
+
+ }
+
    + + @since Nov 8, 2006]]> +     +     + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + o is a IntWritable with the same value.]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + closes the input and output streams + at the end. + + @param in InputStrem to read from + @param out OutputStream to write to + @param conf the Configuration object]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + ignore any {@link Throwable} or + null pointers. Must only be used for cleanup in exception handlers. + + @param log the log to record problems to at debug level. Can be null. + @param closeables the objects to close + @deprecated use {@link #cleanupWithLogger(Logger, java.io.Closeable...)} + instead]]> + + + + + + + ignore any {@link Throwable} or + null pointers. Must only be used for cleanup in exception handlers. + + @param logger the log to record problems to at debug level. Can be null. + @param closeables the objects to close]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + This is better than File#listDir because it does not ignore IOExceptions. + + @param dir The directory to list. + @param filter If non-null, the filter to use when listing + this directory. + @return The list of files in the directory. + + @throws IOException On I/O error]]> + + + + + + + + Borrowed from Uwe Schindler in LUCENE-5588 + @param fileToSync the file to fsync]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + o is a LongWritable with the same value.]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + A map is a directory containing two files, the data file, + containing all keys and values in the map, and a smaller index + file, containing a fraction of the keys. The fraction is determined by + {@link Writer#getIndexInterval()}. + +

    The index file is read entirely into memory. Thus key implementations + should try to keep themselves small. + +

    Map files are created by adding entries in-order. To maintain a large + database, perform updates by copying the previous version of a database and + merging in a sorted change list, to create a new version of the database in + a new file. Sorting large change lists can be done with {@link + SequenceFile.Sorter}.]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + o is an MD5Hash whose digest contains the + same values.]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + className by first finding + it in the specified conf. If the specified conf is null, + try load it directly.]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + A {@link Comparator} that operates directly on byte representations of + objects. +

    + @param + @see DeserializerComparator]]> +     + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + SequenceFiles are flat files consisting of binary key/value + pairs. + +

    SequenceFile provides {@link SequenceFile.Writer}, + {@link SequenceFile.Reader} and {@link Sorter} classes for writing, + reading and sorting respectively.

    + + There are three SequenceFile Writers based on the + {@link CompressionType} used to compress key/value pairs: +
      +
    1. + Writer : Uncompressed records. +
      2. +
    2. + RecordCompressWriter : Record-compressed files, only compress + values. +
      3. +
    3. + BlockCompressWriter : Block-compressed files, both keys & + values are collected in 'blocks' + separately and compressed. The size of + the 'block' is configurable. +
    + +

    The actual compression algorithm used to compress key and/or values can be + specified by using the appropriate {@link CompressionCodec}.

    + +

    The recommended way is to use the static createWriter methods + provided by the SequenceFile to chose the preferred format.

    + +

    The {@link SequenceFile.Reader} acts as the bridge and can read any of the + above SequenceFile formats.

    + +

    SequenceFile Formats

    + +

    Essentially there are 3 different formats for SequenceFiles + depending on the CompressionType specified. All of them share a + common header described below. + +

    +
      +
    • + version - 3 bytes of magic header SEQ, followed by 1 byte of actual + version number (e.g. SEQ4 or SEQ6) +
      • +
    • + keyClassName -key class +
      • +
    • + valueClassName - value class +
      • +
    • + compression - A boolean which specifies if compression is turned on for + keys/values in this file. +
      • +
    • + blockCompression - A boolean which specifies if block-compression is + turned on for keys/values in this file. +
      • +
    • + compression codec - CompressionCodec class which is used for + compression of keys and/or values (if compression is + enabled). +
      • +
    • + metadata - {@link Metadata} for this file. +
      • +
    • + sync - A sync marker to denote end of the header. +
      • +
    + +
    Uncompressed SequenceFile Format
    +
      +
    • + Header +
      • +
    • + Record +
        +
      • Record length
        • +
      • Key length
        • +
      • Key
        • +
      • Value
        • +
      +
      • +
    • + A sync-marker every few 100 kilobytes or so. +
      • +
    + +
    Record-Compressed SequenceFile Format
    +
      +
    • + Header +
      • +
    • + Record +
        +
      • Record length
        • +
      • Key length
        • +
      • Key
        • +
      • Compressed Value
        • +
      +
      • +
    • + A sync-marker every few 100 kilobytes or so. +
      • +
    + +
    Block-Compressed SequenceFile Format
    +
      +
    • + Header +
      • +
    • + Record Block +
        +
      • Uncompressed number of records in the block
        • +
      • Compressed key-lengths block-size
        • +
      • Compressed key-lengths block
        • +
      • Compressed keys block-size
        • +
      • Compressed keys block
        • +
      • Compressed value-lengths block-size
        • +
      • Compressed value-lengths block
        • +
      • Compressed values block-size
        • +
      • Compressed values block
        • +
      +
      • +
    • + A sync-marker every block. +
      • +
    + +

    The compressed blocks of key lengths and value lengths consist of the + actual lengths of individual keys/values encoded in ZeroCompressedInteger + format.

    + + @see CompressionCodec]]> +     +     + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + o is a ShortWritable with the same value.]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + the class of the objects to stringify]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + position. Note that this + method avoids using the converter or doing String instantiation + @return the Unicode scalar value at position or -1 + if the position is invalid or points to a + trailing byte]]> + + + + + + + + + + what in the backing + buffer, starting as position start. The starting + position is measured in bytes and the return value is in + terms of byte position in the buffer. The backing buffer is + not converted to a string for this operation. + @return byte position of the first occurrence of the search + string in the UTF-8 buffer or -1 if not found]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Note: For performance reasons, this call does not clear the + underlying byte array that is retrievable via {@link #getBytes()}. + In order to free the byte-array memory, call {@link #set(byte[])} + with an empty byte array (For example, new byte[0]).]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + o is a Text with the same contents.]]> + + + + + + + + + + + + + + + + + + + + + + + + + replace is true, then + malformed input is replaced with the + substitution character, which is U+FFFD. Otherwise the + method throws a MalformedInputException.]]> + + + + + + + + + + + + + + + replace is true, then + malformed input is replaced with the + substitution character, which is U+FFFD. Otherwise the + method throws a MalformedInputException. + @return ByteBuffer: bytes stores at ByteBuffer.array() + and length is ByteBuffer.limit()]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + In + addition, it provides methods for string traversal without converting the + byte array to a string.

    Also includes utilities for + serializing/deserialing a string, coding/decoding a string, checking if a + byte array contains valid UTF8 code, calculating the length of an encoded + string.]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + This is useful when a class may evolve, so that instances written by the + old version of the class may still be processed by the new version. To + handle this situation, {@link #readFields(DataInput)} + implementations should catch {@link VersionMismatchException}.]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + o is a VIntWritable with the same value.]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + o is a VLongWritable with the same value.]]> + + + + + + + + + + + + + + + + + + + + + + + + out. + + @param out DataOuput to serialize this object into. + @throws IOException]]> + + + + + + + in. + +

    For efficiency, implementations should attempt to re-use storage in the + existing object where possible.

    + + @param in DataInput to deseriablize this object from. + @throws IOException]]> +     + + + Any key or value type in the Hadoop Map-Reduce + framework implements this interface.

    + +

    Implementations typically implement a static read(DataInput) + method which constructs a new instance, calls {@link #readFields(DataInput)} + and returns the instance.

    + +

    Example:

    + 
    +     public class MyWritable implements Writable {
+       // Some data
+       private int counter;
+       private long timestamp;
+
+       // Default constructor to allow (de)serialization
+       MyWritable() { }
+
+       public void write(DataOutput out) throws IOException {
+         out.writeInt(counter);
+         out.writeLong(timestamp);
+       }
+
+       public void readFields(DataInput in) throws IOException {
+         counter = in.readInt();
+         timestamp = in.readLong();
+       }
+
+       public static MyWritable read(DataInput in) throws IOException {
+         MyWritable w = new MyWritable();
+         w.readFields(in);
+         return w;
+       }
+     }
+
    ]]> +     + + + + + + + + WritableComparables can be compared to each other, typically + via Comparators. Any type which is to be used as a + key in the Hadoop Map-Reduce framework should implement this + interface.

    + +

    Note that hashCode() is frequently used in Hadoop to partition + keys. It's important that your implementation of hashCode() returns the same + result across different instances of the JVM. Note also that the default + hashCode() implementation in Object does not + satisfy this property.

    + +

    Example:

    + 
    +     public class MyWritableComparable implements
+      WritableComparable{@literal } {
+       // Some data
+       private int counter;
+       private long timestamp;
+       
+       public void write(DataOutput out) throws IOException {
+         out.writeInt(counter);
+         out.writeLong(timestamp);
+       }
+       
+       public void readFields(DataInput in) throws IOException {
+         counter = in.readInt();
+         timestamp = in.readLong();
+       }
+       
+       public int compareTo(MyWritableComparable o) {
+         int thisValue = this.value;
+         int thatValue = o.value;
+         return (thisValue < thatValue ? -1 : (thisValue==thatValue ? 0 : 1));
+       }
+
+       public int hashCode() {
+         final int prime = 31;
+         int result = 1;
+         result = prime * result + counter;
+         result = prime * result + (int) (timestamp ^ (timestamp >>> 32));
+         return result
+       }
+     }
+
    ]]> +     +     + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + The default implementation reads the data into two {@link + WritableComparable}s (using {@link + Writable#readFields(DataInput)}, then calls {@link + #compare(WritableComparable,WritableComparable)}.]]> + + + + + + + The default implementation uses the natural ordering, calling {@link + Comparable#compareTo(Object)}.]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + This base implementation uses the natural ordering. To define alternate + orderings, override {@link #compare(WritableComparable,WritableComparable)}. + +

    One may optimize compare-intensive operations by overriding + {@link #compare(byte[],int,int,byte[],int,int)}. Static utility methods are + provided to assist in optimized implementations of this method.]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Enum type + @param in DataInput to read from + @param enumType Class type of Enum + @return Enum represented by String read from DataInput + @throws IOException]]> + + + + + + + + + + + + + + + + len number of bytes in input streamin + @param in input stream + @param len number of bytes to skip + @throws IOException when skipped less number of bytes]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + CompressionCodec for which to get the + Compressor + @param conf the Configuration object which contains confs for creating or reinit the compressor + @return Compressor for the given + CompressionCodec from the pool or a new one]]> + + + + + + + + + CompressionCodec for which to get the + Decompressor + @return Decompressor for the given + CompressionCodec the pool or a new one]]> + + + + + + Compressor to be returned to the pool]]> + + + + + + Decompressor to be returned to the + pool]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Codec aliases are case insensitive. +

    + The code alias is the short class name (without the package name). + If the short class name ends with 'Codec', then there are two aliases for + the codec, the complete short class name and the short class name without + the 'Codec' ending. For example for the 'GzipCodec' codec class name the + alias are 'gzip' and 'gzipcodec'. + + @param codecName the canonical class name of the codec + @return the codec object]]> + + + + + + + Codec aliases are case insensitive. +

    + The code alias is the short class name (without the package name). + If the short class name ends with 'Codec', then there are two aliases for + the codec, the complete short class name and the short class name without + the 'Codec' ending. For example for the 'GzipCodec' codec class name the + alias are 'gzip' and 'gzipcodec'. + + @param codecName the canonical class name of the codec + @return the codec class]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Implementations are assumed to be buffered. This permits clients to + reposition the underlying input stream then call {@link #resetState()}, + without having to also synchronize client buffers.]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + true indicating that more input data is required. + + @param b Input data + @param off Start offset + @param len Length]]> + + + + + true if the input data buffer is empty and + #setInput() should be called in order to provide more input.]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + true if the end of the compressed + data output stream has been reached.]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + true indicating that more input data is required. + (Both native and non-native versions of various Decompressors require + that the data passed in via b[] remain unmodified until + the caller is explicitly notified--via {@link #needsInput()}--that the + buffer may be safely modified. With this requirement, an extra + buffer-copy can be avoided.) + + @param b Input data + @param off Start offset + @param len Length]]> + + + + + true if the input data buffer is empty and + {@link #setInput(byte[], int, int)} should be called to + provide more input. + + @return true if the input data buffer is empty and + {@link #setInput(byte[], int, int)} should be called in + order to provide more input.]]> + + + + + + + + + + + + + true if a preset dictionary is needed for decompression. + @return true if a preset dictionary is needed for decompression]]> + + + + + true if the end of the decompressed + data output stream has been reached. Indicates a concatenated data stream + when finished() returns true and {@link #getRemaining()} + returns a positive value. finished() will be reset with the + {@link #reset()} method. + @return true if the end of the decompressed + data output stream has been reached.]]> + + + + + + + + + + + + + + true and getRemaining() returns a positive value. If + {@link #finished()} returns true and getRemaining() returns + a zero value, indicates that the end of data stream has been reached and + is not a concatenated data stream. + @return The number of bytes remaining in the compressed data buffer.]]> + + + + + true and {@link #getRemaining()} returns a positive value, + reset() is called before processing of the next data stream in the + concatenated data stream. {@link #finished()} will be reset and will + return false when reset() is called.]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + io.compression.codecs = org.apache.hadoop.io.compress.PassthroughCodec + io.compress.passthrough.extension = .gz + + + Note: this is not a Splittable codec: it doesn't know the + capabilities of the passed in stream. It should be possible to + extend this in a subclass: the inner classes are marked as protected + to enable this. Do not retrofit splitting to this class..]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +

  • "none" - No compression. +
  • "lzo" - LZO compression. +
  • "gz" - GZIP compression. + ]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
  • Block Compression. +
  • Named meta data blocks. +
  • Sorted or unsorted keys. +
  • Seek by key or by file offset. + + The memory footprint of a TFile includes the following: +
      +
    • Some constant overhead of reading or writing a compressed block. +
        +
      • Each compressed block requires one compression/decompression codec for + I/O. +
      • Temporary space to buffer the key. +
      • Temporary space to buffer the value (for TFile.Writer only). Values are + chunk encoded, so that we buffer at most one chunk of user data. By default, + the chunk buffer is 1MB. Reading chunked value does not require additional + memory. +
      +
    • TFile index, which is proportional to the total number of Data Blocks. + The total amount of memory needed to hold the index can be estimated as + (56+AvgKeySize)*NumBlocks. +
    • MetaBlock index, which is proportional to the total number of Meta + Blocks.The total amount of memory needed to hold the index for Meta Blocks + can be estimated as (40+AvgMetaBlockName)*NumMetaBlock. +
    +

    + The behavior of TFile can be customized by the following variables through + Configuration: +

      +
    • tfile.io.chunk.size: Value chunk size. Integer (in bytes). Default + to 1MB. Values of the length less than the chunk size is guaranteed to have + known value length in read time (See + {@link TFile.Reader.Scanner.Entry#isValueLengthKnown()}). +
    • tfile.fs.output.buffer.size: Buffer size used for + FSDataOutputStream. Integer (in bytes). Default to 256KB. +
    • tfile.fs.input.buffer.size: Buffer size used for + FSDataInputStream. Integer (in bytes). Default to 256KB. +
    +

    + Suggestions on performance optimization. +

      +
    • Minimum block size. We recommend a setting of minimum block size between + 256KB to 1MB for general usage. Larger block size is preferred if files are + primarily for sequential access. However, it would lead to inefficient random + access (because there are more data to decompress). Smaller blocks are good + for random access, but require more memory to hold the block index, and may + be slower to create (because we must flush the compressor stream at the + conclusion of each data block, which leads to an FS I/O flush). Further, due + to the internal caching in Compression codec, the smallest possible block + size would be around 20KB-30KB. +
    • The current implementation does not offer true multi-threading for + reading. The implementation uses FSDataInputStream seek()+read(), which is + shown to be much faster than positioned-read call in single thread mode. + However, it also means that if multiple threads attempt to access the same + TFile (using multiple scanners) simultaneously, the actual I/O is carried out + sequentially even if they access different DFS blocks. +
    • Compression codec. Use "none" if the data is not very compressable (by + compressable, I mean a compression ratio at least 2:1). Generally, use "lzo" + as the starting point for experimenting. "gz" overs slightly better + compression ratio over "lzo" but requires 4x CPU to compress and 2x CPU to + decompress, comparing to "lzo". +
    • File system buffering, if the underlying FSDataInputStream and + FSDataOutputStream is already adequately buffered; or if applications + reads/writes keys and values in large buffers, we can reduce the sizes of + input/output buffering in TFile layer by setting the configuration parameters + "tfile.fs.input.buffer.size" and "tfile.fs.output.buffer.size". +
    + + Some design rationale behind TFile can be found at Hadoop-3315.]]> + + + + + + + + + + + Utils#writeVLong(out, n). + + @param out + output stream + @param n + The integer to be encoded + @throws IOException + @see Utils#writeVLong(DataOutput, long)]]> + + + + + + + + +
  • if n in [-32, 127): encode in one byte with the actual value. + Otherwise, +
  • if n in [-20*2^8, 20*2^8): encode in two bytes: byte[0] = n/256 - 52; + byte[1]=n&0xff. Otherwise, +
  • if n IN [-16*2^16, 16*2^16): encode in three bytes: byte[0]=n/2^16 - + 88; byte[1]=(n>>8)&0xff; byte[2]=n&0xff. Otherwise, +
  • if n in [-8*2^24, 8*2^24): encode in four bytes: byte[0]=n/2^24 - 112; + byte[1] = (n>>16)&0xff; byte[2] = (n>>8)&0xff; + byte[3]=n&0xff. + Otherwise: +
  • if n in [-2^31, 2^31): encode in five bytes: byte[0]=-125; byte[1] = + (n>>24)&0xff; byte[2]=(n>>16)&0xff; + byte[3]=(n>>8)&0xff; byte[4]=n&0xff; +
  • if n in [-2^39, 2^39): encode in six bytes: byte[0]=-124; byte[1] = + (n>>32)&0xff; byte[2]=(n>>24)&0xff; + byte[3]=(n>>16)&0xff; byte[4]=(n>>8)&0xff; + byte[5]=n&0xff +
  • if n in [-2^47, 2^47): encode in seven bytes: byte[0]=-123; byte[1] = + (n>>40)&0xff; byte[2]=(n>>32)&0xff; + byte[3]=(n>>24)&0xff; byte[4]=(n>>16)&0xff; + byte[5]=(n>>8)&0xff; byte[6]=n&0xff; +
  • if n in [-2^55, 2^55): encode in eight bytes: byte[0]=-122; byte[1] = + (n>>48)&0xff; byte[2] = (n>>40)&0xff; + byte[3]=(n>>32)&0xff; byte[4]=(n>>24)&0xff; byte[5]= + (n>>16)&0xff; byte[6]=(n>>8)&0xff; byte[7]=n&0xff; +
  • if n in [-2^63, 2^63): encode in nine bytes: byte[0]=-121; byte[1] = + (n>>54)&0xff; byte[2] = (n>>48)&0xff; + byte[3] = (n>>40)&0xff; byte[4]=(n>>32)&0xff; + byte[5]=(n>>24)&0xff; byte[6]=(n>>16)&0xff; byte[7]= + (n>>8)&0xff; byte[8]=n&0xff; + + + @param out + output stream + @param n + the integer number + @throws IOException]]> + + + + + + + (int)Utils#readVLong(in). + + @param in + input stream + @return the decoded integer + @throws IOException + + @see Utils#readVLong(DataInput)]]> + + + + + + + +
  • if (FB >= -32), return (long)FB; +
  • if (FB in [-72, -33]), return (FB+52)<<8 + NB[0]&0xff; +
  • if (FB in [-104, -73]), return (FB+88)<<16 + + (NB[0]&0xff)<<8 + NB[1]&0xff; +
  • if (FB in [-120, -105]), return (FB+112)<<24 + (NB[0]&0xff) + <<16 + (NB[1]&0xff)<<8 + NB[2]&0xff; +
  • if (FB in [-128, -121]), return interpret NB[FB+129] as a signed + big-endian integer. + + @param in + input stream + @return the decoded long integer. + @throws IOException]]> + + + + + + + + + + + + + + + + + + + + + + + + Type of the input key. + @param list + The list + @param key + The input key. + @param cmp + Comparator for the key. + @return The index to the desired element if it exists; or list.size() + otherwise.]]> + + + + + + + + + Type of the input key. + @param list + The list + @param key + The input key. + @param cmp + Comparator for the key. + @return The index to the desired element if it exists; or list.size() + otherwise.]]> + + + + + + + + Type of the input key. + @param list + The list + @param key + The input key. + @return The index to the desired element if it exists; or list.size() + otherwise.]]> + + + + + + + + Type of the input key. + @param list + The list + @param key + The input key. + @return The index to the desired element if it exists; or list.size() + otherwise.]]> + + + + + + + + + + + + + + + + + An experimental {@link Serialization} for Java {@link Serializable} classes. +

    + @see JavaSerializationComparator]]> +     +     + + + + + + + + + A {@link RawComparator} that uses a {@link JavaSerialization} + {@link Deserializer} to deserialize objects that are then compared via + their {@link Comparable} interfaces. +

    + @param + @see JavaSerialization]]> +     +     + + + + + + + + + + + + + +This package provides a mechanism for using different serialization frameworks +in Hadoop. The property "io.serializations" defines a list of +{@link org.apache.hadoop.io.serializer.Serialization}s that know how to create +{@link org.apache.hadoop.io.serializer.Serializer}s and +{@link org.apache.hadoop.io.serializer.Deserializer}s. +

    + +

    +To add a new serialization framework write an implementation of +{@link org.apache.hadoop.io.serializer.Serialization} and add its name to the +"io.serializations" property. +

    ]]> +     +     + + + + + + + + + + + + + + + + + + + avro.reflect.pkgs or implement + {@link AvroReflectSerializable} interface.]]> + + + + + + + + + + + + + + + + + + + + + + + + + + +This package provides Avro serialization in Hadoop. This can be used to +serialize/deserialize Avro types in Hadoop. +

    + +

    +Use {@link org.apache.hadoop.io.serializer.avro.AvroSpecificSerialization} for +serialization of classes generated by Avro's 'specific' compiler. +

    + +

    +Use {@link org.apache.hadoop.io.serializer.avro.AvroReflectSerialization} for +other classes. +{@link org.apache.hadoop.io.serializer.avro.AvroReflectSerialization} work for +any class which is either in the package list configured via +{@link org.apache.hadoop.io.serializer.avro.AvroReflectSerialization#AVRO_REFLECT_PACKAGES} +or implement {@link org.apache.hadoop.io.serializer.avro.AvroReflectSerializable} +interface. +

    ]]> +     +     + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Implementations of this interface consume the {@link MetricsRecord} generated + from {@link MetricsSource}. It registers with {@link MetricsSystem} which + periodically pushes the {@link MetricsRecord} to the sink using + {@link #putMetrics(MetricsRecord)} method. If the implementing class also + implements {@link Closeable}, then the MetricsSystem will close the sink when + it is stopped.]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + the actual type of the source object + @param source object to register + @return the source object + @exception MetricsException]]> + + + + + + + + the actual type of the source object + @param source object to register + @param name of the source. Must be unique or null (then extracted from + the annotations of the source object.) + @param desc the description of the source (or null. See above.) + @return the source object + @exception MetricsException]]> + + + + + + + + + + + + + + + + + + + + +
  • {@link MetricsSource} generate and update metrics information.
    • +
  • {@link MetricsSink} consume the metrics information
    • + + + {@link MetricsSource} and {@link MetricsSink} register with the metrics + system. Implementations of {@link MetricsSystem} polls the + {@link MetricsSource}s periodically and pass the {@link MetricsRecord}s to + {@link MetricsSink}.]]> +     +     + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +     + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + } (aggregate). + Filter out entries that don't have at least minSamples. + + @return a map of peer DataNode Id to the average latency to that + node seen over the measurement period.]]> + + + + + + + + + + + This class maintains a group of rolling average metrics. It implements the + algorithm of rolling average, i.e. a number of sliding windows are kept to + roll over and evict old subsets of samples. Each window has a subset of + samples in a stream, where sub-sum and sub-total are collected. All sub-sums + and sub-totals in all windows will be aggregated to final-sum and final-total + used to compute final average, which is called rolling average. +

    ]]> +     +     + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +     + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + This class is a metrics sink that uses + {@link org.apache.hadoop.fs.FileSystem} to write the metrics logs. Every + roll interval a new directory will be created under the path specified by the + basepath property. All metrics will be logged to a file in the + current interval's directory in a file named <hostname>.log, where + <hostname> is the name of the host on which the metrics logging + process is running. The base path is set by the + <prefix>.sink.<instance>.basepath property. The + time zone used to create the current interval's directory name is GMT. If + the basepath property isn't specified, it will default to + "/tmp", which is the temp directory on whatever default file + system is configured for the cluster.

    + +

    The <prefix>.sink.<instance>.ignore-error + property controls whether an exception is thrown when an error is encountered + writing a log file. The default value is true. When set to + false, file errors are quietly swallowed.

    + +

    The roll-interval property sets the amount of time before + rolling the directory. The default value is 1 hour. The roll interval may + not be less than 1 minute. The property's value should be given as + number unit, where number is an integer value, and + unit is a valid unit. Valid units are minute, hour, + and day. The units are case insensitive and may be abbreviated or + plural. If no units are specified, hours are assumed. For example, + "2", "2h", "2 hour", and + "2 hours" are all valid ways to specify two hours.

    + +

    The roll-offset-interval-millis property sets the upper + bound on a random time interval (in milliseconds) that is used to delay + before the initial roll. All subsequent rolls will happen an integer + number of roll intervals after the initial roll, hence retaining the original + offset. The purpose of this property is to insert some variance in the roll + times so that large clusters using this sink on every node don't cause a + performance impact on HDFS by rolling simultaneously. The default value is + 30000 (30s). When writing to HDFS, as a rule of thumb, the roll offset in + millis should be no less than the number of sink instances times 5. + +

    The primary use of this class is for logging to HDFS. As it uses + {@link org.apache.hadoop.fs.FileSystem} to access the target file system, + however, it can be used to write to the local file system, Amazon S3, or any + other supported file system. The base path for the sink will determine the + file system used. An unqualified path will write to the default file system + set by the configuration.

    + +

    Not all file systems support the ability to append to files. In file + systems without the ability to append to files, only one writer can write to + a file at a time. To allow for concurrent writes from multiple daemons on a + single host, the source property is used to set unique headers + for the log files. The property should be set to the name of + the source daemon, e.g. namenode. The value of the + source property should typically be the same as the property's + prefix. If this property is not set, the source is taken to be + unknown.

    + +

    Instead of appending to an existing file, by default the sink + will create a new file with a suffix of ".<n>", where + n is the next lowest integer that isn't already used in a file name, + similar to the Hadoop daemon logs. NOTE: the file with the highest + sequence number is the newest file, unlike the Hadoop daemon logs.

    + +

    For file systems that allow append, the sink supports appending to the + existing file instead. If the allow-append property is set to + true, the sink will instead append to the existing file on file systems that + support appends. By default, the allow-append property is + false.

    + +

    Note that when writing to HDFS with allow-append set to true, + there is a minimum acceptable number of data nodes. If the number of data + nodes drops below that minimum, the append will succeed, but reading the + data will fail with an IOException in the DataStreamer class. The minimum + number of data nodes required for a successful append is generally 2 or + 3.

    + +

    Note also that when writing to HDFS, the file size information is not + updated until the file is closed (at the end of the interval) even though + the data is being written successfully. This is a known HDFS limitation that + exists because of the performance cost of updating the metadata. See + HDFS-5478.

    + +

    When using this sink in a secure (Kerberos) environment, two additional + properties must be set: keytab-key and + principal-key. keytab-key should contain the key by + which the keytab file can be found in the configuration, for example, + yarn.nodemanager.keytab. principal-key should + contain the key by which the principal can be found in the configuration, + for example, yarn.nodemanager.principal.]]> + + + + + + + + + + + + + + + + + + + + + + + + + CollectD StatsD plugin). +
    + To configure this plugin, you will need to add the following + entries to your hadoop-metrics2.properties file: +
    + 

    + *.sink.statsd.class=org.apache.hadoop.metrics2.sink.StatsDSink
+ [prefix].sink.statsd.server.host=
+ [prefix].sink.statsd.server.port=
+ [prefix].sink.statsd.skip.hostname=true|false (optional)
+ [prefix].sink.statsd.service.name=NameNode (name you want for service)
+
    ]]> +     +     + +     + + + + + + + + + + + + + ,name=}" + Where the {@literal and } are the supplied + parameters. + + @param serviceName + @param nameName + @param theMbean - the MBean to register + @return the named used to register the MBean]]> + + + + + + + + + ,name=}" + Where the {@literal and } are the supplied + parameters. + + @param serviceName + @param nameName + @param properties - Key value pairs to define additional JMX ObjectName + properties. + @param theMbean - the MBean to register + @return the named used to register the MBean]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + hostname or hostname:port. If + the specs string is null, defaults to localhost:defaultPort. + + @param specs server specs (see description) + @param defaultPort the default port if not specified + @return a list of InetSocketAddress objects.]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + This method is used when parts of Hadoop need know whether to apply + single rack vs multi-rack policies, such as during block placement. + Such algorithms behave differently if they are on multi-switch systems. +

    + + @return true if the mapping thinks that it is on a single switch]]> +     +     + + + + + + + + + + + + + + + + + This predicate simply assumes that all mappings not derived from + this class are multi-switch. + @param mapping the mapping to query + @return true if the base class says it is single switch, or the mapping + is not derived from this class.]]> + + + + It is not mandatory to + derive {@link DNSToSwitchMapping} implementations from it, but it is strongly + recommended, as it makes it easy for the Hadoop developers to add new methods + to this base class that are automatically picked up by all implementations. +

    + + This class does not extend the Configured + base class, and should not be changed to do so, as it causes problems + for subclasses. The constructor of the Configured calls + the {@link #setConf(Configuration)} method, which will call into the + subclasses before they have been fully constructed.]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + If a name cannot be resolved to a rack, the implementation + should return {@link NetworkTopology#DEFAULT_RACK}. This + is what the bundled implementations do, though it is not a formal requirement + + @param names the list of hosts to resolve (can be empty) + @return list of resolved network paths. + If names is empty, the returned list is also empty]]> + + + + + + + + + + + + + + + + + + + + + + + + Calling {@link #setConf(Configuration)} will trigger a + re-evaluation of the configuration settings and so be used to + set up the mapping script.]]> + + + + + + + + + + + + + + + + + + + + + This will get called in the superclass constructor, so a check is needed + to ensure that the raw mapping is defined before trying to relaying a null + configuration. + @param conf]]> + + + + + + + + + + It contains a static class RawScriptBasedMapping that performs + the work: reading the configuration parameters, executing any defined + script, handling errors and such like. The outer + class extends {@link CachedDNSToSwitchMapping} to cache the delegated + queries. +

    + This DNS mapper's {@link #isSingleSwitch()} predicate returns + true if and only if a script is defined.]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Simple {@link DNSToSwitchMapping} implementation that reads a 2 column text + file. The columns are separated by whitespace. The first column is a DNS or + IP address and the second column specifies the rack where the address maps. +

    +

    + This class uses the configuration parameter {@code + net.topology.table.file.name} to locate the mapping file. +

    +

    + Calls to {@link #resolve(List)} will look up the address as defined in the + mapping file. If no entry corresponding to the address is found, the value + {@code /default-rack} is returned. +

    ]]> +     +     + +     + + + + + + + + + + + + + + + + + + (cause==null ? null : cause.toString()) (which + typically contains the class and detail message of cause). + @param cause the cause (which is saved for later retrieval by the + {@link #getCause()} method). (A null value is + permitted, and indicates that the cause is nonexistent or + unknown.)]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + } mapping and {@literal <}groupId, groupName{@literal >} + mapping.]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + }/host@realm. + @param principalName principal name of format as described above + @return host name if the the string conforms to the above format, else null]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + } "jack" + + @param userName + @return userName without login method]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + the return type of the run method + @param action the method to execute + @return the value from the run method]]> + + + + + + + + the return type of the run method + @param action the method to execute + @return the value from the run method + @throws IOException if the action throws an IOException + @throws Error if the action throws an Error + @throws RuntimeException if the action throws a RuntimeException + @throws InterruptedException if the action throws an InterruptedException + @throws UndeclaredThrowableException if the action throws something else]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + CredentialProvider implementations must be thread safe.]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + (cause==null ? null : cause.toString()) (which + typically contains the class and detail message of cause). + @param cause the cause (which is saved for later retrieval by the + {@link #getCause()} method). (A null value is + permitted, and indicates that the cause is nonexistent or + unknown.)]]> + + + + + + + + + + + + + + does not provide the stack trace for security purposes.]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + A User-Agent String is considered to be a browser if it matches + any of the regex patterns from browser-useragent-regex; the default + behavior is to consider everything a browser that matches the following: + "^Mozilla.*,^Opera.*". Subclasses can optionally override + this method to use different behavior. + + @param userAgent The User-Agent String, or null if there isn't one + @return true if the User-Agent String refers to a browser, false if not]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + The type of the token identifier]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + T extends TokenIdentifier]]> + + + + + + + + + + DelegationTokenAuthenticatedURL. +

    + An instance of the default {@link DelegationTokenAuthenticator} will be + used.]]> + + + + + DelegationTokenAuthenticatedURL. + + @param authenticator the {@link DelegationTokenAuthenticator} instance to + use, if null the default one will be used.]]> + + + + + DelegationTokenAuthenticatedURL using the default + {@link DelegationTokenAuthenticator} class. + + @param connConfigurator a connection configurator.]]> + + + + + DelegationTokenAuthenticatedURL. + + @param authenticator the {@link DelegationTokenAuthenticator} instance to + use, if null the default one will be used. + @param connConfigurator a connection configurator.]]> + + + + + + + + + + + + The default class is {@link KerberosDelegationTokenAuthenticator} + + @return the delegation token authenticator class to use as default.]]> + + + + + + + This method is provided to enable WebHDFS backwards compatibility. + + @param useQueryString TRUE if the token is transmitted in the + URL query string, FALSE if the delegation token is transmitted + using the {@link DelegationTokenAuthenticator#DELEGATION_TOKEN_HEADER} HTTP + header.]]> + + + + + TRUE if the token is transmitted in the URL query + string, FALSE if the delegation token is transmitted using the + {@link DelegationTokenAuthenticator#DELEGATION_TOKEN_HEADER} HTTP header.]]> + + + + + + + + + + + + + + + + + + Authenticator. + + @param url the URL to connect to. Only HTTP/S URLs are supported. + @param token the authentication token being used for the user. + @return an authenticated {@link HttpURLConnection}. + @throws IOException if an IO error occurred. + @throws AuthenticationException if an authentication exception occurred.]]> + + + + + + + + + + Authenticator. If the doAs parameter is not NULL, + the request will be done on behalf of the specified doAs user. + + @param url the URL to connect to. Only HTTP/S URLs are supported. + @param token the authentication token being used for the user. + @param doAs user to do the the request on behalf of, if NULL the request is + as self. + @return an authenticated {@link HttpURLConnection}. + @throws IOException if an IO error occurred. + @throws AuthenticationException if an authentication exception occurred.]]> + + + + + + + + + + Authenticator + for authentication. + + @param url the URL to get the delegation token from. Only HTTP/S URLs are + supported. + @param token the authentication token being used for the user where the + Delegation token will be stored. + @param renewer the renewer user. + @return a delegation token. + @throws IOException if an IO error occurred. + @throws AuthenticationException if an authentication exception occurred.]]> + + + + + + + + + + + Authenticator + for authentication. + + @param url the URL to get the delegation token from. Only HTTP/S URLs are + supported. + @param token the authentication token being used for the user where the + Delegation token will be stored. + @param renewer the renewer user. + @param doAsUser the user to do as, which will be the token owner. + @return a delegation token. + @throws IOException if an IO error occurred. + @throws AuthenticationException if an authentication exception occurred.]]> + + + + + + + + + Authenticator for authentication. + + @param url the URL to renew the delegation token from. Only HTTP/S URLs are + supported. + @param token the authentication token with the Delegation Token to renew. + @throws IOException if an IO error occurred. + @throws AuthenticationException if an authentication exception occurred.]]> + + + + + + + + + + Authenticator for authentication. + + @param url the URL to renew the delegation token from. Only HTTP/S URLs are + supported. + @param token the authentication token with the Delegation Token to renew. + @param doAsUser the user to do as, which will be the token owner. + @throws IOException if an IO error occurred. + @throws AuthenticationException if an authentication exception occurred.]]> + + + + + + + + Authenticator. + + @param url the URL to cancel the delegation token from. Only HTTP/S URLs + are supported. + @param token the authentication token with the Delegation Token to cancel. + @throws IOException if an IO error occurred.]]> + + + + + + + + + Authenticator. + + @param url the URL to cancel the delegation token from. Only HTTP/S URLs + are supported. + @param token the authentication token with the Delegation Token to cancel. + @param doAsUser the user to do as, which will be the token owner. + @throws IOException if an IO error occurred.]]> + + + + DelegationTokenAuthenticatedURL is a + {@link AuthenticatedURL} sub-class with built-in Hadoop Delegation Token + functionality. +

    + The authentication mechanisms supported by default are Hadoop Simple + authentication (also known as pseudo authentication) and Kerberos SPNEGO + authentication. +

    + Additional authentication mechanisms can be supported via {@link + DelegationTokenAuthenticator} implementations. +

    + The default {@link DelegationTokenAuthenticator} is the {@link + KerberosDelegationTokenAuthenticator} class which supports + automatic fallback from Kerberos SPNEGO to Hadoop Simple authentication via + the {@link PseudoDelegationTokenAuthenticator} class. +

    + AuthenticatedURL instances are not thread-safe.]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Authenticator + for authentication. + + @param url the URL to get the delegation token from. Only HTTP/S URLs are + supported. + @param token the authentication token being used for the user where the + Delegation token will be stored. + @param renewer the renewer user. + @throws IOException if an IO error occurred. + @throws AuthenticationException if an authentication exception occurred.]]> + + + + + + + + + + + Authenticator + for authentication. + + @param url the URL to get the delegation token from. Only HTTP/S URLs are + supported. + @param token the authentication token being used for the user where the + Delegation token will be stored. + @param renewer the renewer user. + @param doAsUser the user to do as, which will be the token owner. + @throws IOException if an IO error occurred. + @throws AuthenticationException if an authentication exception occurred.]]> + + + + + + + + + + Authenticator for authentication. + + @param url the URL to renew the delegation token from. Only HTTP/S URLs are + supported. + @param token the authentication token with the Delegation Token to renew. + @throws IOException if an IO error occurred. + @throws AuthenticationException if an authentication exception occurred.]]> + + + + + + + + + + + Authenticator for authentication. + + @param url the URL to renew the delegation token from. Only HTTP/S URLs are + supported. + @param token the authentication token with the Delegation Token to renew. + @param doAsUser the user to do as, which will be the token owner. + @throws IOException if an IO error occurred. + @throws AuthenticationException if an authentication exception occurred.]]> + + + + + + + + + Authenticator. + + @param url the URL to cancel the delegation token from. Only HTTP/S URLs + are supported. + @param token the authentication token with the Delegation Token to cancel. + @throws IOException if an IO error occurred.]]> + + + + + + + + + + Authenticator. + + @param url the URL to cancel the delegation token from. Only HTTP/S URLs + are supported. + @param token the authentication token with the Delegation Token to cancel. + @param doAsUser the user to do as, which will be the token owner. + @throws IOException if an IO error occurred.]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + KerberosDelegationTokenAuthenticator provides support for + Kerberos SPNEGO authentication mechanism and support for Hadoop Delegation + Token operations. +

    + It falls back to the {@link PseudoDelegationTokenAuthenticator} if the HTTP + endpoint does not trigger a SPNEGO authentication]]> + + + + + + + + + PseudoDelegationTokenAuthenticator provides support for + Hadoop's pseudo authentication mechanism that accepts + the user name specified as a query string parameter and support for Hadoop + Delegation Token operations. +

    + This mimics the model of Hadoop Simple authentication trusting the + {@link UserGroupInformation#getCurrentUser()} value.]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + live. + @return a (snapshotted) map of blocker name->description values]]> + + + + + + + + + + + + + Do nothing if the service is null or not + in a state in which it can be/needs to be stopped. +

    + The service state is checked before the operation begins. + This process is not thread safe. + @param service a service or null]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +

  • Any long-lived operation here will prevent the service state + change from completing in a timely manner.
    • +
  • If another thread is somehow invoked from the listener, and + that thread invokes the methods of the service (including + subclass-specific methods), there is a risk of a deadlock.
    • + + + + @param service the service that has changed.]]> +     + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +     + + + + + + + + + + + + + + + + The base implementation logs all arguments at the debug level, + then returns the passed in config unchanged.]]> + + + + + + + The action is to signal success by returning the exit code 0.]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + This method is called before {@link #init(Configuration)}; + Any non-null configuration that is returned from this operation + becomes the one that is passed on to that {@link #init(Configuration)} + operation. +

    + This permits implementations to change the configuration before + the init operation. As the ServiceLauncher only creates + an instance of the base {@link Configuration} class, it is + recommended to instantiate any subclass (such as YarnConfiguration) + that injects new resources. +

    + @param config the initial configuration build up by the + service launcher. + @param args list of arguments passed to the command line + after any launcher-specific commands have been stripped. + @return the configuration to init the service with. + Recommended: pass down the config parameter with any changes + @throws Exception any problem]]> + + + + + + + The return value becomes the exit code of the launched process. +

    + If an exception is raised, the policy is: +

      +
    1. Any subset of {@link org.apache.hadoop.util.ExitUtil.ExitException}: + the exception is passed up unmodified. +
      2. +
    2. Any exception which implements + {@link org.apache.hadoop.util.ExitCodeProvider}: + A new {@link ServiceLaunchException} is created with the exit code + and message of the thrown exception; the thrown exception becomes the + cause.
      3. +
    3. Any other exception: a new {@link ServiceLaunchException} is created + with the exit code {@link LauncherExitCodes#EXIT_EXCEPTION_THROWN} and + the message of the original exception (which becomes the cause).
      4. +
    + @return the exit code + @throws org.apache.hadoop.util.ExitUtil.ExitException an exception passed + up as the exit code and error text. + @throws Exception any exception to report. If it provides an exit code + this is used in a wrapping exception.]]> +     +     + + + The command line options will be passed down before the + {@link Service#init(Configuration)} operation is invoked via an + invocation of {@link LaunchableService#bindArgs(Configuration, List)} + After the service has been successfully started via {@link Service#start()} + the {@link LaunchableService#execute()} method is called to execute the + service. When this method returns, the service launcher will exit, using + the return code from the method as its exit option.]]> + +     + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Approximate HTTP equivalent: {@code 400 Bad Request}]]> + + + + + + approximate HTTP equivalent: Approximate HTTP equivalent: {@code 401 Unauthorized}]]> + + + + + + + + + + + Approximate HTTP equivalent: Approximate HTTP equivalent: {@code 403: Forbidden}]]> + + + + + + Approximate HTTP equivalent: {@code 404: Not Found}]]> + + + + + + Approximate HTTP equivalent: {@code 405: Not allowed}]]> + + + + + + Approximate HTTP equivalent: {@code 406: Not Acceptable}]]> + + + + + + Approximate HTTP equivalent: {@code 408: Request Timeout}]]> + + + + + + Approximate HTTP equivalent: {@code 409: Conflict}]]> + + + + + + Approximate HTTP equivalent: {@code 500 Internal Server Error}]]> + + + + + + Approximate HTTP equivalent: {@code 501: Not Implemented}]]> + + + + + + Approximate HTTP equivalent: {@code 503 Service Unavailable}]]> + + + + + + If raised, this is expected to be raised server-side and likely due + to client/server version incompatibilities. +

    + Approximate HTTP equivalent: {@code 505: Version Not Supported}]]> + + + + + + + + + + + + + + + Codes with a YARN prefix are YARN-related. +

    + Many of the exit codes are designed to resemble HTTP error codes, + squashed into a single byte. e.g 44 , "not found" is the equivalent + of 404. The various 2XX HTTP error codes aren't followed; + the Unix standard of "0" for success is used. + 

    +    0-10: general command issues
+   30-39: equivalent to the 3XX responses, where those responses are
+          considered errors by the application.
+   40-49: client-side/CLI/config problems
+   50-59: service-side problems.
+   60+  : application specific error codes
+
    ]]> +     +     + + + + + + + + + + + + + + + + + + + This uses {@link String#format(String, Object...)} + to build the formatted exception in the ENGLISH locale. +

    + If the last argument is a throwable, it becomes the cause of the exception. + It will also be used as a parameter for the format. + @param exitCode exit code + @param format format for message to use in exception + @param args list of arguments]]> + + + + + + This uses {@link String#format(String, Object...)} + to build the formatted exception in the ENGLISH locale. + @param exitCode exit code + @param cause inner cause + @param format format for message to use in exception + @param args list of arguments]]> + + + + + When caught by the ServiceLauncher, it will convert that + into a process exit code. + + The {@link #ServiceLaunchException(int, String, Object...)} constructor + generates formatted exceptions.]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +

    + This will be 0 until a call + to {@link #finished()} has been made. + @return the currently recorded duration.]]> +     + + + + + + + + + +     + + + + + + + + + + Clients and/or applications can use the provided Progressable + to explicitly report progress to the Hadoop framework. This is especially + important for operations which take significant amount of time since, + in-lieu of the reported progress, the framework has to assume that an error + has occurred and time-out the operation.

    ]]> +     +     + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Class is to be obtained + @return the correctly typed Class of the given object.]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + kill -0 command or equivalent]]> + + + + + + + + + + + + + + + + + + + ".cmd" on Windows, or ".sh" otherwise. + + @param parent File parent directory + @param basename String script file basename + @return File referencing the script in the directory]]> + + + + + + ".cmd" on Windows, or ".sh" otherwise. + + @param basename String script file basename + @return String script file name]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + IOException. + @return the path to {@link #WINUTILS_EXE} + @throws RuntimeException if the path is not resolvable]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Shell. + @return the thread that ran runCommand() that spawned this shell + or null if no thread is waiting for this shell to complete]]> + + + + + + + + + + + + Shell interface. + @param cmd shell command to execute. + @return the output of the executed command.]]> + + + + + + + + + Shell interface. + @param env the map of environment key=value + @param cmd shell command to execute. + @param timeout time in milliseconds after which script should be marked timeout + @return the output of the executed command. + @throws IOException on any problem.]]> + + + + + + + + Shell interface. + @param env the map of environment key=value + @param cmd shell command to execute. + @return the output of the executed command. + @throws IOException on any problem.]]> + + + + + Shell processes. + Iterates through a map of all currently running Shell + processes and destroys them one by one. This method is thread safe]]> + + + + + Shell objects.]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + CreateProcess synchronization object.]]> + + + + + os.name property.]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Important: caller must check for this value being null. + The lack of such checks has led to many support issues being raised. +

    + @deprecated use one of the exception-raising getter methods, + specifically {@link #getWinUtilsPath()} or {@link #getWinUtilsFile()}]]> + + + + + + + + + + + + + + Shell can be used to run shell commands like du or + df. It also offers facilities to gate commands by + time-intervals.]]> + + + + + + + + ShutdownHookManager singleton. + + @return ShutdownHookManager singleton.]]> + + + + + + + Runnable + @param priority priority of the shutdownHook.]]> + + + + + + + + + Runnable + @param priority priority of the shutdownHook + @param timeout timeout of the shutdownHook + @param unit unit of the timeout TimeUnit]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + ShutdownHookManager enables running shutdownHook + in a deterministic order, higher priority first. +

    + The JVM runs ShutdownHooks in a non-deterministic order or in parallel. + This class registers a single JVM shutdownHook and run all the + shutdownHooks registered to it (to this class) in order based on their + priority. + + Unless a hook was registered with a shutdown explicitly set through + {@link #addShutdownHook(Runnable, int, long, TimeUnit)}, + the shutdown time allocated to it is set by the configuration option + {@link CommonConfigurationKeysPublic#SERVICE_SHUTDOWN_TIMEOUT} in + {@code core-site.xml}, with a default value of + {@link CommonConfigurationKeysPublic#SERVICE_SHUTDOWN_TIMEOUT_DEFAULT} + seconds.]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Tool, is the standard for any Map-Reduce tool/application. + The tool/application should delegate the handling of + + standard command-line options to {@link ToolRunner#run(Tool, String[])} + and only handle its custom arguments.

    + +

    Here is how a typical Tool is implemented:

    + 

    +     public class MyApp extends Configured implements Tool {
+     
+       public int run(String[] args) throws Exception {
+         // Configuration processed by ToolRunner
+         Configuration conf = getConf();
+         
+         // Create a JobConf using the processed conf
+         JobConf job = new JobConf(conf, MyApp.class);
+         
+         // Process custom command-line options
+         Path in = new Path(args[1]);
+         Path out = new Path(args[2]);
+         
+         // Specify various job-specific parameters     
+         job.setJobName("my-app");
+         job.setInputPath(in);
+         job.setOutputPath(out);
+         job.setMapperClass(MyMapper.class);
+         job.setReducerClass(MyReducer.class);
+
+         // Submit the job, then poll for progress until the job is complete
+         RunningJob runningJob = JobClient.runJob(job);
+         if (runningJob.isSuccessful()) {
+           return 0;
+         } else {
+           return 1;
+         }
+       }
+       
+       public static void main(String[] args) throws Exception {
+         // Let ToolRunner handle generic command-line options 
+         int res = ToolRunner.run(new Configuration(), new MyApp(), args);
+         
+         System.exit(res);
+       }
+     }
+

    + + @see GenericOptionsParser + @see ToolRunner]]> + + + + + + + + + + + + + + Tool by {@link Tool#run(String[])}, after + parsing with the given generic arguments. Uses the given + Configuration, or builds one if null. + + Sets the Tool's configuration with the possibly modified + version of the conf. + + @param conf Configuration for the Tool. + @param tool Tool to run. + @param args command-line arguments to the tool. + @return exit code of the {@link Tool#run(String[])} method.]]> + + + + + + + + Tool with its Configuration. + + Equivalent to run(tool.getConf(), tool, args). + + @param tool Tool to run. + @param args command-line arguments to the tool. + @return exit code of the {@link Tool#run(String[])} method.]]> + + + + + + + + + + + + + + + + + ToolRunner can be used to run classes implementing + Tool interface. It works in conjunction with + {@link GenericOptionsParser} to parse the + + generic hadoop command line arguments and modifies the + Configuration of the Tool. The + application-specific options are passed along without being modified. +

    + + @see Tool + @see GenericOptionsParser]]> +     +     + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +     + + + + + + + + + + + this filter. + @param nbHash The number of hash function to consider. + @param hashType type of the hashing function (see + {@link org.apache.hadoop.util.hash.Hash}).]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Bloom filter, as defined by Bloom in 1970. +

    + The Bloom filter is a data structure that was introduced in 1970 and that has been adopted by + the networking research community in the past decade thanks to the bandwidth efficiencies that it + offers for the transmission of set membership information between networked hosts. A sender encodes + the information into a bit vector, the Bloom filter, that is more compact than a conventional + representation. Computation and space costs for construction are linear in the number of elements. + The receiver uses the filter to test whether various elements are members of the set. Though the + filter will occasionally return a false positive, it will never return a false negative. When creating + the filter, the sender can choose its desired point in a trade-off between the false positive rate and the size. + +

    + Originally created by + European Commission One-Lab Project 034819. + + @see Filter The general behavior of a filter + + @see Space/Time Trade-Offs in Hash Coding with Allowable Errors]]> + + + + + + + + + + + + + this filter. + @param nbHash The number of hash function to consider. + @param hashType type of the hashing function (see + {@link org.apache.hadoop.util.hash.Hash}).]]> + + + + + + + + + this counting Bloom filter. +

    + Invariant: nothing happens if the specified key does not belong to this counter Bloom filter. + @param key The key to remove.]]> + + + + + + + + + + + + key -> count map. +

    NOTE: due to the bucket size of this filter, inserting the same + key more than 15 times will cause an overflow at all filter positions + associated with this key, and it will significantly increase the error + rate for this and other keys. For this reason the filter can only be + used to store small count values 0 <= N << 15. + @param key key to be tested + @return 0 if the key is not present. Otherwise, a positive value v will + be returned such that v == count with probability equal to the + error rate of this filter, and v > count otherwise. + Additionally, if the filter experienced an underflow as a result of + {@link #delete(Key)} operation, the return value may be lower than the + count with the probability of the false negative rate of such + filter.]]> + + + + + + + + + + + + + + + + + + + + + + counting Bloom filter, as defined by Fan et al. in a ToN + 2000 paper. +

    + A counting Bloom filter is an improvement to standard a Bloom filter as it + allows dynamic additions and deletions of set membership information. This + is achieved through the use of a counting vector instead of a bit vector. +

    + Originally created by + European Commission One-Lab Project 034819. + + @see Filter The general behavior of a filter + + @see Summary cache: a scalable wide-area web cache sharing protocol]]> + + + + + + + + + + + + + + Builds an empty Dynamic Bloom filter. + @param vectorSize The number of bits in the vector. + @param nbHash The number of hash function to consider. + @param hashType type of the hashing function (see + {@link org.apache.hadoop.util.hash.Hash}). + @param nr The threshold for the maximum number of keys to record in a + dynamic Bloom filter row.]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + dynamic Bloom filter, as defined in the INFOCOM 2006 paper. +

    + A dynamic Bloom filter (DBF) makes use of a s * m bit matrix but + each of the s rows is a standard Bloom filter. The creation + process of a DBF is iterative. At the start, the DBF is a 1 * m + bit matrix, i.e., it is composed of a single standard Bloom filter. + It assumes that nr elements are recorded in the + initial bit vector, where nr {@literal <=} n + (n is the cardinality of the set A to record in + the filter). +

    + As the size of A grows during the execution of the application, + several keys must be inserted in the DBF. When inserting a key into the DBF, + one must first get an active Bloom filter in the matrix. A Bloom filter is + active when the number of recorded keys, nr, is + strictly less than the current cardinality of A, n. + If an active Bloom filter is found, the key is inserted and + nr is incremented by one. On the other hand, if there + is no active Bloom filter, a new one is created (i.e., a new row is added to + the matrix) according to the current size of A and the element + is added in this new Bloom filter and the nr value of + this new Bloom filter is set to one. A given key is said to belong to the + DBF if the k positions are set to one in one of the matrix rows. +

    + Originally created by + European Commission One-Lab Project 034819. + + @see Filter The general behavior of a filter + @see BloomFilter A Bloom filter + + @see Theory and Network Applications of Dynamic Bloom Filters]]> + + + + + + + + + Builds a hash function that must obey to a given maximum number of returned values and a highest value. + @param maxValue The maximum highest returned value. + @param nbHash The number of resulting hashed values. + @param hashType type of the hashing function (see {@link Hash}).]]> + + + + + this hash function. A NOOP]]> + + + + + + + + + + + + + + + + + + + The idea is to randomly select a bit to reset.]]> + + + + + + The idea is to select the bit to reset that will generate the minimum + number of false negative.]]> + + + + + + The idea is to select the bit to reset that will remove the maximum number + of false positive.]]> + + + + + + The idea is to select the bit to reset that will, at the same time, remove + the maximum number of false positve while minimizing the amount of false + negative generated.]]> + + + + + Originally created by + European Commission One-Lab Project 034819.]]> + + + + + + + + + + + + + + this filter. + @param nbHash The number of hash function to consider. + @param hashType type of the hashing function (see + {@link org.apache.hadoop.util.hash.Hash}).]]> + + + + + + + + + this retouched Bloom filter. +

    + Invariant: if the false positive is null, nothing happens. + @param key The false positive key to add.]]> + + + + + + this retouched Bloom filter. + @param coll The collection of false positive.]]> + + + + + + this retouched Bloom filter. + @param keys The list of false positive.]]> + + + + + + this retouched Bloom filter. + @param keys The array of false positive.]]> + + + + + + + this retouched Bloom filter. + @param scheme The selective clearing scheme to apply.]]> + + + + + + + + + + + + retouched Bloom filter, as defined in the CoNEXT 2006 paper. +

    + It allows the removal of selected false positives at the cost of introducing + random false negatives, and with the benefit of eliminating some random false + positives at the same time. + +

    + Originally created by + European Commission One-Lab Project 034819. + + @see Filter The general behavior of a filter + @see BloomFilter A Bloom filter + @see RemoveScheme The different selective clearing algorithms + + @see Retouched Bloom Filters: Allowing Networked Applications to Trade Off Selected False Positives Against False Negatives]]> + + + + + + + + + + + + + + + + + Any exception generated in the future is + extracted and rethrown. +

    + @param future future to evaluate + @param type of the result. + @return the result, if all went well. + @throws InterruptedIOException future was interrupted + @throws IOException if something went wrong + @throws RuntimeException any nested RTE thrown]]> +     + + + + + + + + + + + + Any exception generated in the future is + extracted and rethrown. +

    + @param future future to evaluate + @param type of the result. + @return the result, if all went well. + @throws InterruptedIOException future was interrupted + @throws IOException if something went wrong + @throws RuntimeException any nested RTE thrown + @throws TimeoutException the future timed out.]]> +     +     + + + + + type of return value. + @return nothing, ever. + @throws IOException either the inner IOException, or a wrapper around + any non-Runtime-Exception + @throws RuntimeException if that is the inner cause.]]> + + + + + + + type of return value. + @return nothing, ever. + @throws IOException either the inner IOException, or a wrapper around + any non-Runtime-Exception + @throws RuntimeException if that is the inner cause.]]> + + + + + + +
  • If it is an IOE: Return.
    • +
  • If it is a {@link UncheckedIOException}: return the cause
    • +
  • Completion/Execution Exceptions: extract and repeat
    • +
  • If it is an RTE or Error: throw.
    • +
  • Any other type: wrap in an IOE
    • + + + Recursively handles wrapped Execution and Completion Exceptions in + case something very complicated has happened. + @param e exception. + @return an IOException extracted or built from the cause. + @throws RuntimeException if that is the inner cause. + @throws Error if that is the inner cause.]]> +     +     + + + Contains methods promoted from + {@link org.apache.hadoop.fs.impl.FutureIOSupport} because they + are a key part of integrating async IO in application code. +

    +

    + One key feature is that the {@link #awaitFuture(Future)} and + {@link #awaitFuture(Future, long, TimeUnit)} calls will + extract and rethrow exceptions raised in the future's execution, + including extracting the inner IOException of any + {@code UncheckedIOException} raised in the future. + This makes it somewhat easier to execute IOException-raising + code inside futures. +

    ]]> +     +     + + + + + + + type + @return a remote iterator]]> + + + + + + type + @return a remote iterator]]> + + + + + + type + @return a remote iterator]]> + + + + + + type + @return a remote iterator]]> + + + + + + + source type + @param result type + @param iterator source + @param mapper transformation + @return a remote iterator]]> + + + + + + source type + @param result type + @param iterator source + @return a remote iterator]]> + + + + + + +

    + Elements are filtered in the hasNext() method; if not used + the filtering will be done on demand in the {@code next()} + call. + @param type + @param iterator source + @param filter filter + @return a remote iterator]]> +     +     + + + + + source type. + @return a new iterator]]> + + + + + + + type + @return a list of the values. + @throws IOException if the source RemoteIterator raises it.]]> + + + + + + + + type + @return an array of the values. + @throws IOException if the source RemoteIterator raises it.]]> + + + + + + + +

    + If the iterator is an IOStatisticsSource returning a non-null + set of statistics, and this classes log is set to DEBUG, + then the statistics of the operation are evaluated and logged at + debug. +

    + The number of entries processed is returned, as it is useful to + know this, especially during tests or when reporting values + to users. +

    + This does not close the iterator afterwards. + @param source iterator source + @param consumer consumer of the values. + @return the number of elements processed + @param type of source + @throws IOException if the source RemoteIterator or the consumer raise one.]]> +     +     + + + + type of source]]> + + + +

    + This aims to make it straightforward to use lambda-expressions to + transform the results of an iterator, without losing the statistics + in the process, and to chain the operations together. +

    + The closeable operation will be passed through RemoteIterators which + wrap other RemoteIterators. This is to support any iterator which + can be closed to release held connections, file handles etc. + Unless client code is written to assume that RemoteIterator instances + may be closed, this is not likely to be broadly used. It is added + to make it possible to adopt this feature in a managed way. +

    + One notable feature is that the + {@link #foreach(RemoteIterator, ConsumerRaisingIOE)} method will + LOG at debug any IOStatistics provided by the iterator, if such + statistics are provided. There's no attempt at retrieval and logging + if the LOG is not set to debug, so it is a zero cost feature unless + the logger {@code org.apache.hadoop.fs.functional.RemoteIterators} + is at DEBUG. +

    + Based on the S3A Listing code, and some some work on moving other code + to using iterative listings so as to pick up the statistics.]]> +     +     + +     + + + +     diff --git a/hadoop-common-project/hadoop-common/pom.xml b/hadoop-common-project/hadoop-common/pom.xml index d8e2dd3542223..5b5ffe1b00641 100644 --- a/hadoop-common-project/hadoop-common/pom.xml +++ b/hadoop-common-project/hadoop-common/pom.xml @@ -841,6 +841,36 @@     + + + org.apache.maven.plugins + maven-antrun-plugin + 1.8 + + + validate + + run + + + true + + + + + + + + + Skip platform toolset version detection = ${skip.platformToolsetDetection} + + + + + org.codehaus.mojo exec-maven-plugin @@ -852,6 +882,7 @@ exec + ${skip.platformToolsetDetection} ${basedir}\..\..\dev-support\bin\win-vs-upgrade.cmd ${basedir}\src\main\winutils @@ -866,6 +897,7 @@ exec + ${skip.platformToolsetDetection} msbuild ${basedir}/src/main/winutils/winutils.sln @@ -878,6 +910,27 @@     + + compile-ms-winutils-using-build-tools + compile + + exec + + + ${skip.platformToolsetDetection.negated} + msbuild + + ${basedir}/src/main/winutils/winutils.sln + /nologo + /p:Configuration=Release + /p:OutDir=${project.build.directory}/bin/ + /p:IntermediateOutputPath=${project.build.directory}/winutils/ + /p:WsceConfigDir=${wsce.config.dir} + /p:WsceConfigFile=${wsce.config.file} + /p:PlatformToolset=${use.platformToolsetVersion} + + + convert-ms-native-dll generate-sources @@ -885,6 +938,7 @@ exec + ${skip.platformToolsetDetection} ${basedir}\..\..\dev-support\bin\win-vs-upgrade.cmd ${basedir}\src\main\native @@ -899,6 +953,35 @@ exec + ${skip.platformToolsetDetection} + msbuild + + ${basedir}/src/main/native/native.sln + /nologo + /p:Configuration=Release + /p:OutDir=${project.build.directory}/bin/ + /p:CustomZstdPrefix=${zstd.prefix} + /p:CustomZstdLib=${zstd.lib} + /p:CustomZstdInclude=${zstd.include} + /p:RequireZstd=${require.zstd} + /p:CustomOpensslPrefix=${openssl.prefix} + /p:CustomOpensslLib=${openssl.lib} + /p:CustomOpensslInclude=${openssl.include} + /p:RequireOpenssl=${require.openssl} + /p:RequireIsal=${require.isal} + /p:CustomIsalPrefix=${isal.prefix} + /p:CustomIsalLib=${isal.lib} + + + + + compile-ms-native-dll-using-build-tools + compile + + exec + + + ${skip.platformToolsetDetection.negated} msbuild ${basedir}/src/main/native/native.sln @@ -916,6 +999,7 @@ /p:RequireIsal=${require.isal} /p:CustomIsalPrefix=${isal.prefix} /p:CustomIsalLib=${isal.lib} + /p:PlatformToolset=${use.platformToolsetVersion} @@ -1151,7 +1235,7 @@ src-test-compile-protoc-legacy generate-test-sources - compile + test-compile false @@ -1160,7 +1244,7 @@ com.google.protobuf:protoc:${protobuf.version}:exe:${os.detected.classifier} false - ${basedir}/src/test/proto + ${basedir}/src/test/proto ${project.build.directory}/generated-test-sources/java false diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ChecksumFileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ChecksumFileSystem.java index 1cca9fe2bfdb1..4c7569d6ecd81 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ChecksumFileSystem.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ChecksumFileSystem.java @@ -174,6 +174,7 @@ private static class ChecksumFSInputChecker extends FSInputChecker implements private static final int HEADER_LENGTH = 8; private int bytesPerSum = 1; + private long fileLen = -1L; public ChecksumFSInputChecker(ChecksumFileSystem fs, Path file) throws IOException { @@ -320,6 +321,18 @@ public static long findChecksumOffset(long dataOffset, return HEADER_LENGTH + (dataOffset/bytesPerSum) * FSInputChecker.CHECKSUM_SIZE; } + /** + * Calculate length of file if not already cached. + * @return file length. + * @throws IOException any IOE. + */ + private long getFileLength() throws IOException { + if (fileLen == -1L) { + fileLen = fs.getFileStatus(file).getLen(); + } + return fileLen; + } + /** * Find the checksum ranges that correspond to the given data ranges. * @param dataRanges the input data ranges, which are assumed to be sorted @@ -371,13 +384,28 @@ static ByteBuffer checkBytes(ByteBuffer sumsBytes, IntBuffer sums = sumsBytes.asIntBuffer(); sums.position(offset / FSInputChecker.CHECKSUM_SIZE); ByteBuffer current = data.duplicate(); - int numChunks = data.remaining() / bytesPerSum; + int numFullChunks = data.remaining() / bytesPerSum; + boolean partialChunk = ((data.remaining() % bytesPerSum) != 0); + int totalChunks = numFullChunks; + if (partialChunk) { + totalChunks++; + } CRC32 crc = new CRC32(); // check each chunk to ensure they match - for(int c = 0; c < numChunks; ++c) { - // set the buffer position and the limit - current.limit((c + 1) * bytesPerSum); + for(int c = 0; c < totalChunks; ++c) { + // set the buffer position to the start of every chunk. current.position(c * bytesPerSum); + + if (c == numFullChunks) { + // During last chunk, there may be less than chunk size + // data preset, so setting the limit accordingly. + int lastIncompleteChunk = data.remaining() % bytesPerSum; + current.limit((c * bytesPerSum) + lastIncompleteChunk); + } else { + // set the buffer limit to end of every chunk. + current.limit((c + 1) * bytesPerSum); + } + // compute the crc crc.reset(); crc.update(current); @@ -396,11 +424,34 @@ static ByteBuffer checkBytes(ByteBuffer sumsBytes, return data; } + /** + * Validates range parameters. + * In case of CheckSum FS, we already have calculated + * fileLength so failing fast here. + * @param ranges requested ranges. + * @param fileLength length of file. + * @throws EOFException end of file exception. + */ + private void validateRangeRequest(List ranges, + final long fileLength) throws EOFException { + for (FileRange range : ranges) { + VectoredReadUtils.validateRangeRequest(range); + if (range.getOffset() + range.getLength() > fileLength) { + final String errMsg = String.format("Requested range [%d, %d) is beyond EOF for path %s", + range.getOffset(), range.getLength(), file); + LOG.warn(errMsg); + throw new EOFException(errMsg); + } + } + } + @Override public void readVectored(List ranges, IntFunction allocate) throws IOException { + final long length = getFileLength(); + validateRangeRequest(ranges, length); + // If the stream doesn't have checksums, just delegate. - VectoredReadUtils.validateVectoredReadRanges(ranges); if (sums == null) { datas.readVectored(ranges, allocate); return; @@ -410,15 +461,18 @@ public void readVectored(List ranges, List dataRanges = VectoredReadUtils.mergeSortedRanges(Arrays.asList(sortRanges(ranges)), bytesPerSum, minSeek, maxReadSizeForVectorReads()); + // While merging the ranges above, they are rounded up based on the value of bytesPerSum + // which leads to some ranges crossing the EOF thus they need to be fixed else it will + // cause EOFException during actual reads. + for (CombinedFileRange range : dataRanges) { + if (range.getOffset() + range.getLength() > length) { + range.setLength((int) (length - range.getOffset())); + } + } List checksumRanges = findChecksumRanges(dataRanges, bytesPerSum, minSeek, maxSize); sums.readVectored(checksumRanges, allocate); datas.readVectored(dataRanges, allocate); - // Data read is correct. I have verified content of dataRanges. - // There is some bug below here as test (testVectoredReadMultipleRanges) - // is failing, should be - // somewhere while slicing the merged data into smaller user ranges. - // Spend some time figuring out but it is a complex code. for(CombinedFileRange checksumRange: checksumRanges) { for(FileRange dataRange: checksumRange.getUnderlying()) { // when we have both the ranges, validate the checksum diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeys.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeys.java index 16144206eeea4..9d6224366d1ba 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeys.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeys.java @@ -475,4 +475,21 @@ public class CommonConfigurationKeys extends CommonConfigurationKeysPublic { * default hadoop temp dir on local system: {@value}. */ public static final String HADOOP_TMP_DIR = "hadoop.tmp.dir"; + + /** + * Thread-level IOStats Support. + * {@value} + */ + public static final String IOSTATISTICS_THREAD_LEVEL_ENABLED = + "fs.iostatistics.thread.level.enabled"; + + /** + * Default value for Thread-level IOStats Support is true. + */ + public static final boolean IOSTATISTICS_THREAD_LEVEL_ENABLED_DEFAULT = + true; + + public static final String HADOOP_SECURITY_RESOLVER_IMPL = + "hadoop.security.resolver.impl"; + } diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/DelegationTokenRenewer.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/DelegationTokenRenewer.java index 6f6e30410659c..794855508c63f 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/DelegationTokenRenewer.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/DelegationTokenRenewer.java @@ -256,9 +256,8 @@ public void removeRenewAction( try { action.cancel(); } catch (InterruptedException ie) { - LOG.error("Interrupted while canceling token for " + fs.getUri() - + "filesystem"); - LOG.debug("Exception in removeRenewAction: {}", ie); + LOG.error("Interrupted while canceling token for {} filesystem.", fs.getUri()); + LOG.debug("Exception in removeRenewAction.", ie); } } } diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FSDataInputStream.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FSDataInputStream.java index 52644402ca459..cca6c28da11a3 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FSDataInputStream.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FSDataInputStream.java @@ -144,7 +144,8 @@ public boolean seekToNewSource(long targetPos) throws IOException { * * @return the underlying input stream */ - @InterfaceAudience.LimitedPrivate({"HDFS"}) + @InterfaceAudience.Public + @InterfaceStability.Stable public InputStream getWrappedStream() { return in; } diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/LocalDirAllocator.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/LocalDirAllocator.java index f6c9d3c7cb0dd..774e015b37343 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/LocalDirAllocator.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/LocalDirAllocator.java @@ -396,6 +396,10 @@ public Path getLocalPathForWrite(String pathStr, long size, Context ctx = confChanged(conf); int numDirs = ctx.localDirs.length; int numDirsSearched = 0; + // Max capacity in any directory + long maxCapacity = 0; + String errorText = null; + IOException diskException = null; //remove the leading slash from the path (to make sure that the uri //resolution results in a valid path on the dir being checked) if (pathStr.startsWith("/")) { @@ -444,9 +448,18 @@ public Path getLocalPathForWrite(String pathStr, long size, int dirNum = ctx.getAndIncrDirNumLastAccessed(randomInc); while (numDirsSearched < numDirs) { long capacity = ctx.dirDF[dirNum].getAvailable(); + if (capacity > maxCapacity) { + maxCapacity = capacity; + } if (capacity > size) { - returnPath = - createPath(ctx.localDirs[dirNum], pathStr, checkWrite); + try { + returnPath = createPath(ctx.localDirs[dirNum], pathStr, + checkWrite); + } catch (IOException e) { + errorText = e.getMessage(); + diskException = e; + LOG.debug("DiskException caught for dir {}", ctx.localDirs[dirNum], e); + } if (returnPath != null) { ctx.getAndIncrDirNumLastAccessed(numDirsSearched); break; @@ -462,8 +475,13 @@ public Path getLocalPathForWrite(String pathStr, long size, } //no path found - throw new DiskErrorException("Could not find any valid local " + - "directory for " + pathStr); + String newErrorText = "Could not find any valid local directory for " + + pathStr + " with requested size " + size + + " as the max capacity in any directory is " + maxCapacity; + if (errorText != null) { + newErrorText = newErrorText + " due to " + errorText; + } + throw new DiskErrorException(newErrorText, diskException); } /** Creates a file on the local FS. Pass size as diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/PositionedReadable.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/PositionedReadable.java index de76090512705..7380402eb6156 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/PositionedReadable.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/PositionedReadable.java @@ -114,6 +114,16 @@ default int maxReadSizeForVectorReads() { * As a result of the call, each range will have FileRange.setData(CompletableFuture) * called with a future that when complete will have a ByteBuffer with the * data from the file's range. + *

    + * The position returned by getPos() after readVectored() is undefined. + *

    + *

    + * If a file is changed while the readVectored() operation is in progress, the output is + * undefined. Some ranges may have old data, some may have new and some may have both. + *

    + *

    + * While a readVectored() operation is in progress, normal read api calls may block. + *

    * @param ranges the byte ranges to read * @param allocate the function to allocate ByteBuffer * @throws IOException any IOE. diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/RawLocalFileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/RawLocalFileSystem.java index f525c3cba78fe..2f4f93099b5c9 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/RawLocalFileSystem.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/RawLocalFileSystem.java @@ -57,6 +57,8 @@ import org.apache.hadoop.fs.impl.StoreImplementationUtils; import org.apache.hadoop.fs.permission.FsPermission; import org.apache.hadoop.fs.statistics.IOStatistics; +import org.apache.hadoop.fs.statistics.IOStatisticsAggregator; +import org.apache.hadoop.fs.statistics.IOStatisticsContext; import org.apache.hadoop.fs.statistics.IOStatisticsSource; import org.apache.hadoop.fs.statistics.BufferedIOStatisticsOutputStream; import org.apache.hadoop.fs.statistics.impl.IOStatisticsStore; @@ -156,11 +158,19 @@ class LocalFSFileInputStream extends FSInputStream implements /** Reference to the bytes read counter for slightly faster counting. */ private final AtomicLong bytesRead; + /** + * Thread level IOStatistics aggregator to update in close(). + */ + private final IOStatisticsAggregator + ioStatisticsAggregator; + public LocalFSFileInputStream(Path f) throws IOException { name = pathToFile(f); fis = new FileInputStream(name); bytesRead = ioStatistics.getCounterReference( STREAM_READ_BYTES); + ioStatisticsAggregator = + IOStatisticsContext.getCurrentIOStatisticsContext().getAggregator(); } @Override @@ -193,9 +203,13 @@ public boolean seekToNewSource(long targetPos) throws IOException { @Override public void close() throws IOException { - fis.close(); - if (asyncChannel != null) { - asyncChannel.close(); + try { + fis.close(); + if (asyncChannel != null) { + asyncChannel.close(); + } + } finally { + ioStatisticsAggregator.aggregate(ioStatistics); } } @@ -278,6 +292,7 @@ public boolean hasCapability(String capability) { // new capabilities. switch (capability.toLowerCase(Locale.ENGLISH)) { case StreamCapabilities.IOSTATISTICS: + case StreamCapabilities.IOSTATISTICS_CONTEXT: case StreamCapabilities.VECTOREDIO: return true; default: @@ -407,9 +422,19 @@ final class LocalFSFileOutputStream extends OutputStream implements STREAM_WRITE_EXCEPTIONS) .build(); + /** + * Thread level IOStatistics aggregator to update in close(). + */ + private final IOStatisticsAggregator + ioStatisticsAggregator; + private LocalFSFileOutputStream(Path f, boolean append, FsPermission permission) throws IOException { File file = pathToFile(f); + // store the aggregator before attempting any IO. + ioStatisticsAggregator = + IOStatisticsContext.getCurrentIOStatisticsContext().getAggregator(); + if (!append && permission == null) { permission = FsPermission.getFileDefault(); } @@ -436,10 +461,17 @@ private LocalFSFileOutputStream(Path f, boolean append, } /* - * Just forward to the fos + * Close the fos; update the IOStatisticsContext. */ @Override - public void close() throws IOException { fos.close(); } + public void close() throws IOException { + try { + fos.close(); + } finally { + ioStatisticsAggregator.aggregate(ioStatistics); + } + } + @Override public void flush() throws IOException { fos.flush(); } @Override @@ -485,6 +517,7 @@ public boolean hasCapability(String capability) { // new capabilities. switch (capability.toLowerCase(Locale.ENGLISH)) { case StreamCapabilities.IOSTATISTICS: + case StreamCapabilities.IOSTATISTICS_CONTEXT: return true; default: return StoreImplementationUtils.isProbeForSyncable(capability); @@ -1293,4 +1326,9 @@ public boolean hasPathCapability(final Path path, final String capability) return super.hasPathCapability(path, capability); } } + + @VisibleForTesting + static void setUseDeprecatedFileStatus(boolean useDeprecatedFileStatus) { + RawLocalFileSystem.useDeprecatedFileStatus = useDeprecatedFileStatus; + } } diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/StreamCapabilities.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/StreamCapabilities.java index d68ef505dc3fe..c925e50889d53 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/StreamCapabilities.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/StreamCapabilities.java @@ -93,6 +93,12 @@ public interface StreamCapabilities { */ String ABORTABLE_STREAM = CommonPathCapabilities.ABORTABLE_STREAM; + /** + * Streams that support IOStatistics context and capture thread-level + * IOStatistics. + */ + String IOSTATISTICS_CONTEXT = "fs.capability.iocontext.supported"; + /** * Capabilities that a stream can support and be queried for. */ diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/VectoredReadUtils.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/VectoredReadUtils.java index 64107f1a18f89..50cab7dc4ccf8 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/VectoredReadUtils.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/VectoredReadUtils.java @@ -30,6 +30,7 @@ import org.apache.hadoop.fs.impl.CombinedFileRange; import org.apache.hadoop.util.Preconditions; +import org.apache.hadoop.util.functional.Function4RaisingIOE; /** * Utility class which implements helper methods used @@ -37,6 +38,8 @@ */ public final class VectoredReadUtils { + private static final int TMP_BUFFER_MAX_SIZE = 64 * 1024; + /** * Validate a single range. * @param range file range. @@ -114,7 +117,12 @@ private static void readNonByteBufferPositionedReadable(PositionedReadable strea FileRange range, ByteBuffer buffer) throws IOException { if (buffer.isDirect()) { - buffer.put(readInDirectBuffer(stream, range)); + readInDirectBuffer(range.getLength(), + buffer, + (position, buffer1, offset, length) -> { + stream.readFully(position, buffer1, offset, length); + return null; + }); buffer.flip(); } else { stream.readFully(range.getOffset(), buffer.array(), @@ -122,13 +130,34 @@ private static void readNonByteBufferPositionedReadable(PositionedReadable strea } } - private static byte[] readInDirectBuffer(PositionedReadable stream, - FileRange range) throws IOException { - // if we need to read data from a direct buffer and the stream doesn't - // support it, we allocate a byte array to use. - byte[] tmp = new byte[range.getLength()]; - stream.readFully(range.getOffset(), tmp, 0, tmp.length); - return tmp; + /** + * Read bytes from stream into a byte buffer using an + * intermediate byte array. + * @param length number of bytes to read. + * @param buffer buffer to fill. + * @param operation operation to use for reading data. + * @throws IOException any IOE. + */ + public static void readInDirectBuffer(int length, + ByteBuffer buffer, + Function4RaisingIOE operation) throws IOException { + if (length == 0) { + return; + } + int readBytes = 0; + int position = 0; + int tmpBufferMaxSize = Math.min(TMP_BUFFER_MAX_SIZE, length); + byte[] tmp = new byte[tmpBufferMaxSize]; + while (readBytes < length) { + int currentLength = (readBytes + tmpBufferMaxSize) < length ? + tmpBufferMaxSize + : (length - readBytes); + operation.apply(position, tmp, 0, currentLength); + buffer.put(tmp, 0, currentLength); + position = position + currentLength; + readBytes = readBytes + currentLength; + } } /** @@ -210,6 +239,7 @@ public static List validateNonOverlappingAndReturnSortedRan if (sortedRanges[i].getOffset() < prev.getOffset() + prev.getLength()) { throw new UnsupportedOperationException("Overlapping ranges are not supported"); } + prev = sortedRanges[i]; } return Arrays.asList(sortedRanges); } diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/impl/WeakReferenceThreadMap.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/impl/WeakReferenceThreadMap.java index b24bef2a816bf..16fe0da7c5a81 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/impl/WeakReferenceThreadMap.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/impl/WeakReferenceThreadMap.java @@ -18,6 +18,7 @@ package org.apache.hadoop.fs.impl; +import java.lang.ref.WeakReference; import java.util.function.Consumer; import java.util.function.Function; import javax.annotation.Nullable; @@ -48,7 +49,17 @@ public long currentThreadId() { } public V setForCurrentThread(V newVal) { - return put(currentThreadId(), newVal); + long id = currentThreadId(); + + // if the same object is already in the map, just return it. + WeakReference ref = lookup(id); + // Reference value could be set to null. Thus, ref.get() could return + // null. Should be handled accordingly while using the returned value. + if (ref != null && ref.get() == newVal) { + return ref.get(); + } + + return put(id, newVal); } } diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/impl/prefetch/BlockCache.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/impl/prefetch/BlockCache.java new file mode 100644 index 0000000000000..c18dc519188ba --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/impl/prefetch/BlockCache.java @@ -0,0 +1,70 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hadoop.fs.impl.prefetch; + +import java.io.Closeable; +import java.io.IOException; +import java.nio.ByteBuffer; + +/** + * Provides functionality necessary for caching blocks of data read from FileSystem. + */ +public interface BlockCache extends Closeable { + + /** + * Indicates whether the given block is in this cache. + * + * @param blockNumber the id of the given block. + * @return true if the given block is in this cache, false otherwise. + */ + boolean containsBlock(int blockNumber); + + /** + * Gets the blocks in this cache. + * + * @return the blocks in this cache. + */ + Iterable blocks(); + + /** + * Gets the number of blocks in this cache. + * + * @return the number of blocks in this cache. + */ + int size(); + + /** + * Gets the block having the given {@code blockNumber}. + * + * @param blockNumber the id of the desired block. + * @param buffer contents of the desired block are copied to this buffer. + * @throws IOException if there is an error reading the given block. + */ + void get(int blockNumber, ByteBuffer buffer) throws IOException; + + /** + * Puts the given block in this cache. + * + * @param blockNumber the id of the given block. + * @param buffer contents of the given block to be added to this cache. + * @throws IOException if there is an error writing the given block. + */ + void put(int blockNumber, ByteBuffer buffer) throws IOException; +} diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/impl/prefetch/BlockData.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/impl/prefetch/BlockData.java new file mode 100644 index 0000000000000..ecb8bc7243be0 --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/impl/prefetch/BlockData.java @@ -0,0 +1,250 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hadoop.fs.impl.prefetch; + +import static org.apache.hadoop.fs.impl.prefetch.Validate.checkNotNegative; +import static org.apache.hadoop.fs.impl.prefetch.Validate.checkPositiveInteger; +import static org.apache.hadoop.fs.impl.prefetch.Validate.checkWithinRange; + +/** + * Holds information about blocks of data in a file. + */ +public final class BlockData { + + // State of each block of data. + enum State { + + /** Data is not yet ready to be read from this block (still being prefetched). */ + NOT_READY, + + /** A read of this block has been enqueued in the prefetch queue. */ + QUEUED, + + /** A read of this block has been enqueued in the prefetch queue. */ + READY, + + /** This block has been cached in the local disk cache. */ + CACHED + } + + /** + * State of all blocks in a file. + */ + private State[] state; + + /** + * The size of a file. + */ + private final long fileSize; + + /** + * The file is divided into blocks of this size. + */ + private final int blockSize; + + /** + * The file has these many blocks. + */ + private final int numBlocks; + + /** + * Constructs an instance of {@link BlockData}. + * @param fileSize the size of a file. + * @param blockSize the file is divided into blocks of this size. + * @throws IllegalArgumentException if fileSize is negative. + * @throws IllegalArgumentException if blockSize is negative. + * @throws IllegalArgumentException if blockSize is zero or negative. + */ + public BlockData(long fileSize, int blockSize) { + checkNotNegative(fileSize, "fileSize"); + if (fileSize == 0) { + checkNotNegative(blockSize, "blockSize"); + } else { + checkPositiveInteger(blockSize, "blockSize"); + } + + this.fileSize = fileSize; + this.blockSize = blockSize; + this.numBlocks = + (fileSize == 0) + ? 0 + : ((int) (fileSize / blockSize)) + (fileSize % blockSize > 0 + ? 1 + : 0); + this.state = new State[this.numBlocks]; + for (int b = 0; b < this.numBlocks; b++) { + setState(b, State.NOT_READY); + } + } + + /** + * Gets the size of each block. + * @return the size of each block. + */ + public int getBlockSize() { + return blockSize; + } + + /** + * Gets the size of the associated file. + * @return the size of the associated file. + */ + public long getFileSize() { + return fileSize; + } + + /** + * Gets the number of blocks in the associated file. + * @return the number of blocks in the associated file. + */ + public int getNumBlocks() { + return numBlocks; + } + + /** + * Indicates whether the given block is the last block in the associated file. + * @param blockNumber the id of the desired block. + * @return true if the given block is the last block in the associated file, false otherwise. + * @throws IllegalArgumentException if blockNumber is invalid. + */ + public boolean isLastBlock(int blockNumber) { + if (fileSize == 0) { + return false; + } + + throwIfInvalidBlockNumber(blockNumber); + + return blockNumber == (numBlocks - 1); + } + + /** + * Gets the id of the block that contains the given absolute offset. + * @param offset the absolute offset to check. + * @return the id of the block that contains the given absolute offset. + * @throws IllegalArgumentException if offset is invalid. + */ + public int getBlockNumber(long offset) { + throwIfInvalidOffset(offset); + + return (int) (offset / blockSize); + } + + /** + * Gets the size of the given block. + * @param blockNumber the id of the desired block. + * @return the size of the given block. + */ + public int getSize(int blockNumber) { + if (fileSize == 0) { + return 0; + } + + if (isLastBlock(blockNumber)) { + return (int) (fileSize - (((long) blockSize) * (numBlocks - 1))); + } else { + return blockSize; + } + } + + /** + * Indicates whether the given absolute offset is valid. + * @param offset absolute offset in the file.. + * @return true if the given absolute offset is valid, false otherwise. + */ + public boolean isValidOffset(long offset) { + return (offset >= 0) && (offset < fileSize); + } + + /** + * Gets the start offset of the given block. + * @param blockNumber the id of the given block. + * @return the start offset of the given block. + * @throws IllegalArgumentException if blockNumber is invalid. + */ + public long getStartOffset(int blockNumber) { + throwIfInvalidBlockNumber(blockNumber); + + return blockNumber * (long) blockSize; + } + + /** + * Gets the relative offset corresponding to the given block and the absolute offset. + * @param blockNumber the id of the given block. + * @param offset absolute offset in the file. + * @return the relative offset corresponding to the given block and the absolute offset. + * @throws IllegalArgumentException if either blockNumber or offset is invalid. + */ + public int getRelativeOffset(int blockNumber, long offset) { + throwIfInvalidOffset(offset); + + return (int) (offset - getStartOffset(blockNumber)); + } + + /** + * Gets the state of the given block. + * @param blockNumber the id of the given block. + * @return the state of the given block. + * @throws IllegalArgumentException if blockNumber is invalid. + */ + public State getState(int blockNumber) { + throwIfInvalidBlockNumber(blockNumber); + + return state[blockNumber]; + } + + /** + * Sets the state of the given block to the given value. + * @param blockNumber the id of the given block. + * @param blockState the target state. + * @throws IllegalArgumentException if blockNumber is invalid. + */ + public void setState(int blockNumber, State blockState) { + throwIfInvalidBlockNumber(blockNumber); + + state[blockNumber] = blockState; + } + + // Debug helper. + public String getStateString() { + StringBuilder sb = new StringBuilder(); + int blockNumber = 0; + while (blockNumber < numBlocks) { + State tstate = getState(blockNumber); + int endBlockNumber = blockNumber; + while ((endBlockNumber < numBlocks) && (getState(endBlockNumber) + == tstate)) { + endBlockNumber++; + } + sb.append( + String.format("[%03d ~ %03d] %s%n", blockNumber, endBlockNumber - 1, + tstate)); + blockNumber = endBlockNumber; + } + return sb.toString(); + } + + private void throwIfInvalidBlockNumber(int blockNumber) { + checkWithinRange(blockNumber, "blockNumber", 0, numBlocks - 1); + } + + private void throwIfInvalidOffset(long offset) { + checkWithinRange(offset, "offset", 0, fileSize - 1); + } +} diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/impl/prefetch/BlockManager.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/impl/prefetch/BlockManager.java new file mode 100644 index 0000000000000..45f0aabe7dcd9 --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/impl/prefetch/BlockManager.java @@ -0,0 +1,145 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hadoop.fs.impl.prefetch; + +import java.io.Closeable; +import java.io.IOException; +import java.nio.ByteBuffer; + +import static org.apache.hadoop.fs.impl.prefetch.Validate.checkNotNegative; +import static org.apache.hadoop.fs.impl.prefetch.Validate.checkNotNull; + +/** + * Provides read access to the underlying file one block at a time. + * + * This class is the simplest form of a {@code BlockManager} that does + * perform prefetching or caching. + */ +public abstract class BlockManager implements Closeable { + + /** + * Information about each block of the underlying file. + */ + private final BlockData blockData; + + /** + * Constructs an instance of {@code BlockManager}. + * + * @param blockData information about each block of the underlying file. + * + * @throws IllegalArgumentException if blockData is null. + */ + public BlockManager(BlockData blockData) { + checkNotNull(blockData, "blockData"); + + this.blockData = blockData; + } + + /** + * Gets block data information. + * + * @return instance of {@code BlockData}. + */ + public BlockData getBlockData() { + return blockData; + } + + /** + * Gets the block having the given {@code blockNumber}. + * + * The entire block is read into memory and returned as a {@code BufferData}. + * The blocks are treated as a limited resource and must be released when + * one is done reading them. + * + * @param blockNumber the number of the block to be read and returned. + * @return {@code BufferData} having data from the given block. + * + * @throws IOException if there an error reading the given block. + * @throws IllegalArgumentException if blockNumber is negative. + */ + public BufferData get(int blockNumber) throws IOException { + checkNotNegative(blockNumber, "blockNumber"); + + int size = blockData.getSize(blockNumber); + ByteBuffer buffer = ByteBuffer.allocate(size); + long startOffset = blockData.getStartOffset(blockNumber); + read(buffer, startOffset, size); + buffer.flip(); + return new BufferData(blockNumber, buffer); + } + + /** + * Reads into the given {@code buffer} {@code size} bytes from the underlying file + * starting at {@code startOffset}. + * + * @param buffer the buffer to read data in to. + * @param startOffset the offset at which reading starts. + * @param size the number bytes to read. + * @return number of bytes read. + * @throws IOException if there an error reading the given block. + */ + public abstract int read(ByteBuffer buffer, long startOffset, int size) throws IOException; + + /** + * Releases resources allocated to the given block. + * + * @param data the {@code BufferData} to release. + * + * @throws IllegalArgumentException if data is null. + */ + public void release(BufferData data) { + checkNotNull(data, "data"); + + // Do nothing because we allocate a new buffer each time. + } + + /** + * Requests optional prefetching of the given block. + * + * @param blockNumber the id of the block to prefetch. + * + * @throws IllegalArgumentException if blockNumber is negative. + */ + public void requestPrefetch(int blockNumber) { + checkNotNegative(blockNumber, "blockNumber"); + + // Do nothing because we do not support prefetches. + } + + /** + * Requests cancellation of any previously issued prefetch requests. + */ + public void cancelPrefetches() { + // Do nothing because we do not support prefetches. + } + + /** + * Requests that the given block should be copied to the cache. Optional operation. + * + * @param data the {@code BufferData} instance to optionally cache. + */ + public void requestCaching(BufferData data) { + // Do nothing because we do not support caching. + } + + @Override + public void close() { + } +} diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/impl/prefetch/BlockOperations.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/impl/prefetch/BlockOperations.java new file mode 100644 index 0000000000000..2744334a3bd7a --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/impl/prefetch/BlockOperations.java @@ -0,0 +1,425 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hadoop.fs.impl.prefetch; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.DoubleSummaryStatistics; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import static org.apache.hadoop.fs.impl.prefetch.Validate.checkNotNegative; + +/** + * Block level operations performed on a file. + * This class is meant to be used by {@code BlockManager}. + * It is separated out in its own file due to its size. + * + * This class is used for debugging/logging. Calls to this class + * can be safely removed without affecting the overall operation. + */ +public final class BlockOperations { + private static final Logger LOG = LoggerFactory.getLogger(BlockOperations.class); + + /** + * Operation kind. + */ + public enum Kind { + UNKNOWN("??", "unknown", false), + CANCEL_PREFETCHES("CP", "cancelPrefetches", false), + CLOSE("CX", "close", false), + CACHE_PUT("C+", "putC", true), + GET_CACHED("GC", "getCached", true), + GET_PREFETCHED("GP", "getPrefetched", true), + GET_READ("GR", "getRead", true), + PREFETCH("PF", "prefetch", true), + RELEASE("RL", "release", true), + REQUEST_CACHING("RC", "requestCaching", true), + REQUEST_PREFETCH("RP", "requestPrefetch", true); + + private String shortName; + private String name; + private boolean hasBlock; + + Kind(String shortName, String name, boolean hasBlock) { + this.shortName = shortName; + this.name = name; + this.hasBlock = hasBlock; + } + + private static Map shortNameToKind = new HashMap<>(); + + public static Kind fromShortName(String shortName) { + if (shortNameToKind.isEmpty()) { + for (Kind kind : Kind.values()) { + shortNameToKind.put(kind.shortName, kind); + } + } + return shortNameToKind.get(shortName); + } + } + + public static class Operation { + private final Kind kind; + private final int blockNumber; + private final long timestamp; + + public Operation(Kind kind, int blockNumber) { + this.kind = kind; + this.blockNumber = blockNumber; + this.timestamp = System.nanoTime(); + } + + public Kind getKind() { + return kind; + } + + public int getBlockNumber() { + return blockNumber; + } + + public long getTimestamp() { + return timestamp; + } + + public void getSummary(StringBuilder sb) { + if (kind.hasBlock) { + sb.append(String.format("%s(%d)", kind.shortName, blockNumber)); + } else { + sb.append(String.format("%s", kind.shortName)); + } + } + + public String getDebugInfo() { + if (kind.hasBlock) { + return String.format("--- %s(%d)", kind.name, blockNumber); + } else { + return String.format("... %s()", kind.name); + } + } + } + + public static class End extends Operation { + private Operation op; + + public End(Operation op) { + super(op.kind, op.blockNumber); + this.op = op; + } + + @Override + public void getSummary(StringBuilder sb) { + sb.append("E"); + super.getSummary(sb); + } + + @Override + public String getDebugInfo() { + return "***" + super.getDebugInfo().substring(3); + } + + public double duration() { + return (getTimestamp() - op.getTimestamp()) / 1e9; + } + } + + private ArrayList ops; + private boolean debugMode; + + public BlockOperations() { + this.ops = new ArrayList<>(); + } + + public synchronized void setDebug(boolean state) { + debugMode = state; + } + + private synchronized Operation add(Operation op) { + if (debugMode) { + LOG.info(op.getDebugInfo()); + } + ops.add(op); + return op; + } + + public Operation getPrefetched(int blockNumber) { + checkNotNegative(blockNumber, "blockNumber"); + + return add(new Operation(Kind.GET_PREFETCHED, blockNumber)); + } + + public Operation getCached(int blockNumber) { + checkNotNegative(blockNumber, "blockNumber"); + + return add(new Operation(Kind.GET_CACHED, blockNumber)); + } + + public Operation getRead(int blockNumber) { + checkNotNegative(blockNumber, "blockNumber"); + + return add(new Operation(Kind.GET_READ, blockNumber)); + } + + public Operation release(int blockNumber) { + checkNotNegative(blockNumber, "blockNumber"); + + return add(new Operation(Kind.RELEASE, blockNumber)); + } + + public Operation requestPrefetch(int blockNumber) { + checkNotNegative(blockNumber, "blockNumber"); + + return add(new Operation(Kind.REQUEST_PREFETCH, blockNumber)); + } + + public Operation prefetch(int blockNumber) { + checkNotNegative(blockNumber, "blockNumber"); + + return add(new Operation(Kind.PREFETCH, blockNumber)); + } + + public Operation cancelPrefetches() { + return add(new Operation(Kind.CANCEL_PREFETCHES, -1)); + } + + public Operation close() { + return add(new Operation(Kind.CLOSE, -1)); + } + + public Operation requestCaching(int blockNumber) { + checkNotNegative(blockNumber, "blockNumber"); + + return add(new Operation(Kind.REQUEST_CACHING, blockNumber)); + } + + public Operation addToCache(int blockNumber) { + checkNotNegative(blockNumber, "blockNumber"); + + return add(new Operation(Kind.CACHE_PUT, blockNumber)); + } + + public Operation end(Operation op) { + return add(new End(op)); + } + + private static void append(StringBuilder sb, String format, Object... args) { + sb.append(String.format(format, args)); + } + + public synchronized String getSummary(boolean showDebugInfo) { + StringBuilder sb = new StringBuilder(); + for (Operation op : ops) { + if (op != null) { + if (showDebugInfo) { + sb.append(op.getDebugInfo()); + sb.append("\n"); + } else { + op.getSummary(sb); + sb.append(";"); + } + } + } + + sb.append("\n"); + getDurationInfo(sb); + + return sb.toString(); + } + + public synchronized void getDurationInfo(StringBuilder sb) { + Map durations = new HashMap<>(); + for (Operation op : ops) { + if (op instanceof End) { + End endOp = (End) op; + DoubleSummaryStatistics stats = durations.get(endOp.getKind()); + if (stats == null) { + stats = new DoubleSummaryStatistics(); + durations.put(endOp.getKind(), stats); + } + stats.accept(endOp.duration()); + } + } + + List kinds = Arrays.asList( + Kind.GET_CACHED, + Kind.GET_PREFETCHED, + Kind.GET_READ, + Kind.CACHE_PUT, + Kind.PREFETCH, + Kind.REQUEST_CACHING, + Kind.REQUEST_PREFETCH, + Kind.CANCEL_PREFETCHES, + Kind.RELEASE, + Kind.CLOSE + ); + + for (Kind kind : kinds) { + append(sb, "%-18s : ", kind); + DoubleSummaryStatistics stats = durations.get(kind); + if (stats == null) { + append(sb, "--\n"); + } else { + append( + sb, + "#ops = %3d, total = %5.1f, min: %3.1f, avg: %3.1f, max: %3.1f\n", + stats.getCount(), + stats.getSum(), + stats.getMin(), + stats.getAverage(), + stats.getMax()); + } + } + } + + public synchronized void analyze(StringBuilder sb) { + Map> blockOps = new HashMap<>(); + + // Group-by block number. + for (Operation op : ops) { + if (op.blockNumber < 0) { + continue; + } + + List perBlockOps; + if (!blockOps.containsKey(op.blockNumber)) { + perBlockOps = new ArrayList<>(); + blockOps.put(op.blockNumber, perBlockOps); + } + + perBlockOps = blockOps.get(op.blockNumber); + perBlockOps.add(op); + } + + List prefetchedNotUsed = new ArrayList<>(); + List cachedNotUsed = new ArrayList<>(); + + for (Map.Entry> entry : blockOps.entrySet()) { + Integer blockNumber = entry.getKey(); + List perBlockOps = entry.getValue(); + Map kindCounts = new HashMap<>(); + Map endKindCounts = new HashMap<>(); + + for (Operation op : perBlockOps) { + if (op instanceof End) { + int endCount = endKindCounts.getOrDefault(op.kind, 0) + 1; + endKindCounts.put(op.kind, endCount); + } else { + int count = kindCounts.getOrDefault(op.kind, 0) + 1; + kindCounts.put(op.kind, count); + } + } + + for (Kind kind : kindCounts.keySet()) { + int count = kindCounts.getOrDefault(kind, 0); + int endCount = endKindCounts.getOrDefault(kind, 0); + if (count != endCount) { + append(sb, "[%d] %s : #ops(%d) != #end-ops(%d)\n", blockNumber, kind, count, endCount); + } + + if (count > 1) { + append(sb, "[%d] %s = %d\n", blockNumber, kind, count); + } + } + + int prefetchCount = kindCounts.getOrDefault(Kind.PREFETCH, 0); + int getPrefetchedCount = kindCounts.getOrDefault(Kind.GET_PREFETCHED, 0); + if ((prefetchCount > 0) && (getPrefetchedCount < prefetchCount)) { + prefetchedNotUsed.add(blockNumber); + } + + int cacheCount = kindCounts.getOrDefault(Kind.CACHE_PUT, 0); + int getCachedCount = kindCounts.getOrDefault(Kind.GET_CACHED, 0); + if ((cacheCount > 0) && (getCachedCount < cacheCount)) { + cachedNotUsed.add(blockNumber); + } + } + + if (!prefetchedNotUsed.isEmpty()) { + append(sb, "Prefetched but not used: %s\n", getIntList(prefetchedNotUsed)); + } + + if (!cachedNotUsed.isEmpty()) { + append(sb, "Cached but not used: %s\n", getIntList(cachedNotUsed)); + } + } + + private static String getIntList(Iterable nums) { + List numList = new ArrayList<>(); + for (Integer n : nums) { + numList.add(n.toString()); + } + return String.join(", ", numList); + } + + public static BlockOperations fromSummary(String summary) { + BlockOperations ops = new BlockOperations(); + ops.setDebug(true); + Pattern blockOpPattern = Pattern.compile("([A-Z+]+)(\\(([0-9]+)?\\))?"); + String[] tokens = summary.split(";"); + for (String token : tokens) { + Matcher matcher = blockOpPattern.matcher(token); + if (!matcher.matches()) { + String message = String.format("Unknown summary format: %s", token); + throw new IllegalArgumentException(message); + } + + String shortName = matcher.group(1); + String blockNumberStr = matcher.group(3); + int blockNumber = (blockNumberStr == null) ? -1 : Integer.parseInt(blockNumberStr); + Kind kind = Kind.fromShortName(shortName); + Kind endKind = null; + if (kind == null) { + if (shortName.charAt(0) == 'E') { + endKind = Kind.fromShortName(shortName.substring(1)); + } + } + + if (kind == null && endKind == null) { + String message = String.format("Unknown short name: %s (token = %s)", shortName, token); + throw new IllegalArgumentException(message); + } + + if (kind != null) { + ops.add(new Operation(kind, blockNumber)); + } else { + Operation op = null; + for (int i = ops.ops.size() - 1; i >= 0; i--) { + op = ops.ops.get(i); + if ((op.blockNumber == blockNumber) && (op.kind == endKind) && !(op instanceof End)) { + ops.add(new End(op)); + break; + } + } + + if (op == null) { + LOG.warn("Start op not found: {}({})", endKind, blockNumber); + } + } + } + + return ops; + } +} diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/impl/prefetch/BoundedResourcePool.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/impl/prefetch/BoundedResourcePool.java new file mode 100644 index 0000000000000..a871f8237729f --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/impl/prefetch/BoundedResourcePool.java @@ -0,0 +1,195 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hadoop.fs.impl.prefetch; + +import java.util.Collections; +import java.util.IdentityHashMap; +import java.util.Set; +import java.util.concurrent.ArrayBlockingQueue; + +import static org.apache.hadoop.fs.impl.prefetch.Validate.checkNotNull; + +/** + * Manages a fixed pool of resources. + * + * Avoids creating a new resource if a previously created instance is already available. + */ +public abstract class BoundedResourcePool extends ResourcePool { + /** + * The size of this pool. Fixed at creation time. + */ + private final int size; + + /** + * Items currently available in the pool. + */ + private ArrayBlockingQueue items; + + /** + * Items that have been created so far (regardless of whether they are currently available). + */ + private Set createdItems; + + /** + * Constructs a resource pool of the given size. + * + * @param size the size of this pool. Cannot be changed post creation. + * + * @throws IllegalArgumentException if size is zero or negative. + */ + public BoundedResourcePool(int size) { + Validate.checkPositiveInteger(size, "size"); + + this.size = size; + this.items = new ArrayBlockingQueue<>(size); + + // The created items are identified based on their object reference. + this.createdItems = Collections.newSetFromMap(new IdentityHashMap()); + } + + /** + * Acquires a resource blocking if necessary until one becomes available. + */ + @Override + public T acquire() { + return this.acquireHelper(true); + } + + /** + * Acquires a resource blocking if one is immediately available. Otherwise returns null. + */ + @Override + public T tryAcquire() { + return this.acquireHelper(false); + } + + /** + * Releases a previously acquired resource. + * + * @throws IllegalArgumentException if item is null. + */ + @Override + public void release(T item) { + checkNotNull(item, "item"); + + synchronized (createdItems) { + if (!createdItems.contains(item)) { + throw new IllegalArgumentException("This item is not a part of this pool"); + } + } + + // Return if this item was released earlier. + // We cannot use items.contains() because that check is not based on reference equality. + for (T entry : items) { + if (entry == item) { + return; + } + } + + try { + items.put(item); + } catch (InterruptedException e) { + throw new IllegalStateException("release() should never block", e); + } + } + + @Override + public synchronized void close() { + for (T item : createdItems) { + close(item); + } + + items.clear(); + items = null; + + createdItems.clear(); + createdItems = null; + } + + /** + * Derived classes may implement a way to cleanup each item. + */ + @Override + protected synchronized void close(T item) { + // Do nothing in this class. Allow overriding classes to take any cleanup action. + } + + /** + * Number of items created so far. Mostly for testing purposes. + * @return the count. + */ + public int numCreated() { + synchronized (createdItems) { + return createdItems.size(); + } + } + + /** + * Number of items available to be acquired. Mostly for testing purposes. + * @return the number available. + */ + public synchronized int numAvailable() { + return (size - numCreated()) + items.size(); + } + + // For debugging purposes. + @Override + public synchronized String toString() { + return String.format( + "size = %d, #created = %d, #in-queue = %d, #available = %d", + size, numCreated(), items.size(), numAvailable()); + } + + /** + * Derived classes must implement a way to create an instance of a resource. + */ + protected abstract T createNew(); + + private T acquireHelper(boolean canBlock) { + + // Prefer reusing an item if one is available. + // That avoids unnecessarily creating new instances. + T result = items.poll(); + if (result != null) { + return result; + } + + synchronized (createdItems) { + // Create a new instance if allowed by the capacity of this pool. + if (createdItems.size() < size) { + T item = createNew(); + createdItems.add(item); + return item; + } + } + + if (canBlock) { + try { + // Block for an instance to be available. + return items.take(); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + return null; + } + } else { + return null; + } + } +} diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/impl/prefetch/BufferData.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/impl/prefetch/BufferData.java new file mode 100644 index 0000000000000..de68269ab700c --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/impl/prefetch/BufferData.java @@ -0,0 +1,319 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hadoop.fs.impl.prefetch; + +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.Future; +import java.util.zip.CRC32; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Holds the state of a ByteBuffer that is in use by {@code CachingBlockManager}. + * + * This class is not meant to be of general use. It exists into its own file due to its size. + * We use the term block and buffer interchangeably in this file because one buffer + * holds exactly one block of data. + * + * Holding all of the state associated with a block allows us to validate and control + * state transitions in a synchronized fashion. + */ +public final class BufferData { + + private static final Logger LOG = LoggerFactory.getLogger(BufferData.class); + + public enum State { + /** + * Unknown / invalid state. + */ + UNKNOWN, + + /** + * Buffer has been acquired but has no data. + */ + BLANK, + + /** + * This block is being prefetched. + */ + PREFETCHING, + + /** + * This block is being added to the local cache. + */ + CACHING, + + /** + * This block has data and is ready to be read. + */ + READY, + + /** + * This block is no longer in-use and should not be used once in this state. + */ + DONE + } + + /** + * Number of the block associated with this buffer. + */ + private final int blockNumber; + + /** + * The buffer associated with this block. + */ + private ByteBuffer buffer; + + /** + * Current state of this block. + */ + private volatile State state; + + /** + * Future of the action being performed on this block (eg, prefetching or caching). + */ + private Future action; + + /** + * Checksum of the buffer contents once in READY state. + */ + private long checksum = 0; + + /** + * Constructs an instances of this class. + * + * @param blockNumber Number of the block associated with this buffer. + * @param buffer The buffer associated with this block. + * + * @throws IllegalArgumentException if blockNumber is negative. + * @throws IllegalArgumentException if buffer is null. + */ + public BufferData(int blockNumber, ByteBuffer buffer) { + Validate.checkNotNegative(blockNumber, "blockNumber"); + Validate.checkNotNull(buffer, "buffer"); + + this.blockNumber = blockNumber; + this.buffer = buffer; + this.state = State.BLANK; + } + + /** + * Gets the id of this block. + * + * @return the id of this block. + */ + public int getBlockNumber() { + return this.blockNumber; + } + + /** + * Gets the buffer associated with this block. + * + * @return the buffer associated with this block. + */ + public ByteBuffer getBuffer() { + return this.buffer; + } + + /** + * Gets the state of this block. + * + * @return the state of this block. + */ + public State getState() { + return this.state; + } + + /** + * Gets the checksum of data in this block. + * + * @return the checksum of data in this block. + */ + public long getChecksum() { + return this.checksum; + } + + /** + * Computes CRC32 checksum of the given buffer's contents. + * + * @param buffer the buffer whose content's checksum is to be computed. + * @return the computed checksum. + */ + public static long getChecksum(ByteBuffer buffer) { + ByteBuffer tempBuffer = buffer.duplicate(); + tempBuffer.rewind(); + CRC32 crc32 = new CRC32(); + crc32.update(tempBuffer); + return crc32.getValue(); + } + + public synchronized Future getActionFuture() { + return this.action; + } + + /** + * Indicates that a prefetch operation is in progress. + * + * @param actionFuture the {@code Future} of a prefetch action. + * + * @throws IllegalArgumentException if actionFuture is null. + */ + public synchronized void setPrefetch(Future actionFuture) { + Validate.checkNotNull(actionFuture, "actionFuture"); + + this.updateState(State.PREFETCHING, State.BLANK); + this.action = actionFuture; + } + + /** + * Indicates that a caching operation is in progress. + * + * @param actionFuture the {@code Future} of a caching action. + * + * @throws IllegalArgumentException if actionFuture is null. + */ + public synchronized void setCaching(Future actionFuture) { + Validate.checkNotNull(actionFuture, "actionFuture"); + + this.throwIfStateIncorrect(State.PREFETCHING, State.READY); + this.state = State.CACHING; + this.action = actionFuture; + } + + /** + * Marks the completion of reading data into the buffer. + * The buffer cannot be modified once in this state. + * + * @param expectedCurrentState the collection of states from which transition to READY is allowed. + */ + public synchronized void setReady(State... expectedCurrentState) { + if (this.checksum != 0) { + throw new IllegalStateException("Checksum cannot be changed once set"); + } + + this.buffer = this.buffer.asReadOnlyBuffer(); + this.checksum = getChecksum(this.buffer); + this.buffer.rewind(); + this.updateState(State.READY, expectedCurrentState); + } + + /** + * Indicates that this block is no longer of use and can be reclaimed. + */ + public synchronized void setDone() { + if (this.checksum != 0) { + if (getChecksum(this.buffer) != this.checksum) { + throw new IllegalStateException("checksum changed after setReady()"); + } + } + this.state = State.DONE; + this.action = null; + } + + /** + * Updates the current state to the specified value. + * Asserts that the current state is as expected. + * @param newState the state to transition to. + * @param expectedCurrentState the collection of states from which + * transition to {@code newState} is allowed. + * + * @throws IllegalArgumentException if newState is null. + * @throws IllegalArgumentException if expectedCurrentState is null. + */ + public synchronized void updateState(State newState, + State... expectedCurrentState) { + Validate.checkNotNull(newState, "newState"); + Validate.checkNotNull(expectedCurrentState, "expectedCurrentState"); + + this.throwIfStateIncorrect(expectedCurrentState); + this.state = newState; + } + + /** + * Helper that asserts the current state is one of the expected values. + * + * @param states the collection of allowed states. + * + * @throws IllegalArgumentException if states is null. + */ + public void throwIfStateIncorrect(State... states) { + Validate.checkNotNull(states, "states"); + + if (this.stateEqualsOneOf(states)) { + return; + } + + List statesStr = new ArrayList(); + for (State s : states) { + statesStr.add(s.toString()); + } + + String message = String.format( + "Expected buffer state to be '%s' but found: %s", + String.join(" or ", statesStr), this); + throw new IllegalStateException(message); + } + + public boolean stateEqualsOneOf(State... states) { + State currentState = this.state; + + for (State s : states) { + if (currentState == s) { + return true; + } + } + + return false; + } + + public String toString() { + + return String.format( + "[%03d] id: %03d, %s: buf: %s, checksum: %d, future: %s", + this.blockNumber, + System.identityHashCode(this), + this.state, + this.getBufferStr(this.buffer), + this.checksum, + this.getFutureStr(this.action)); + } + + private String getFutureStr(Future f) { + if (f == null) { + return "--"; + } else { + return this.action.isDone() ? "done" : "not done"; + } + } + + private String getBufferStr(ByteBuffer buf) { + if (buf == null) { + return "--"; + } else { + return String.format( + "(id = %d, pos = %d, lim = %d)", + System.identityHashCode(buf), + buf.position(), buf.limit()); + } + } +} diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/impl/prefetch/BufferPool.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/impl/prefetch/BufferPool.java new file mode 100644 index 0000000000000..189357f6bd04f --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/impl/prefetch/BufferPool.java @@ -0,0 +1,323 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hadoop.fs.impl.prefetch; + +import java.io.Closeable; +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.Collections; +import java.util.IdentityHashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.Future; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import static java.util.Objects.requireNonNull; +import static org.apache.hadoop.fs.impl.prefetch.Validate.checkNotNegative; +import static org.apache.hadoop.fs.impl.prefetch.Validate.checkState; +import static org.apache.hadoop.util.Preconditions.checkArgument; +import static org.apache.hadoop.util.Preconditions.checkNotNull; + +/** + * Manages a fixed pool of {@code ByteBuffer} instances. + *

    + * Avoids creating a new buffer if a previously created buffer is already available. + */ +public class BufferPool implements Closeable { + + private static final Logger LOG = LoggerFactory.getLogger(BufferPool.class); + + /** + * Max number of buffers in this pool. + */ + private final int size; + + /** + * Size in bytes of each buffer. + */ + private final int bufferSize; + + /* + Invariants for internal state. + -- a buffer is either in this.pool or in this.allocated + -- transition between this.pool <==> this.allocated must be atomic + -- only one buffer allocated for a given blockNumber + */ + + + /** + * Underlying bounded resource pool. + */ + private BoundedResourcePool pool; + + /** + * Allows associating metadata to each buffer in the pool. + */ + private Map allocated; + + /** + * Prefetching stats. + */ + private PrefetchingStatistics prefetchingStatistics; + + /** + * Initializes a new instance of the {@code BufferPool} class. + * @param size number of buffer in this pool. + * @param bufferSize size in bytes of each buffer. + * @param prefetchingStatistics statistics for this stream. + * @throws IllegalArgumentException if size is zero or negative. + * @throws IllegalArgumentException if bufferSize is zero or negative. + */ + public BufferPool(int size, + int bufferSize, + PrefetchingStatistics prefetchingStatistics) { + Validate.checkPositiveInteger(size, "size"); + Validate.checkPositiveInteger(bufferSize, "bufferSize"); + + this.size = size; + this.bufferSize = bufferSize; + this.allocated = new IdentityHashMap(); + this.prefetchingStatistics = requireNonNull(prefetchingStatistics); + this.pool = new BoundedResourcePool(size) { + @Override + public ByteBuffer createNew() { + ByteBuffer buffer = ByteBuffer.allocate(bufferSize); + prefetchingStatistics.memoryAllocated(bufferSize); + return buffer; + } + }; + } + + /** + * Gets a list of all blocks in this pool. + * @return a list of all blocks in this pool. + */ + public List getAll() { + synchronized (allocated) { + return Collections.unmodifiableList(new ArrayList<>(allocated.keySet())); + } + } + + /** + * Acquires a {@code ByteBuffer}; blocking if necessary until one becomes available. + * @param blockNumber the id of the block to acquire. + * @return the acquired block's {@code BufferData}. + */ + public synchronized BufferData acquire(int blockNumber) { + BufferData data; + final int maxRetryDelayMs = 600 * 1000; + final int statusUpdateDelayMs = 120 * 1000; + Retryer retryer = new Retryer(10, maxRetryDelayMs, statusUpdateDelayMs); + + do { + if (retryer.updateStatus()) { + if (LOG.isDebugEnabled()) { + LOG.debug("waiting to acquire block: {}", blockNumber); + LOG.debug("state = {}", this); + } + releaseReadyBlock(blockNumber); + } + data = tryAcquire(blockNumber); + } + while ((data == null) && retryer.continueRetry()); + + if (data != null) { + return data; + } else { + String message = + String.format("Wait failed for acquire(%d)", blockNumber); + throw new IllegalStateException(message); + } + } + + /** + * Acquires a buffer if one is immediately available. Otherwise returns null. + * @param blockNumber the id of the block to try acquire. + * @return the acquired block's {@code BufferData} or null. + */ + public synchronized BufferData tryAcquire(int blockNumber) { + return acquireHelper(blockNumber, false); + } + + private synchronized BufferData acquireHelper(int blockNumber, + boolean canBlock) { + checkNotNegative(blockNumber, "blockNumber"); + + releaseDoneBlocks(); + + BufferData data = find(blockNumber); + if (data != null) { + return data; + } + + ByteBuffer buffer = canBlock ? pool.acquire() : pool.tryAcquire(); + if (buffer == null) { + return null; + } + + buffer.clear(); + data = new BufferData(blockNumber, buffer.duplicate()); + + synchronized (allocated) { + checkState(find(blockNumber) == null, "buffer data already exists"); + + allocated.put(data, buffer); + } + + return data; + } + + /** + * Releases resources for any blocks marked as 'done'. + */ + private synchronized void releaseDoneBlocks() { + for (BufferData data : getAll()) { + if (data.stateEqualsOneOf(BufferData.State.DONE)) { + release(data); + } + } + } + + /** + * If no blocks were released after calling releaseDoneBlocks() a few times, + * we may end up waiting forever. To avoid that situation, we try releasing + * a 'ready' block farthest away from the given block. + */ + private synchronized void releaseReadyBlock(int blockNumber) { + BufferData releaseTarget = null; + for (BufferData data : getAll()) { + if (data.stateEqualsOneOf(BufferData.State.READY)) { + if (releaseTarget == null) { + releaseTarget = data; + } else { + if (distance(data, blockNumber) > distance(releaseTarget, + blockNumber)) { + releaseTarget = data; + } + } + } + } + + if (releaseTarget != null) { + LOG.warn("releasing 'ready' block: {}", releaseTarget); + releaseTarget.setDone(); + } + } + + private int distance(BufferData data, int blockNumber) { + return Math.abs(data.getBlockNumber() - blockNumber); + } + + /** + * Releases a previously acquired resource. + * @param data the {@code BufferData} instance to release. + * @throws IllegalArgumentException if data is null. + * @throws IllegalArgumentException if data cannot be released due to its state. + */ + public synchronized void release(BufferData data) { + checkNotNull(data, "data"); + + synchronized (data) { + checkArgument( + canRelease(data), + String.format("Unable to release buffer: %s", data)); + + ByteBuffer buffer = allocated.get(data); + if (buffer == null) { + // Likely released earlier. + return; + } + buffer.clear(); + pool.release(buffer); + allocated.remove(data); + } + + releaseDoneBlocks(); + } + + @Override + public synchronized void close() { + for (BufferData data : getAll()) { + Future actionFuture = data.getActionFuture(); + if (actionFuture != null) { + actionFuture.cancel(true); + } + } + + int currentPoolSize = pool.numCreated(); + + pool.close(); + pool = null; + + allocated.clear(); + allocated = null; + + prefetchingStatistics.memoryFreed(currentPoolSize * bufferSize); + } + + // For debugging purposes. + @Override + public String toString() { + StringBuilder sb = new StringBuilder(); + sb.append(pool.toString()); + sb.append("\n"); + List allData = new ArrayList<>(getAll()); + Collections.sort(allData, + (d1, d2) -> d1.getBlockNumber() - d2.getBlockNumber()); + for (BufferData data : allData) { + sb.append(data.toString()); + sb.append("\n"); + } + + return sb.toString(); + } + + // Number of ByteBuffers created so far. + public synchronized int numCreated() { + return pool.numCreated(); + } + + // Number of ByteBuffers available to be acquired. + public synchronized int numAvailable() { + releaseDoneBlocks(); + return pool.numAvailable(); + } + + private BufferData find(int blockNumber) { + synchronized (allocated) { + for (BufferData data : allocated.keySet()) { + if ((data.getBlockNumber() == blockNumber) + && !data.stateEqualsOneOf(BufferData.State.DONE)) { + return data; + } + } + } + + return null; + } + + private boolean canRelease(BufferData data) { + return data.stateEqualsOneOf( + BufferData.State.DONE, + BufferData.State.READY); + } +} diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/impl/prefetch/CachingBlockManager.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/impl/prefetch/CachingBlockManager.java new file mode 100644 index 0000000000000..31084c7bf2648 --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/impl/prefetch/CachingBlockManager.java @@ -0,0 +1,638 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hadoop.fs.impl.prefetch; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.time.Duration; +import java.time.Instant; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.function.Supplier; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import org.apache.hadoop.fs.statistics.DurationTracker; + +import static java.util.Objects.requireNonNull; + +import static org.apache.hadoop.fs.impl.prefetch.Validate.checkNotNegative; +import static org.apache.hadoop.io.IOUtils.cleanupWithLogger; + +/** + * Provides read access to the underlying file one block at a time. + * Improve read performance by prefetching and locall caching blocks. + */ +public abstract class CachingBlockManager extends BlockManager { + private static final Logger LOG = LoggerFactory.getLogger(CachingBlockManager.class); + private static final int TIMEOUT_MINUTES = 60; + + /** + * Asynchronous tasks are performed in this pool. + */ + private final ExecutorServiceFuturePool futurePool; + + /** + * Pool of shared ByteBuffer instances. + */ + private BufferPool bufferPool; + + /** + * Size of the in-memory cache in terms of number of blocks. + * Total memory consumption is up to bufferPoolSize * blockSize. + */ + private final int bufferPoolSize; + + /** + * Local block cache. + */ + private BlockCache cache; + + /** + * Error counts. For testing purposes. + */ + private final AtomicInteger numCachingErrors; + private final AtomicInteger numReadErrors; + + /** + * Operations performed by this block manager. + */ + private final BlockOperations ops; + + private boolean closed; + + /** + * If a single caching operation takes more than this time (in seconds), + * we disable caching to prevent further perf degradation due to caching. + */ + private static final int SLOW_CACHING_THRESHOLD = 5; + + /** + * Once set to true, any further caching requests will be ignored. + */ + private final AtomicBoolean cachingDisabled; + + private final PrefetchingStatistics prefetchingStatistics; + + /** + * Constructs an instance of a {@code CachingBlockManager}. + * + * @param futurePool asynchronous tasks are performed in this pool. + * @param blockData information about each block of the underlying file. + * @param bufferPoolSize size of the in-memory cache in terms of number of blocks. + * @param prefetchingStatistics statistics for this stream. + * + * @throws IllegalArgumentException if bufferPoolSize is zero or negative. + */ + public CachingBlockManager( + ExecutorServiceFuturePool futurePool, + BlockData blockData, + int bufferPoolSize, + PrefetchingStatistics prefetchingStatistics) { + super(blockData); + + Validate.checkPositiveInteger(bufferPoolSize, "bufferPoolSize"); + + this.futurePool = requireNonNull(futurePool); + this.bufferPoolSize = bufferPoolSize; + this.numCachingErrors = new AtomicInteger(); + this.numReadErrors = new AtomicInteger(); + this.cachingDisabled = new AtomicBoolean(); + this.prefetchingStatistics = requireNonNull(prefetchingStatistics); + + if (this.getBlockData().getFileSize() > 0) { + this.bufferPool = new BufferPool(bufferPoolSize, this.getBlockData().getBlockSize(), + this.prefetchingStatistics); + this.cache = this.createCache(); + } + + this.ops = new BlockOperations(); + this.ops.setDebug(false); + } + + /** + * Gets the block having the given {@code blockNumber}. + * + * @throws IllegalArgumentException if blockNumber is negative. + */ + @Override + public BufferData get(int blockNumber) throws IOException { + checkNotNegative(blockNumber, "blockNumber"); + + BufferData data; + final int maxRetryDelayMs = bufferPoolSize * 120 * 1000; + final int statusUpdateDelayMs = 120 * 1000; + Retryer retryer = new Retryer(10, maxRetryDelayMs, statusUpdateDelayMs); + boolean done; + + do { + if (closed) { + throw new IOException("this stream is already closed"); + } + + data = bufferPool.acquire(blockNumber); + done = getInternal(data); + + if (retryer.updateStatus()) { + LOG.warn("waiting to get block: {}", blockNumber); + LOG.info("state = {}", this.toString()); + } + } + while (!done && retryer.continueRetry()); + + if (done) { + return data; + } else { + String message = String.format("Wait failed for get(%d)", blockNumber); + throw new IllegalStateException(message); + } + } + + private boolean getInternal(BufferData data) throws IOException { + Validate.checkNotNull(data, "data"); + + // Opportunistic check without locking. + if (data.stateEqualsOneOf( + BufferData.State.PREFETCHING, + BufferData.State.CACHING, + BufferData.State.DONE)) { + return false; + } + + synchronized (data) { + // Reconfirm state after locking. + if (data.stateEqualsOneOf( + BufferData.State.PREFETCHING, + BufferData.State.CACHING, + BufferData.State.DONE)) { + return false; + } + + int blockNumber = data.getBlockNumber(); + if (data.getState() == BufferData.State.READY) { + BlockOperations.Operation op = ops.getPrefetched(blockNumber); + ops.end(op); + return true; + } + + data.throwIfStateIncorrect(BufferData.State.BLANK); + read(data); + return true; + } + } + + /** + * Releases resources allocated to the given block. + * + * @throws IllegalArgumentException if data is null. + */ + @Override + public void release(BufferData data) { + if (closed) { + return; + } + + Validate.checkNotNull(data, "data"); + + BlockOperations.Operation op = ops.release(data.getBlockNumber()); + bufferPool.release(data); + ops.end(op); + } + + @Override + public synchronized void close() { + if (closed) { + return; + } + + closed = true; + + final BlockOperations.Operation op = ops.close(); + + // Cancel any prefetches in progress. + cancelPrefetches(); + + cleanupWithLogger(LOG, cache); + + ops.end(op); + LOG.info(ops.getSummary(false)); + + bufferPool.close(); + bufferPool = null; + } + + /** + * Requests optional prefetching of the given block. + * The block is prefetched only if we can acquire a free buffer. + * + * @throws IllegalArgumentException if blockNumber is negative. + */ + @Override + public void requestPrefetch(int blockNumber) { + checkNotNegative(blockNumber, "blockNumber"); + + if (closed) { + return; + } + + // We initiate a prefetch only if we can acquire a buffer from the shared pool. + BufferData data = bufferPool.tryAcquire(blockNumber); + if (data == null) { + return; + } + + // Opportunistic check without locking. + if (!data.stateEqualsOneOf(BufferData.State.BLANK)) { + // The block is ready or being prefetched/cached. + return; + } + + synchronized (data) { + // Reconfirm state after locking. + if (!data.stateEqualsOneOf(BufferData.State.BLANK)) { + // The block is ready or being prefetched/cached. + return; + } + + BlockOperations.Operation op = ops.requestPrefetch(blockNumber); + PrefetchTask prefetchTask = new PrefetchTask(data, this, Instant.now()); + Future prefetchFuture = futurePool.executeFunction(prefetchTask); + data.setPrefetch(prefetchFuture); + ops.end(op); + } + } + + /** + * Requests cancellation of any previously issued prefetch requests. + */ + @Override + public void cancelPrefetches() { + BlockOperations.Operation op = ops.cancelPrefetches(); + + for (BufferData data : bufferPool.getAll()) { + // We add blocks being prefetched to the local cache so that the prefetch is not wasted. + if (data.stateEqualsOneOf(BufferData.State.PREFETCHING, BufferData.State.READY)) { + requestCaching(data); + } + } + + ops.end(op); + } + + private void read(BufferData data) throws IOException { + synchronized (data) { + readBlock(data, false, BufferData.State.BLANK); + } + } + + private void prefetch(BufferData data, Instant taskQueuedStartTime) throws IOException { + synchronized (data) { + prefetchingStatistics.executorAcquired( + Duration.between(taskQueuedStartTime, Instant.now())); + readBlock( + data, + true, + BufferData.State.PREFETCHING, + BufferData.State.CACHING); + } + } + + private void readBlock(BufferData data, boolean isPrefetch, BufferData.State... expectedState) + throws IOException { + + if (closed) { + return; + } + + BlockOperations.Operation op = null; + DurationTracker tracker = null; + + synchronized (data) { + try { + if (data.stateEqualsOneOf(BufferData.State.DONE, BufferData.State.READY)) { + // DONE : Block was released, likely due to caching being disabled on slow perf. + // READY : Block was already fetched by another thread. No need to re-read. + return; + } + + data.throwIfStateIncorrect(expectedState); + int blockNumber = data.getBlockNumber(); + + // Prefer reading from cache over reading from network. + if (cache.containsBlock(blockNumber)) { + op = ops.getCached(blockNumber); + cache.get(blockNumber, data.getBuffer()); + data.setReady(expectedState); + return; + } + + if (isPrefetch) { + tracker = prefetchingStatistics.prefetchOperationStarted(); + op = ops.prefetch(data.getBlockNumber()); + } else { + op = ops.getRead(data.getBlockNumber()); + } + + long offset = getBlockData().getStartOffset(data.getBlockNumber()); + int size = getBlockData().getSize(data.getBlockNumber()); + ByteBuffer buffer = data.getBuffer(); + buffer.clear(); + read(buffer, offset, size); + buffer.flip(); + data.setReady(expectedState); + } catch (Exception e) { + String message = String.format("error during readBlock(%s)", data.getBlockNumber()); + LOG.error(message, e); + + if (isPrefetch && tracker != null) { + tracker.failed(); + } + + numReadErrors.incrementAndGet(); + data.setDone(); + throw e; + } finally { + if (op != null) { + ops.end(op); + } + + if (isPrefetch) { + prefetchingStatistics.prefetchOperationCompleted(); + if (tracker != null) { + tracker.close(); + } + } + } + } + } + + /** + * Read task that is submitted to the future pool. + */ + private static class PrefetchTask implements Supplier { + private final BufferData data; + private final CachingBlockManager blockManager; + private final Instant taskQueuedStartTime; + + PrefetchTask(BufferData data, CachingBlockManager blockManager, Instant taskQueuedStartTime) { + this.data = data; + this.blockManager = blockManager; + this.taskQueuedStartTime = taskQueuedStartTime; + } + + @Override + public Void get() { + try { + blockManager.prefetch(data, taskQueuedStartTime); + } catch (Exception e) { + LOG.error("error during prefetch", e); + } + return null; + } + } + + private static final BufferData.State[] EXPECTED_STATE_AT_CACHING = + new BufferData.State[] { + BufferData.State.PREFETCHING, BufferData.State.READY + }; + + /** + * Requests that the given block should be copied to the local cache. + * The block must not be accessed by the caller after calling this method + * because it will released asynchronously relative to the caller. + * + * @throws IllegalArgumentException if data is null. + */ + @Override + public void requestCaching(BufferData data) { + if (closed) { + return; + } + + if (cachingDisabled.get()) { + data.setDone(); + return; + } + + Validate.checkNotNull(data, "data"); + + // Opportunistic check without locking. + if (!data.stateEqualsOneOf(EXPECTED_STATE_AT_CACHING)) { + return; + } + + synchronized (data) { + // Reconfirm state after locking. + if (!data.stateEqualsOneOf(EXPECTED_STATE_AT_CACHING)) { + return; + } + + if (cache.containsBlock(data.getBlockNumber())) { + data.setDone(); + return; + } + + BufferData.State state = data.getState(); + + BlockOperations.Operation op = ops.requestCaching(data.getBlockNumber()); + Future blockFuture; + if (state == BufferData.State.PREFETCHING) { + blockFuture = data.getActionFuture(); + } else { + CompletableFuture cf = new CompletableFuture<>(); + cf.complete(null); + blockFuture = cf; + } + + CachePutTask task = new CachePutTask(data, blockFuture, this, Instant.now()); + Future actionFuture = futurePool.executeFunction(task); + data.setCaching(actionFuture); + ops.end(op); + } + } + + private void addToCacheAndRelease(BufferData data, Future blockFuture, + Instant taskQueuedStartTime) { + prefetchingStatistics.executorAcquired( + Duration.between(taskQueuedStartTime, Instant.now())); + + if (closed) { + return; + } + + if (cachingDisabled.get()) { + data.setDone(); + return; + } + + try { + blockFuture.get(TIMEOUT_MINUTES, TimeUnit.MINUTES); + if (data.stateEqualsOneOf(BufferData.State.DONE)) { + // There was an error during prefetch. + return; + } + } catch (Exception e) { + LOG.error("error waiting on blockFuture: {}", data, e); + data.setDone(); + return; + } + + if (cachingDisabled.get()) { + data.setDone(); + return; + } + + BlockOperations.Operation op = null; + + synchronized (data) { + try { + if (data.stateEqualsOneOf(BufferData.State.DONE)) { + return; + } + + if (cache.containsBlock(data.getBlockNumber())) { + data.setDone(); + return; + } + + op = ops.addToCache(data.getBlockNumber()); + ByteBuffer buffer = data.getBuffer().duplicate(); + buffer.rewind(); + cachePut(data.getBlockNumber(), buffer); + data.setDone(); + } catch (Exception e) { + numCachingErrors.incrementAndGet(); + String message = String.format("error adding block to cache after wait: %s", data); + LOG.error(message, e); + data.setDone(); + } + + if (op != null) { + BlockOperations.End endOp = (BlockOperations.End) ops.end(op); + if (endOp.duration() > SLOW_CACHING_THRESHOLD) { + if (!cachingDisabled.getAndSet(true)) { + String message = String.format( + "Caching disabled because of slow operation (%.1f sec)", endOp.duration()); + LOG.warn(message); + } + } + } + } + } + + protected BlockCache createCache() { + return new SingleFilePerBlockCache(prefetchingStatistics); + } + + protected void cachePut(int blockNumber, ByteBuffer buffer) throws IOException { + if (closed) { + return; + } + + cache.put(blockNumber, buffer); + } + + private static class CachePutTask implements Supplier { + private final BufferData data; + + // Block being asynchronously fetched. + private final Future blockFuture; + + // Block manager that manages this block. + private final CachingBlockManager blockManager; + + private final Instant taskQueuedStartTime; + + CachePutTask( + BufferData data, + Future blockFuture, + CachingBlockManager blockManager, + Instant taskQueuedStartTime) { + this.data = data; + this.blockFuture = blockFuture; + this.blockManager = blockManager; + this.taskQueuedStartTime = taskQueuedStartTime; + } + + @Override + public Void get() { + blockManager.addToCacheAndRelease(data, blockFuture, taskQueuedStartTime); + return null; + } + } + + /** + * Number of ByteBuffers available to be acquired. + * + * @return the number of available buffers. + */ + public int numAvailable() { + return bufferPool.numAvailable(); + } + + /** + * Number of caching operations completed. + * + * @return the number of cached buffers. + */ + public int numCached() { + return cache.size(); + } + + /** + * Number of errors encountered when caching. + * + * @return the number of errors encountered when caching. + */ + public int numCachingErrors() { + return numCachingErrors.get(); + } + + /** + * Number of errors encountered when reading. + * + * @return the number of errors encountered when reading. + */ + public int numReadErrors() { + return numReadErrors.get(); + } + + BufferData getData(int blockNumber) { + return bufferPool.tryAcquire(blockNumber); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder(); + + sb.append("cache("); + sb.append(cache.toString()); + sb.append("); "); + + sb.append("pool: "); + sb.append(bufferPool.toString()); + + return sb.toString(); + } +} diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/impl/prefetch/EmptyPrefetchingStatistics.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/impl/prefetch/EmptyPrefetchingStatistics.java new file mode 100644 index 0000000000000..177ff7abab8b7 --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/impl/prefetch/EmptyPrefetchingStatistics.java @@ -0,0 +1,80 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hadoop.fs.impl.prefetch; + +import java.time.Duration; + +import org.apache.hadoop.fs.statistics.DurationTracker; + +import static org.apache.hadoop.fs.statistics.IOStatisticsSupport.stubDurationTracker; + +/** + * Empty implementation of the prefetching statistics interface. + */ +public final class EmptyPrefetchingStatistics + implements PrefetchingStatistics { + + private static final EmptyPrefetchingStatistics + EMPTY_PREFETCHING_STATISTICS = + new EmptyPrefetchingStatistics(); + + private EmptyPrefetchingStatistics() { + } + + public static EmptyPrefetchingStatistics getInstance() { + return EMPTY_PREFETCHING_STATISTICS; + } + + @Override + public DurationTracker prefetchOperationStarted() { + return stubDurationTracker(); + } + + @Override + public void blockAddedToFileCache() { + + } + + @Override + public void blockRemovedFromFileCache() { + + } + + @Override + public void prefetchOperationCompleted() { + + } + + @Override + public void executorAcquired(Duration timeInQueue) { + + } + + @Override + public void memoryAllocated(int size) { + + } + + @Override + public void memoryFreed(int size) { + + } +} + diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/impl/prefetch/ExecutorServiceFuturePool.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/impl/prefetch/ExecutorServiceFuturePool.java new file mode 100644 index 0000000000000..9ef50e50d7e5e --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/impl/prefetch/ExecutorServiceFuturePool.java @@ -0,0 +1,70 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hadoop.fs.impl.prefetch; + +import java.util.Locale; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Future; +import java.util.function.Supplier; + +/** + * A FuturePool implementation backed by a java.util.concurrent.ExecutorService. + * + * If a piece of work has started, it cannot (currently) be cancelled. + * + * This class is a simplified version of com.twitter:util-core_2.11 + * ExecutorServiceFuturePool designed to avoid depending on that Scala library. + * One problem with using a Scala library is that many downstream projects + * (eg Apache Spark) use Scala, and they might want to use a different version of Scala + * from the version that Hadoop chooses to use. + * + */ +public class ExecutorServiceFuturePool { + private ExecutorService executor; + + public ExecutorServiceFuturePool(ExecutorService executor) { + this.executor = executor; + } + + /** + * @param f function to run in future on executor pool + * @return future + * @throws java.util.concurrent.RejectedExecutionException can be thrown + * @throws NullPointerException if f param is null + */ + public Future executeFunction(final Supplier f) { + return executor.submit(f::get); + } + + /** + * @param r runnable to run in future on executor pool + * @return future + * @throws java.util.concurrent.RejectedExecutionException can be thrown + * @throws NullPointerException if r param is null + */ + @SuppressWarnings("unchecked") + public Future executeRunnable(final Runnable r) { + return (Future) executor.submit(r::run); + } + + public String toString() { + return String.format(Locale.ROOT, "ExecutorServiceFuturePool(executor=%s)", executor); + } +} diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/impl/prefetch/FilePosition.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/impl/prefetch/FilePosition.java new file mode 100644 index 0000000000000..7cd3bb3de2b58 --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/impl/prefetch/FilePosition.java @@ -0,0 +1,301 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hadoop.fs.impl.prefetch; + +import java.nio.ByteBuffer; + +import static org.apache.hadoop.fs.impl.prefetch.Validate.checkNotNegative; +import static org.apache.hadoop.fs.impl.prefetch.Validate.checkNotNull; +import static org.apache.hadoop.fs.impl.prefetch.Validate.checkPositiveInteger; +import static org.apache.hadoop.fs.impl.prefetch.Validate.checkState; +import static org.apache.hadoop.fs.impl.prefetch.Validate.checkWithinRange; + +/** + * Provides functionality related to tracking the position within a file. + * + * The file is accessed through an in memory buffer. The absolute position within + * the file is the sum of start offset of the buffer within the file and the relative + * offset of the current access location within the buffer. + * + * A file is made up of equal sized blocks. The last block may be of a smaller size. + * The size of a buffer associated with this file is typically the same as block size. + */ +public final class FilePosition { + + /** + * Holds block based information about a file. + */ + private BlockData blockData; + + /** + * Information about the buffer in use. + */ + private BufferData data; + + /** + * Provides access to the underlying file. + */ + private ByteBuffer buffer; + + /** + * Start offset of the buffer relative to the start of a file. + */ + private long bufferStartOffset; + + /** + * Offset where reading starts relative to the start of a file. + */ + private long readStartOffset; + + // Read stats after a seek (mostly for debugging use). + private int numSingleByteReads; + + private int numBytesRead; + + private int numBufferReads; + + /** + * Constructs an instance of {@link FilePosition}. + * + * @param fileSize size of the associated file. + * @param blockSize size of each block within the file. + * + * @throws IllegalArgumentException if fileSize is negative. + * @throws IllegalArgumentException if blockSize is zero or negative. + */ + public FilePosition(long fileSize, int blockSize) { + checkNotNegative(fileSize, "fileSize"); + if (fileSize == 0) { + checkNotNegative(blockSize, "blockSize"); + } else { + checkPositiveInteger(blockSize, "blockSize"); + } + + this.blockData = new BlockData(fileSize, blockSize); + + // The position is valid only when a valid buffer is associated with this file. + this.invalidate(); + } + + /** + * Associates a buffer with this file. + * + * @param bufferData the buffer associated with this file. + * @param startOffset Start offset of the buffer relative to the start of a file. + * @param readOffset Offset where reading starts relative to the start of a file. + * + * @throws IllegalArgumentException if bufferData is null. + * @throws IllegalArgumentException if startOffset is negative. + * @throws IllegalArgumentException if readOffset is negative. + * @throws IllegalArgumentException if readOffset is outside the range [startOffset, buffer end]. + */ + public void setData(BufferData bufferData, + long startOffset, + long readOffset) { + checkNotNull(bufferData, "bufferData"); + checkNotNegative(startOffset, "startOffset"); + checkNotNegative(readOffset, "readOffset"); + checkWithinRange( + readOffset, + "readOffset", + startOffset, + startOffset + bufferData.getBuffer().limit() - 1); + + data = bufferData; + buffer = bufferData.getBuffer().duplicate(); + bufferStartOffset = startOffset; + readStartOffset = readOffset; + setAbsolute(readOffset); + + resetReadStats(); + } + + public ByteBuffer buffer() { + throwIfInvalidBuffer(); + return buffer; + } + + public BufferData data() { + throwIfInvalidBuffer(); + return data; + } + + /** + * Gets the current absolute position within this file. + * + * @return the current absolute position within this file. + */ + public long absolute() { + throwIfInvalidBuffer(); + return bufferStartOffset + relative(); + } + + /** + * If the given {@code pos} lies within the current buffer, updates the current position to + * the specified value and returns true; otherwise returns false without changing the position. + * + * @param pos the absolute position to change the current position to if possible. + * @return true if the given current position was updated, false otherwise. + */ + public boolean setAbsolute(long pos) { + if (isValid() && isWithinCurrentBuffer(pos)) { + int relativePos = (int) (pos - bufferStartOffset); + buffer.position(relativePos); + return true; + } else { + return false; + } + } + + /** + * Gets the current position within this file relative to the start of the associated buffer. + * + * @return the current position within this file relative to the start of the associated buffer. + */ + public int relative() { + throwIfInvalidBuffer(); + return buffer.position(); + } + + /** + * Determines whether the given absolute position lies within the current buffer. + * + * @param pos the position to check. + * @return true if the given absolute position lies within the current buffer, false otherwise. + */ + public boolean isWithinCurrentBuffer(long pos) { + throwIfInvalidBuffer(); + long bufferEndOffset = bufferStartOffset + buffer.limit() - 1; + return (pos >= bufferStartOffset) && (pos <= bufferEndOffset); + } + + /** + * Gets the id of the current block. + * + * @return the id of the current block. + */ + public int blockNumber() { + throwIfInvalidBuffer(); + return blockData.getBlockNumber(bufferStartOffset); + } + + /** + * Determines whether the current block is the last block in this file. + * + * @return true if the current block is the last block in this file, false otherwise. + */ + public boolean isLastBlock() { + return blockData.isLastBlock(blockNumber()); + } + + /** + * Determines if the current position is valid. + * + * @return true if the current position is valid, false otherwise. + */ + public boolean isValid() { + return buffer != null; + } + + /** + * Marks the current position as invalid. + */ + public void invalidate() { + buffer = null; + bufferStartOffset = -1; + data = null; + } + + /** + * Gets the start of the current block's absolute offset. + * + * @return the start of the current block's absolute offset. + */ + public long bufferStartOffset() { + throwIfInvalidBuffer(); + return bufferStartOffset; + } + + /** + * Determines whether the current buffer has been fully read. + * + * @return true if the current buffer has been fully read, false otherwise. + */ + public boolean bufferFullyRead() { + throwIfInvalidBuffer(); + return (bufferStartOffset == readStartOffset) + && (relative() == buffer.limit()) + && (numBytesRead == buffer.limit()); + } + + public void incrementBytesRead(int n) { + numBytesRead += n; + if (n == 1) { + numSingleByteReads++; + } else { + numBufferReads++; + } + } + + public int numBytesRead() { + return numBytesRead; + } + + public int numSingleByteReads() { + return numSingleByteReads; + } + + public int numBufferReads() { + return numBufferReads; + } + + private void resetReadStats() { + numBytesRead = 0; + numSingleByteReads = 0; + numBufferReads = 0; + } + + public String toString() { + StringBuilder sb = new StringBuilder(); + if (buffer == null) { + sb.append("currentBuffer = null"); + } else { + int pos = buffer.position(); + int val; + if (pos >= buffer.limit()) { + val = -1; + } else { + val = buffer.get(pos); + } + String currentBufferState = + String.format("%d at pos: %d, lim: %d", val, pos, buffer.limit()); + sb.append(String.format( + "block: %d, pos: %d (CBuf: %s)%n", + blockNumber(), absolute(), + currentBufferState)); + sb.append("\n"); + } + return sb.toString(); + } + + private void throwIfInvalidBuffer() { + checkState(buffer != null, "'buffer' must not be null"); + } +} diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/impl/prefetch/PrefetchingStatistics.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/impl/prefetch/PrefetchingStatistics.java new file mode 100644 index 0000000000000..9ce2dec5889f1 --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/impl/prefetch/PrefetchingStatistics.java @@ -0,0 +1,67 @@ + /* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hadoop.fs.impl.prefetch; + +import java.time.Duration; + +import org.apache.hadoop.fs.statistics.DurationTracker; +import org.apache.hadoop.fs.statistics.IOStatisticsSource; + +public interface PrefetchingStatistics extends IOStatisticsSource { + + /** + * A prefetch operation has started. + * @return duration tracker + */ + DurationTracker prefetchOperationStarted(); + + /** + * A block has been saved to the file cache. + */ + void blockAddedToFileCache(); + + /** + * A block has been removed from the file cache. + */ + void blockRemovedFromFileCache(); + + /** + * A prefetch operation has completed. + */ + void prefetchOperationCompleted(); + + /** + * An executor has been acquired, either for prefetching or caching. + * @param timeInQueue time taken to acquire an executor. + */ + void executorAcquired(Duration timeInQueue); + + /** + * A new buffer has been added to the buffer pool. + * @param size size of the new buffer + */ + void memoryAllocated(int size); + + /** + * Previously allocated memory has been freed. + * @param size size of memory freed. + */ + void memoryFreed(int size); +} diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/impl/prefetch/ResourcePool.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/impl/prefetch/ResourcePool.java new file mode 100644 index 0000000000000..77e00972d08c3 --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/impl/prefetch/ResourcePool.java @@ -0,0 +1,71 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hadoop.fs.impl.prefetch; + +import java.io.Closeable; + +/** + * Manages a fixed pool of resources. + * + * Avoids creating a new resource if a previously created instance is already available. + */ +public abstract class ResourcePool implements Closeable { + + /** + * Acquires a resource blocking if necessary until one becomes available. + * + * @return the acquired resource instance. + */ + public abstract T acquire(); + + /** + * Acquires a resource blocking if one is immediately available. Otherwise returns null. + + * @return the acquired resource instance (if immediately available) or null. + */ + public abstract T tryAcquire(); + + /** + * Releases a previously acquired resource. + * + * @param item the resource to release. + */ + public abstract void release(T item); + + @Override + public void close() { + } + + /** + * Derived classes may implement a way to cleanup each item. + * + * @param item the resource to close. + */ + protected void close(T item) { + // Do nothing in this class. Allow overriding classes to take any cleanup action. + } + + /** + * Derived classes must implement a way to create an instance of a resource. + * + * @return the created instance. + */ + protected abstract T createNew(); +} diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/impl/prefetch/Retryer.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/impl/prefetch/Retryer.java new file mode 100644 index 0000000000000..84c17ef9dde8a --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/impl/prefetch/Retryer.java @@ -0,0 +1,93 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hadoop.fs.impl.prefetch; + +import static org.apache.hadoop.fs.impl.prefetch.Validate.checkGreater; +import static org.apache.hadoop.fs.impl.prefetch.Validate.checkPositiveInteger; + +/** + * Provides retry related functionality. + */ +public class Retryer { + + /* Maximum amount of delay (in ms) before retry fails. */ + private int maxDelay; + + /* Per retry delay (in ms). */ + private int perRetryDelay; + + /** + * The time interval (in ms) at which status update would be made. + */ + private int statusUpdateInterval; + + /* Current delay. */ + private int delay; + + /** + * Initializes a new instance of the {@code Retryer} class. + * + * @param perRetryDelay per retry delay (in ms). + * @param maxDelay maximum amount of delay (in ms) before retry fails. + * @param statusUpdateInterval time interval (in ms) at which status update would be made. + * + * @throws IllegalArgumentException if perRetryDelay is zero or negative. + * @throws IllegalArgumentException if maxDelay is less than or equal to perRetryDelay. + * @throws IllegalArgumentException if statusUpdateInterval is zero or negative. + */ + public Retryer(int perRetryDelay, int maxDelay, int statusUpdateInterval) { + checkPositiveInteger(perRetryDelay, "perRetryDelay"); + checkGreater(maxDelay, "maxDelay", perRetryDelay, "perRetryDelay"); + checkPositiveInteger(statusUpdateInterval, "statusUpdateInterval"); + + this.perRetryDelay = perRetryDelay; + this.maxDelay = maxDelay; + this.statusUpdateInterval = statusUpdateInterval; + } + + /** + * Returns true if retrying should continue, false otherwise. + * + * @return true if the caller should retry, false otherwise. + */ + public boolean continueRetry() { + if (this.delay >= this.maxDelay) { + return false; + } + + try { + Thread.sleep(this.perRetryDelay); + } catch (InterruptedException e) { + // Ignore the exception as required by the semantic of this class; + } + + this.delay += this.perRetryDelay; + return true; + } + + /** + * Returns true if status update interval has been reached. + * + * @return true if status update interval has been reached. + */ + public boolean updateStatus() { + return (this.delay > 0) && this.delay % this.statusUpdateInterval == 0; + } +} diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/impl/prefetch/SingleFilePerBlockCache.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/impl/prefetch/SingleFilePerBlockCache.java new file mode 100644 index 0000000000000..c84335a763e87 --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/impl/prefetch/SingleFilePerBlockCache.java @@ -0,0 +1,354 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hadoop.fs.impl.prefetch; + +import java.io.File; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.channels.FileChannel; +import java.nio.channels.WritableByteChannel; +import java.nio.file.Files; +import java.nio.file.OpenOption; +import java.nio.file.Path; +import java.nio.file.StandardOpenOption; +import java.nio.file.attribute.FileAttribute; +import java.nio.file.attribute.PosixFilePermission; +import java.nio.file.attribute.PosixFilePermissions; +import java.util.ArrayList; +import java.util.Collections; +import java.util.EnumSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import static java.util.Objects.requireNonNull; +import static org.apache.hadoop.fs.impl.prefetch.Validate.checkNotNull; + +/** + * Provides functionality necessary for caching blocks of data read from FileSystem. + * Each cache block is stored on the local disk as a separate file. + */ +public class SingleFilePerBlockCache implements BlockCache { + private static final Logger LOG = LoggerFactory.getLogger(SingleFilePerBlockCache.class); + + /** + * Blocks stored in this cache. + */ + private final Map blocks = new ConcurrentHashMap<>(); + + /** + * Number of times a block was read from this cache. + * Used for determining cache utilization factor. + */ + private int numGets = 0; + + private boolean closed; + + private final PrefetchingStatistics prefetchingStatistics; + + /** + * Cache entry. + * Each block is stored as a separate file. + */ + private static final class Entry { + private final int blockNumber; + private final Path path; + private final int size; + private final long checksum; + + Entry(int blockNumber, Path path, int size, long checksum) { + this.blockNumber = blockNumber; + this.path = path; + this.size = size; + this.checksum = checksum; + } + + @Override + public String toString() { + return String.format( + "([%03d] %s: size = %d, checksum = %d)", + blockNumber, path, size, checksum); + } + } + + /** + * Constructs an instance of a {@code SingleFilePerBlockCache}. + * + * @param prefetchingStatistics statistics for this stream. + */ + public SingleFilePerBlockCache(PrefetchingStatistics prefetchingStatistics) { + this.prefetchingStatistics = requireNonNull(prefetchingStatistics); + } + + /** + * Indicates whether the given block is in this cache. + */ + @Override + public boolean containsBlock(int blockNumber) { + return blocks.containsKey(blockNumber); + } + + /** + * Gets the blocks in this cache. + */ + @Override + public Iterable blocks() { + return Collections.unmodifiableList(new ArrayList<>(blocks.keySet())); + } + + /** + * Gets the number of blocks in this cache. + */ + @Override + public int size() { + return blocks.size(); + } + + /** + * Gets the block having the given {@code blockNumber}. + * + * @throws IllegalArgumentException if buffer is null. + */ + @Override + public void get(int blockNumber, ByteBuffer buffer) throws IOException { + if (closed) { + return; + } + + checkNotNull(buffer, "buffer"); + + Entry entry = getEntry(blockNumber); + buffer.clear(); + readFile(entry.path, buffer); + buffer.rewind(); + + validateEntry(entry, buffer); + } + + protected int readFile(Path path, ByteBuffer buffer) throws IOException { + int numBytesRead = 0; + int numBytes; + FileChannel channel = FileChannel.open(path, StandardOpenOption.READ); + while ((numBytes = channel.read(buffer)) > 0) { + numBytesRead += numBytes; + } + buffer.limit(buffer.position()); + channel.close(); + return numBytesRead; + } + + private Entry getEntry(int blockNumber) { + Validate.checkNotNegative(blockNumber, "blockNumber"); + + Entry entry = blocks.get(blockNumber); + if (entry == null) { + throw new IllegalStateException(String.format("block %d not found in cache", blockNumber)); + } + numGets++; + return entry; + } + + /** + * Puts the given block in this cache. + * + * @throws IllegalArgumentException if buffer is null. + * @throws IllegalArgumentException if buffer.limit() is zero or negative. + */ + @Override + public void put(int blockNumber, ByteBuffer buffer) throws IOException { + if (closed) { + return; + } + + checkNotNull(buffer, "buffer"); + + if (blocks.containsKey(blockNumber)) { + Entry entry = blocks.get(blockNumber); + validateEntry(entry, buffer); + return; + } + + Validate.checkPositiveInteger(buffer.limit(), "buffer.limit()"); + + Path blockFilePath = getCacheFilePath(); + long size = Files.size(blockFilePath); + if (size != 0) { + String message = + String.format("[%d] temp file already has data. %s (%d)", + blockNumber, blockFilePath, size); + throw new IllegalStateException(message); + } + + writeFile(blockFilePath, buffer); + prefetchingStatistics.blockAddedToFileCache(); + long checksum = BufferData.getChecksum(buffer); + Entry entry = new Entry(blockNumber, blockFilePath, buffer.limit(), checksum); + blocks.put(blockNumber, entry); + } + + private static final Set CREATE_OPTIONS = + EnumSet.of(StandardOpenOption.WRITE, + StandardOpenOption.CREATE, + StandardOpenOption.TRUNCATE_EXISTING); + + protected void writeFile(Path path, ByteBuffer buffer) throws IOException { + buffer.rewind(); + WritableByteChannel writeChannel = Files.newByteChannel(path, CREATE_OPTIONS); + while (buffer.hasRemaining()) { + writeChannel.write(buffer); + } + writeChannel.close(); + } + + protected Path getCacheFilePath() throws IOException { + return getTempFilePath(); + } + + @Override + public void close() throws IOException { + if (closed) { + return; + } + + closed = true; + + LOG.info(getStats()); + int numFilesDeleted = 0; + + for (Entry entry : blocks.values()) { + try { + Files.deleteIfExists(entry.path); + prefetchingStatistics.blockRemovedFromFileCache(); + numFilesDeleted++; + } catch (IOException e) { + // Ignore while closing so that we can delete as many cache files as possible. + } + } + + if (numFilesDeleted > 0) { + LOG.info("Deleted {} cache files", numFilesDeleted); + } + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder(); + sb.append("stats: "); + sb.append(getStats()); + sb.append(", blocks:["); + sb.append(getIntList(blocks())); + sb.append("]"); + return sb.toString(); + } + + private void validateEntry(Entry entry, ByteBuffer buffer) { + if (entry.size != buffer.limit()) { + String message = String.format( + "[%d] entry.size(%d) != buffer.limit(%d)", + entry.blockNumber, entry.size, buffer.limit()); + throw new IllegalStateException(message); + } + + long checksum = BufferData.getChecksum(buffer); + if (entry.checksum != checksum) { + String message = String.format( + "[%d] entry.checksum(%d) != buffer checksum(%d)", + entry.blockNumber, entry.checksum, checksum); + throw new IllegalStateException(message); + } + } + + /** + * Produces a human readable list of blocks for the purpose of logging. + * This method minimizes the length of returned list by converting + * a contiguous list of blocks into a range. + * for example, + * 1, 3, 4, 5, 6, 8 becomes 1, 3~6, 8 + */ + private String getIntList(Iterable nums) { + List numList = new ArrayList<>(); + List numbers = new ArrayList(); + for (Integer n : nums) { + numbers.add(n); + } + Collections.sort(numbers); + + int index = 0; + while (index < numbers.size()) { + int start = numbers.get(index); + int prev = start; + int end = start; + while ((++index < numbers.size()) && ((end = numbers.get(index)) == prev + 1)) { + prev = end; + } + + if (start == prev) { + numList.add(Integer.toString(start)); + } else { + numList.add(String.format("%d~%d", start, prev)); + } + } + + return String.join(", ", numList); + } + + private String getStats() { + StringBuilder sb = new StringBuilder(); + sb.append(String.format( + "#entries = %d, #gets = %d", + blocks.size(), numGets)); + return sb.toString(); + } + + private static final String CACHE_FILE_PREFIX = "fs-cache-"; + + public static boolean isCacheSpaceAvailable(long fileSize) { + try { + Path cacheFilePath = getTempFilePath(); + long freeSpace = new File(cacheFilePath.toString()).getUsableSpace(); + LOG.info("fileSize = {}, freeSpace = {}", fileSize, freeSpace); + Files.deleteIfExists(cacheFilePath); + return fileSize < freeSpace; + } catch (IOException e) { + LOG.error("isCacheSpaceAvailable", e); + return false; + } + } + + // The suffix (file extension) of each serialized index file. + private static final String BINARY_FILE_SUFFIX = ".bin"; + + // File attributes attached to any intermediate temporary file created during index creation. + private static final FileAttribute> TEMP_FILE_ATTRS = + PosixFilePermissions.asFileAttribute(EnumSet.of(PosixFilePermission.OWNER_READ, + PosixFilePermission.OWNER_WRITE)); + + private static Path getTempFilePath() throws IOException { + return Files.createTempFile( + CACHE_FILE_PREFIX, + BINARY_FILE_SUFFIX, + TEMP_FILE_ATTRS + ); + } +} diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/impl/prefetch/Validate.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/impl/prefetch/Validate.java new file mode 100644 index 0000000000000..17a668a0d3bc3 --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/impl/prefetch/Validate.java @@ -0,0 +1,399 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hadoop.fs.impl.prefetch; + +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Collection; + +import static org.apache.hadoop.util.Preconditions.checkArgument; + +/** + * A superset of Validate class in Apache commons lang3. + *

    + * It provides consistent message strings for frequently encountered checks. + * That simplifies callers because they have to supply only the name of the argument + * that failed a check instead of having to supply the entire message. + */ +public final class Validate { + + private Validate() { + } + + /** + * Validates that the given reference argument is not null. + * @param obj the argument reference to validate. + * @param argName the name of the argument being validated. + */ + public static void checkNotNull(Object obj, String argName) { + checkArgument(obj != null, "'%s' must not be null.", argName); + } + + /** + * Validates that the given integer argument is not zero or negative. + * @param value the argument value to validate + * @param argName the name of the argument being validated. + */ + public static void checkPositiveInteger(long value, String argName) { + checkArgument(value > 0, "'%s' must be a positive integer.", argName); + } + + /** + * Validates that the given integer argument is not negative. + * @param value the argument value to validate + * @param argName the name of the argument being validated. + */ + public static void checkNotNegative(long value, String argName) { + checkArgument(value >= 0, "'%s' must not be negative.", argName); + } + + /** + * Validates that the expression (that checks a required field is present) is true. + * @param isPresent indicates whether the given argument is present. + * @param argName the name of the argument being validated. + */ + public static void checkRequired(boolean isPresent, String argName) { + checkArgument(isPresent, "'%s' is required.", argName); + } + + /** + * Validates that the expression (that checks a field is valid) is true. + * @param isValid indicates whether the given argument is valid. + * @param argName the name of the argument being validated. + */ + public static void checkValid(boolean isValid, String argName) { + checkArgument(isValid, "'%s' is invalid.", argName); + } + + /** + * Validates that the expression (that checks a field is valid) is true. + * @param isValid indicates whether the given argument is valid. + * @param argName the name of the argument being validated. + * @param validValues the list of values that are allowed. + */ + public static void checkValid(boolean isValid, + String argName, + String validValues) { + checkArgument(isValid, "'%s' is invalid. Valid values are: %s.", argName, + validValues); + } + + /** + * Validates that the given string is not null and has non-zero length. + * @param arg the argument reference to validate. + * @param argName the name of the argument being validated. + */ + public static void checkNotNullAndNotEmpty(String arg, String argName) { + checkNotNull(arg, argName); + checkArgument( + !arg.isEmpty(), + "'%s' must not be empty.", + argName); + } + + /** + * Validates that the given array is not null and has at least one element. + * @param the type of array's elements. + * @param array the argument reference to validate. + * @param argName the name of the argument being validated. + */ + public static void checkNotNullAndNotEmpty(T[] array, String argName) { + checkNotNull(array, argName); + checkNotEmpty(array.length, argName); + } + + /** + * Validates that the given array is not null and has at least one element. + * @param array the argument reference to validate. + * @param argName the name of the argument being validated. + */ + public static void checkNotNullAndNotEmpty(byte[] array, String argName) { + checkNotNull(array, argName); + checkNotEmpty(array.length, argName); + } + + /** + * Validates that the given array is not null and has at least one element. + * @param array the argument reference to validate. + * @param argName the name of the argument being validated. + */ + public static void checkNotNullAndNotEmpty(short[] array, String argName) { + checkNotNull(array, argName); + checkNotEmpty(array.length, argName); + } + + /** + * Validates that the given array is not null and has at least one element. + * @param array the argument reference to validate. + * @param argName the name of the argument being validated. + */ + public static void checkNotNullAndNotEmpty(int[] array, String argName) { + checkNotNull(array, argName); + checkNotEmpty(array.length, argName); + } + + /** + * Validates that the given array is not null and has at least one element. + * @param array the argument reference to validate. + * @param argName the name of the argument being validated. + */ + public static void checkNotNullAndNotEmpty(long[] array, String argName) { + checkNotNull(array, argName); + checkNotEmpty(array.length, argName); + } + + /** + * Validates that the given buffer is not null and has non-zero capacity. + * @param the type of iterable's elements. + * @param iter the argument reference to validate. + * @param argName the name of the argument being validated. + */ + public static void checkNotNullAndNotEmpty(Iterable iter, + String argName) { + checkNotNull(iter, argName); + int minNumElements = iter.iterator().hasNext() ? 1 : 0; + checkNotEmpty(minNumElements, argName); + } + + /** + * Validates that the given set is not null and has an exact number of items. + * @param the type of collection's elements. + * @param collection the argument reference to validate. + * @param numElements the expected number of elements in the collection. + * @param argName the name of the argument being validated. + */ + public static void checkNotNullAndNumberOfElements( + Collection collection, int numElements, String argName) { + checkNotNull(collection, argName); + checkArgument( + collection.size() == numElements, + "Number of elements in '%s' must be exactly %s, %s given.", + argName, + numElements, + collection.size() + ); + } + + /** + * Validates that the given two values are equal. + * @param value1 the first value to check. + * @param value1Name the name of the first argument. + * @param value2 the second value to check. + * @param value2Name the name of the second argument. + */ + public static void checkValuesEqual( + long value1, + String value1Name, + long value2, + String value2Name) { + checkArgument( + value1 == value2, + "'%s' (%s) must equal '%s' (%s).", + value1Name, + value1, + value2Name, + value2); + } + + /** + * Validates that the first value is an integer multiple of the second value. + * @param value1 the first value to check. + * @param value1Name the name of the first argument. + * @param value2 the second value to check. + * @param value2Name the name of the second argument. + */ + public static void checkIntegerMultiple( + long value1, + String value1Name, + long value2, + String value2Name) { + checkArgument( + (value1 % value2) == 0, + "'%s' (%s) must be an integer multiple of '%s' (%s).", + value1Name, + value1, + value2Name, + value2); + } + + /** + * Validates that the first value is greater than the second value. + * @param value1 the first value to check. + * @param value1Name the name of the first argument. + * @param value2 the second value to check. + * @param value2Name the name of the second argument. + */ + public static void checkGreater( + long value1, + String value1Name, + long value2, + String value2Name) { + checkArgument( + value1 > value2, + "'%s' (%s) must be greater than '%s' (%s).", + value1Name, + value1, + value2Name, + value2); + } + + /** + * Validates that the first value is greater than or equal to the second value. + * @param value1 the first value to check. + * @param value1Name the name of the first argument. + * @param value2 the second value to check. + * @param value2Name the name of the second argument. + */ + public static void checkGreaterOrEqual( + long value1, + String value1Name, + long value2, + String value2Name) { + checkArgument( + value1 >= value2, + "'%s' (%s) must be greater than or equal to '%s' (%s).", + value1Name, + value1, + value2Name, + value2); + } + + /** + * Validates that the first value is less than or equal to the second value. + * @param value1 the first value to check. + * @param value1Name the name of the first argument. + * @param value2 the second value to check. + * @param value2Name the name of the second argument. + */ + public static void checkLessOrEqual( + long value1, + String value1Name, + long value2, + String value2Name) { + checkArgument( + value1 <= value2, + "'%s' (%s) must be less than or equal to '%s' (%s).", + value1Name, + value1, + value2Name, + value2); + } + + /** + * Validates that the given value is within the given range of values. + * @param value the value to check. + * @param valueName the name of the argument. + * @param minValueInclusive inclusive lower limit for the value. + * @param maxValueInclusive inclusive upper limit for the value. + */ + public static void checkWithinRange( + long value, + String valueName, + long minValueInclusive, + long maxValueInclusive) { + checkArgument( + (value >= minValueInclusive) && (value <= maxValueInclusive), + "'%s' (%s) must be within the range [%s, %s].", + valueName, + value, + minValueInclusive, + maxValueInclusive); + } + + /** + * Validates that the given value is within the given range of values. + * @param value the value to check. + * @param valueName the name of the argument. + * @param minValueInclusive inclusive lower limit for the value. + * @param maxValueInclusive inclusive upper limit for the value. + */ + public static void checkWithinRange( + double value, + String valueName, + double minValueInclusive, + double maxValueInclusive) { + checkArgument( + (value >= minValueInclusive) && (value <= maxValueInclusive), + "'%s' (%s) must be within the range [%s, %s].", + valueName, + value, + minValueInclusive, + maxValueInclusive); + } + + /** + * Validates that the given path exists. + * @param path the path to check. + * @param argName the name of the argument being validated. + */ + public static void checkPathExists(Path path, String argName) { + checkNotNull(path, argName); + checkArgument(Files.exists(path), "Path %s (%s) does not exist.", argName, + path); + } + + /** + * Validates that the given path exists and is a directory. + * @param path the path to check. + * @param argName the name of the argument being validated. + */ + public static void checkPathExistsAsDir(Path path, String argName) { + checkPathExists(path, argName); + checkArgument( + Files.isDirectory(path), + "Path %s (%s) must point to a directory.", + argName, + path); + } + + /** + * Validates that the given path exists and is a file. + * @param path the path to check. + * @param argName the name of the argument being validated. + */ + public static void checkPathExistsAsFile(Path path, String argName) { + checkPathExists(path, argName); + checkArgument(Files.isRegularFile(path), + "Path %s (%s) must point to a file.", argName, path); + } + + + /** + * Check state. + * @param expression expression which must hold. + * @param format format string + * @param args arguments for the error string + * @throws IllegalStateException if the state is not valid. + */ + public static void checkState(boolean expression, + String format, + Object... args) { + if (!expression) { + throw new IllegalStateException(String.format(format, args)); + } + } + + private static void checkNotEmpty(int arraySize, String argName) { + checkArgument( + arraySize > 0, + "'%s' must have at least one element.", + argName); + } +} diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/impl/prefetch/package-info.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/impl/prefetch/package-info.java new file mode 100644 index 0000000000000..1b26da85d95fb --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/impl/prefetch/package-info.java @@ -0,0 +1,28 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * block caching for use in object store clients. + */ + +@InterfaceAudience.Private +@InterfaceStability.Unstable +package org.apache.hadoop.fs.impl.prefetch; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; \ No newline at end of file diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/statistics/IOStatisticsContext.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/statistics/IOStatisticsContext.java new file mode 100644 index 0000000000000..557c57ea4d661 --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/statistics/IOStatisticsContext.java @@ -0,0 +1,93 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.statistics; + +import org.apache.hadoop.fs.statistics.impl.IOStatisticsContextIntegration; + +/** + * An interface defined to capture thread-level IOStatistics by using per + * thread context. + *

    + * The aggregator should be collected in their constructor by statistics-generating + * classes to obtain the aggregator to update across all threads. + *

    + * The {@link #snapshot()} call creates a snapshot of the statistics; + *

    + * The {@link #reset()} call resets the statistics in the context so + * that later snapshots will get the incremental data. + */ +public interface IOStatisticsContext extends IOStatisticsSource { + + /** + * Get the IOStatisticsAggregator for the context. + * + * @return return the aggregator for the context. + */ + IOStatisticsAggregator getAggregator(); + + /** + * Capture the snapshot of the context's IOStatistics. + * + * @return IOStatisticsSnapshot for the context. + */ + IOStatisticsSnapshot snapshot(); + + /** + * Get a unique ID for this context, for logging + * purposes. + * + * @return an ID unique for all contexts in this process. + */ + long getID(); + + /** + * Reset the context's IOStatistics. + */ + void reset(); + + /** + * Get the context's IOStatisticsContext. + * + * @return instance of IOStatisticsContext for the context. + */ + static IOStatisticsContext getCurrentIOStatisticsContext() { + return IOStatisticsContextIntegration.getCurrentIOStatisticsContext(); + } + + /** + * Set the IOStatisticsContext for the current thread. + * @param statisticsContext IOStatistics context instance for the + * current thread. If null, the context is reset. + */ + static void setThreadIOStatisticsContext( + IOStatisticsContext statisticsContext) { + IOStatisticsContextIntegration.setThreadIOStatisticsContext( + statisticsContext); + } + + /** + * Static probe to check if the thread-level IO statistics enabled. + * + * @return if the thread-level IO statistics enabled. + */ + static boolean enabled() { + return IOStatisticsContextIntegration.isIOStatisticsThreadLevelEnabled(); + } + +} diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/statistics/StreamStatisticNames.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/statistics/StreamStatisticNames.java index ca755f0841914..50bbf45505cec 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/statistics/StreamStatisticNames.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/statistics/StreamStatisticNames.java @@ -47,7 +47,7 @@ public final class StreamStatisticNames { public static final String STREAM_READ_ABORTED = "stream_aborted"; /** - * Bytes read from an input stream in read() calls. + * Bytes read from an input stream in read()/readVectored() calls. * Does not include bytes read and then discarded in seek/close etc. * These are the bytes returned to the caller. * Value: {@value}. @@ -110,6 +110,34 @@ public final class StreamStatisticNames { public static final String STREAM_READ_OPERATIONS = "stream_read_operations"; + /** + * Count of readVectored() operations in an input stream. + * Value: {@value}. + */ + public static final String STREAM_READ_VECTORED_OPERATIONS = + "stream_read_vectored_operations"; + + /** + * Count of bytes discarded during readVectored() operation + * in an input stream. + * Value: {@value}. + */ + public static final String STREAM_READ_VECTORED_READ_BYTES_DISCARDED = + "stream_read_vectored_read_bytes_discarded"; + + /** + * Count of incoming file ranges during readVectored() operation. + * Value: {@value} + */ + public static final String STREAM_READ_VECTORED_INCOMING_RANGES = + "stream_read_vectored_incoming_ranges"; + /** + * Count of combined file ranges during readVectored() operation. + * Value: {@value} + */ + public static final String STREAM_READ_VECTORED_COMBINED_RANGES = + "stream_read_vectored_combined_ranges"; + /** * Count of incomplete read() operations in an input stream, * that is, when the bytes returned were less than that requested. @@ -387,6 +415,46 @@ public final class StreamStatisticNames { public static final String BLOCKS_RELEASED = "blocks_released"; + /** + * Total number of prefetching operations executed. + */ + public static final String STREAM_READ_PREFETCH_OPERATIONS + = "stream_read_prefetch_operations"; + + /** + * Total number of block in disk cache. + */ + public static final String STREAM_READ_BLOCKS_IN_FILE_CACHE + = "stream_read_blocks_in_cache"; + + /** + * Total number of active prefetch operations. + */ + public static final String STREAM_READ_ACTIVE_PREFETCH_OPERATIONS + = "stream_read_active_prefetch_operations"; + + /** + * Total bytes of memory in use by this input stream. + */ + public static final String STREAM_READ_ACTIVE_MEMORY_IN_USE + = "stream_read_active_memory_in_use"; + + /** + * count/duration of reading a remote block. + * + * Value: {@value}. + */ + public static final String STREAM_READ_REMOTE_BLOCK_READ + = "stream_read_block_read"; + + /** + * count/duration of acquiring a buffer and reading to it. + * + * Value: {@value}. + */ + public static final String STREAM_READ_BLOCK_ACQUIRE_AND_READ + = "stream_read_block_acquire_read"; + private StreamStatisticNames() { } diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/statistics/impl/EmptyIOStatisticsContextImpl.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/statistics/impl/EmptyIOStatisticsContextImpl.java new file mode 100644 index 0000000000000..b672f6639cb93 --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/statistics/impl/EmptyIOStatisticsContextImpl.java @@ -0,0 +1,81 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.statistics.impl; + +import org.apache.hadoop.fs.statistics.IOStatistics; +import org.apache.hadoop.fs.statistics.IOStatisticsAggregator; +import org.apache.hadoop.fs.statistics.IOStatisticsContext; +import org.apache.hadoop.fs.statistics.IOStatisticsSnapshot; + +/** + * Empty IOStatistics context which serves no-op for all the operations and + * returns an empty Snapshot if asked. + * + */ +final class EmptyIOStatisticsContextImpl implements IOStatisticsContext { + + private static final IOStatisticsContext EMPTY_CONTEXT = new EmptyIOStatisticsContextImpl(); + + private EmptyIOStatisticsContextImpl() { + } + + /** + * Create a new empty snapshot. + * A new one is always created for isolation. + * + * @return a statistics snapshot + */ + @Override + public IOStatisticsSnapshot snapshot() { + return new IOStatisticsSnapshot(); + } + + @Override + public IOStatisticsAggregator getAggregator() { + return EmptyIOStatisticsStore.getInstance(); + } + + @Override + public IOStatistics getIOStatistics() { + return EmptyIOStatistics.getInstance(); + } + + @Override + public void reset() {} + + /** + * The ID is always 0. + * As the real context implementation counter starts at 1, + * we are guaranteed to have unique IDs even between them and + * the empty context. + * @return 0 + */ + @Override + public long getID() { + return 0; + } + + /** + * Get the single instance. + * @return an instance. + */ + static IOStatisticsContext getInstance() { + return EMPTY_CONTEXT; + } +} diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/statistics/impl/IOStatisticsContextImpl.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/statistics/impl/IOStatisticsContextImpl.java new file mode 100644 index 0000000000000..97a85281c4fb8 --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/statistics/impl/IOStatisticsContextImpl.java @@ -0,0 +1,128 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.statistics.impl; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import org.apache.hadoop.fs.statistics.IOStatistics; +import org.apache.hadoop.fs.statistics.IOStatisticsAggregator; +import org.apache.hadoop.fs.statistics.IOStatisticsContext; +import org.apache.hadoop.fs.statistics.IOStatisticsSnapshot; + +/** + * Implementing the IOStatisticsContext. + * + * A Context defined for IOStatistics collection per thread which captures + * each worker thread's work in FS streams and stores it in the form of + * IOStatisticsSnapshot. + * + * For the current thread the IOStatisticsSnapshot can be used as a way to + * move the IOStatistics data between applications using the Serializable + * nature of the class. + */ +public final class IOStatisticsContextImpl implements IOStatisticsContext { + private static final Logger LOG = + LoggerFactory.getLogger(IOStatisticsContextImpl.class); + + /** + * Thread ID. + */ + private final long threadId; + + /** + * Unique ID. + */ + private final long id; + + /** + * IOStatistics to aggregate. + */ + private final IOStatisticsSnapshot ioStatistics = new IOStatisticsSnapshot(); + + /** + * Constructor. + * @param threadId thread ID + * @param id instance ID. + */ + public IOStatisticsContextImpl(final long threadId, final long id) { + this.threadId = threadId; + this.id = id; + } + + @Override + public String toString() { + return "IOStatisticsContextImpl{" + + "id=" + id + + ", threadId=" + threadId + + ", ioStatistics=" + ioStatistics + + '}'; + } + + /** + * Get the IOStatisticsAggregator of the context. + * @return the instance of IOStatisticsAggregator for this context. + */ + @Override + public IOStatisticsAggregator getAggregator() { + return ioStatistics; + } + + /** + * Returns a snapshot of the current thread's IOStatistics. + * + * @return IOStatisticsSnapshot of the context. + */ + @Override + public IOStatisticsSnapshot snapshot() { + LOG.debug("Taking snapshot of IOStatisticsContext id {}", id); + return new IOStatisticsSnapshot(ioStatistics); + } + + /** + * Reset the thread +. + */ + @Override + public void reset() { + LOG.debug("clearing IOStatisticsContext id {}", id); + ioStatistics.clear(); + } + + @Override + public IOStatistics getIOStatistics() { + return ioStatistics; + } + + /** + * ID of this context. + * @return ID. + */ + @Override + public long getID() { + return id; + } + + /** + * Get the thread ID. + * @return thread ID. + */ + public long getThreadID() { + return threadId; + } +} diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/statistics/impl/IOStatisticsContextIntegration.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/statistics/impl/IOStatisticsContextIntegration.java new file mode 100644 index 0000000000000..2a394e6a1cdf1 --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/statistics/impl/IOStatisticsContextIntegration.java @@ -0,0 +1,176 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.statistics.impl; + +import java.lang.ref.WeakReference; +import java.util.concurrent.atomic.AtomicLong; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import org.apache.hadoop.classification.VisibleForTesting; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.impl.WeakReferenceThreadMap; +import org.apache.hadoop.fs.statistics.IOStatisticsContext; + +import static org.apache.hadoop.fs.CommonConfigurationKeys.IOSTATISTICS_THREAD_LEVEL_ENABLED; +import static org.apache.hadoop.fs.CommonConfigurationKeys.IOSTATISTICS_THREAD_LEVEL_ENABLED_DEFAULT; + +/** + * A Utility class for IOStatisticsContext, which helps in creating and + * getting the current active context. Static methods in this class allows to + * get the current context to start aggregating the IOStatistics. + * + * Static initializer is used to work out if the feature to collect + * thread-level IOStatistics is enabled or not and the corresponding + * implementation class is called for it. + * + * Weak Reference thread map to be used to keep track of different context's + * to avoid long-lived memory leakages as these references would be cleaned + * up at GC. + */ +public final class IOStatisticsContextIntegration { + + private static final Logger LOG = + LoggerFactory.getLogger(IOStatisticsContextIntegration.class); + + /** + * Is thread-level IO Statistics enabled? + */ + private static boolean isThreadIOStatsEnabled; + + /** + * ID for next instance to create. + */ + public static final AtomicLong INSTANCE_ID = new AtomicLong(1); + + /** + * Active IOStatistics Context containing different worker thread's + * statistics. Weak Reference so that it gets cleaned up during GC and we + * avoid any memory leak issues due to long lived references. + */ + private static final WeakReferenceThreadMap + ACTIVE_IOSTATS_CONTEXT = + new WeakReferenceThreadMap<>( + IOStatisticsContextIntegration::createNewInstance, + IOStatisticsContextIntegration::referenceLostContext + ); + + static { + // Work out if the current context has thread level IOStatistics enabled. + final Configuration configuration = new Configuration(); + isThreadIOStatsEnabled = + configuration.getBoolean(IOSTATISTICS_THREAD_LEVEL_ENABLED, + IOSTATISTICS_THREAD_LEVEL_ENABLED_DEFAULT); + } + + /** + * Static probe to check if the thread-level IO statistics enabled. + * + * @return if the thread-level IO statistics enabled. + */ + public static boolean isIOStatisticsThreadLevelEnabled() { + return isThreadIOStatsEnabled; + } + + /** + * Private constructor for a utility class to be used in IOStatisticsContext. + */ + private IOStatisticsContextIntegration() {} + + /** + * Creating a new IOStatisticsContext instance for a FS to be used. + * @param key Thread ID that represents which thread the context belongs to. + * @return an instance of IOStatisticsContext. + */ + private static IOStatisticsContext createNewInstance(Long key) { + return new IOStatisticsContextImpl(key, INSTANCE_ID.getAndIncrement()); + } + + /** + * In case of reference loss for IOStatisticsContext. + * @param key ThreadID. + */ + private static void referenceLostContext(Long key) { + LOG.debug("Reference lost for threadID for the context: {}", key); + } + + /** + * Get the current thread's IOStatisticsContext instance. If no instance is + * present for this thread ID, create one using the factory. + * @return instance of IOStatisticsContext. + */ + public static IOStatisticsContext getCurrentIOStatisticsContext() { + return isThreadIOStatsEnabled + ? ACTIVE_IOSTATS_CONTEXT.getForCurrentThread() + : EmptyIOStatisticsContextImpl.getInstance(); + } + + /** + * Set the IOStatisticsContext for the current thread. + * @param statisticsContext IOStatistics context instance for the + * current thread. If null, the context is reset. + */ + public static void setThreadIOStatisticsContext( + IOStatisticsContext statisticsContext) { + if (isThreadIOStatsEnabled) { + if (statisticsContext == null) { + ACTIVE_IOSTATS_CONTEXT.removeForCurrentThread(); + } + if (ACTIVE_IOSTATS_CONTEXT.getForCurrentThread() != statisticsContext) { + ACTIVE_IOSTATS_CONTEXT.setForCurrentThread(statisticsContext); + } + } + } + + /** + * Get thread ID specific IOStatistics values if + * statistics are enabled and the thread ID is in the map. + * @param testThreadId thread ID. + * @return IOStatisticsContext if found in the map. + */ + @VisibleForTesting + public static IOStatisticsContext getThreadSpecificIOStatisticsContext(long testThreadId) { + LOG.debug("IOStatsContext thread ID required: {}", testThreadId); + + if (!isThreadIOStatsEnabled) { + return null; + } + // lookup the weakRef IOStatisticsContext for the thread ID in the + // ThreadMap. + WeakReference ioStatisticsSnapshotWeakReference = + ACTIVE_IOSTATS_CONTEXT.lookup(testThreadId); + if (ioStatisticsSnapshotWeakReference != null) { + return ioStatisticsSnapshotWeakReference.get(); + } + return null; + } + + /** + * A method to enable IOStatisticsContext to override if set otherwise in + * the configurations for tests. + */ + @VisibleForTesting + public static void enableIOStatisticsContext() { + if (!isThreadIOStatsEnabled) { + LOG.info("Enabling Thread IOStatistics.."); + isThreadIOStatsEnabled = true; + } + } +} diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/HttpServer2.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/HttpServer2.java index 2928f88598207..178f761191b1e 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/HttpServer2.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/HttpServer2.java @@ -97,7 +97,7 @@ import org.eclipse.jetty.server.Server; import org.eclipse.jetty.server.ServerConnector; import org.eclipse.jetty.server.SslConnectionFactory; -import org.eclipse.jetty.server.handler.AllowSymLinkAliasChecker; +import org.eclipse.jetty.server.SymlinkAllowedResourceAliasChecker; import org.eclipse.jetty.server.handler.ContextHandlerCollection; import org.eclipse.jetty.server.handler.HandlerCollection; import org.eclipse.jetty.server.handler.RequestLogHandler; @@ -144,7 +144,7 @@ public final class HttpServer2 implements FilterContainer { public static final String HTTP_SOCKET_BACKLOG_SIZE_KEY = "hadoop.http.socket.backlog.size"; - public static final int HTTP_SOCKET_BACKLOG_SIZE_DEFAULT = 128; + public static final int HTTP_SOCKET_BACKLOG_SIZE_DEFAULT = 500; public static final String HTTP_MAX_THREADS_KEY = "hadoop.http.max.threads"; public static final String HTTP_ACCEPTOR_COUNT_KEY = "hadoop.http.acceptor.count"; @@ -939,7 +939,7 @@ protected void addDefaultApps(ContextHandlerCollection parent, handler.setHttpOnly(true); handler.getSessionCookieConfig().setSecure(true); logContext.setSessionHandler(handler); - logContext.addAliasCheck(new AllowSymLinkAliasChecker()); + logContext.addAliasCheck(new SymlinkAllowedResourceAliasChecker(logContext)); setContextAttributes(logContext, conf); addNoCacheFilter(logContext); defaultContexts.put(logContext, true); @@ -958,7 +958,7 @@ protected void addDefaultApps(ContextHandlerCollection parent, handler.setHttpOnly(true); handler.getSessionCookieConfig().setSecure(true); staticContext.setSessionHandler(handler); - staticContext.addAliasCheck(new AllowSymLinkAliasChecker()); + staticContext.addAliasCheck(new SymlinkAllowedResourceAliasChecker(staticContext)); setContextAttributes(staticContext, conf); defaultContexts.put(staticContext, true); } @@ -1967,4 +1967,8 @@ HttpServer2Metrics getMetrics() { return metrics; } + @VisibleForTesting + List getListeners() { + return listeners; + } } diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/AlreadyClosedException.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/AlreadyClosedException.java new file mode 100644 index 0000000000000..993d2678d2a10 --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/AlreadyClosedException.java @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.io.compress; + +import java.io.IOException; + +/** + * An exception class for when a closed compressor/decopressor is being used + * {@link org.apache.hadoop.io.compress.Compressor} + * {@link org.apache.hadoop.io.compress.Decompressor} + */ +public class AlreadyClosedException extends IOException { + + public AlreadyClosedException(String message) { + super(message); + } +} diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/BZip2Codec.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/BZip2Codec.java index 7640f7ed7a6f7..1564ae9085520 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/BZip2Codec.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/BZip2Codec.java @@ -24,6 +24,7 @@ import java.io.OutputStream; import java.nio.charset.StandardCharsets; +import org.apache.hadoop.classification.VisibleForTesting; import org.apache.hadoop.conf.Configurable; import org.apache.hadoop.conf.Configuration; @@ -255,10 +256,7 @@ public BZip2CompressionOutputStream(OutputStream out) private void writeStreamHeader() throws IOException { if (super.out != null) { - // The compressed bzip2 stream should start with the - // identifying characters BZ. Caller of CBZip2OutputStream - // i.e. this class must write these characters. - out.write(HEADER.getBytes(StandardCharsets.UTF_8)); + writeHeader(out); } } @@ -547,4 +545,11 @@ private void updatePos(boolean shouldAddOn) { }// end of BZip2CompressionInputStream + @VisibleForTesting + public static void writeHeader(OutputStream out) throws IOException { + // The compressed bzip2 stream should start with the + // identifying characters BZ. Caller of CBZip2OutputStream + // i.e. this class must write these characters. + out.write(HEADER.getBytes(StandardCharsets.UTF_8)); + } } diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/CodecPool.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/CodecPool.java index 1f095c6c6736e..5b1826f9e30a8 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/CodecPool.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/CodecPool.java @@ -205,6 +205,7 @@ public static void returnCompressor(Compressor compressor) { } // if the compressor can't be reused, don't pool it. if (compressor.getClass().isAnnotationPresent(DoNotPool.class)) { + compressor.end(); return; } compressor.reset(); @@ -225,6 +226,7 @@ public static void returnDecompressor(Decompressor decompressor) { } // if the decompressor can't be reused, don't pool it. if (decompressor.getClass().isAnnotationPresent(DoNotPool.class)) { + decompressor.end(); return; } decompressor.reset(); diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/bzip2/CBZip2InputStream.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/bzip2/CBZip2InputStream.java index 187fe481588c8..61e88d80d8ce4 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/bzip2/CBZip2InputStream.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/bzip2/CBZip2InputStream.java @@ -27,6 +27,7 @@ import java.io.InputStream; import java.io.IOException; +import org.apache.hadoop.classification.VisibleForTesting; import org.apache.hadoop.io.compress.SplittableCompressionCodec.READ_MODE; @@ -312,13 +313,24 @@ private CBZip2InputStream(final InputStream in, READ_MODE readMode, boolean skip } } else if (readMode == READ_MODE.BYBLOCK) { this.currentState = STATE.NO_PROCESS_STATE; - skipResult = this.skipToNextMarker(CBZip2InputStream.BLOCK_DELIMITER,DELIMITER_BIT_LENGTH); + skipResult = skipToNextBlockMarker(); if(!skipDecompression){ changeStateToProcessABlock(); } } } + /** + * Skips bytes in the stream until the start marker of a block is reached + * or end of stream is reached. Used for testing purposes to identify the + * start offsets of blocks. + */ + @VisibleForTesting + boolean skipToNextBlockMarker() throws IOException { + return skipToNextMarker( + CBZip2InputStream.BLOCK_DELIMITER, DELIMITER_BIT_LENGTH); + } + /** * Returns the number of bytes between the current stream position * and the immediate next BZip2 block marker. @@ -428,7 +440,7 @@ public int read(final byte[] dest, final int offs, final int len) //report 'end of block' or 'end of stream' result = b; - skipResult = this.skipToNextMarker(CBZip2InputStream.BLOCK_DELIMITER, DELIMITER_BIT_LENGTH); + skipResult = skipToNextBlockMarker(); changeStateToProcessABlock(); } diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/bzip2/CBZip2OutputStream.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/bzip2/CBZip2OutputStream.java index 39c3638b0f497..50bdddb8136fc 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/bzip2/CBZip2OutputStream.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/bzip2/CBZip2OutputStream.java @@ -27,6 +27,7 @@ import java.io.OutputStream; import java.io.IOException; +import org.apache.hadoop.classification.VisibleForTesting; import org.apache.hadoop.io.IOUtils; /** @@ -781,8 +782,7 @@ private void initBlock() { inUse[i] = false; } - /* 20 is just a paranoia constant */ - this.allowableBlockSize = (this.blockSize100k * BZip2Constants.baseBlockSize) - 20; + this.allowableBlockSize = getAllowableBlockSize(this.blockSize100k); } private void endBlock() throws IOException { @@ -2093,4 +2093,9 @@ private static final class Data extends Object { } + @VisibleForTesting + static int getAllowableBlockSize(int blockSize100k) { + /* 20 is just a paranoia constant */ + return (blockSize100k * BZip2Constants.baseBlockSize) - 20; + } } diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/zlib/BuiltInGzipCompressor.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/zlib/BuiltInGzipCompressor.java index fcb431dce86ca..d44413cc30912 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/zlib/BuiltInGzipCompressor.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/zlib/BuiltInGzipCompressor.java @@ -24,6 +24,7 @@ import java.util.zip.GZIPOutputStream; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.io.compress.AlreadyClosedException; import org.apache.hadoop.io.compress.Compressor; import org.apache.hadoop.io.compress.DoNotPool; import org.apache.hadoop.util.DataChecksum; @@ -83,6 +84,10 @@ public int compress(byte[] b, int off, int len) throws IOException { throw new IOException("compress called on finished compressor"); } + if (state == BuiltInGzipDecompressor.GzipStateLabel.ENDED) { + throw new AlreadyClosedException("compress called on closed compressor"); + } + int compressedBytesWritten = 0; // If we are not within uncompressed data yet, output the header. @@ -139,6 +144,8 @@ public long getBytesWritten() { @Override public void end() { deflater.end(); + + state = BuiltInGzipDecompressor.GzipStateLabel.ENDED; } @Override diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/zlib/BuiltInGzipDecompressor.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/zlib/BuiltInGzipDecompressor.java index 47c21b4e3ea98..d47864a71f481 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/zlib/BuiltInGzipDecompressor.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/zlib/BuiltInGzipDecompressor.java @@ -23,6 +23,7 @@ import java.util.zip.DataFormatException; import java.util.zip.Inflater; +import org.apache.hadoop.io.compress.AlreadyClosedException; import org.apache.hadoop.io.compress.Decompressor; import org.apache.hadoop.io.compress.DoNotPool; import org.apache.hadoop.util.DataChecksum; @@ -109,7 +110,11 @@ public enum GzipStateLabel { * Immediately after the trailer (and potentially prior to the next gzip * member/substream header), without reset() having been called. */ - FINISHED; + FINISHED, + /** + * Immediately after end() has been called. + */ + ENDED; } /** @@ -186,6 +191,10 @@ public synchronized int decompress(byte[] b, int off, int len) throws IOException { int numAvailBytes = 0; + if (state == GzipStateLabel.ENDED) { + throw new AlreadyClosedException("decompress called on closed decompressor"); + } + if (state != GzipStateLabel.DEFLATE_STREAM) { executeHeaderState(); @@ -476,6 +485,8 @@ public synchronized void reset() { @Override public synchronized void end() { inflater.end(); + + state = GzipStateLabel.ENDED; } /** diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/retry/RetryInvocationHandler.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/retry/RetryInvocationHandler.java index 3960b189665f6..9707ee388e1d2 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/retry/RetryInvocationHandler.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/retry/RetryInvocationHandler.java @@ -46,6 +46,10 @@ public class RetryInvocationHandler implements RpcInvocationHandler { public static final Logger LOG = LoggerFactory.getLogger( RetryInvocationHandler.class); + @VisibleForTesting + public static final ThreadLocal SET_CALL_ID_FOR_TEST = + ThreadLocal.withInitial(() -> true); + static class Call { private final Method method; private final Object[] args; @@ -159,7 +163,7 @@ CallReturn invoke() throws Throwable { } Object invokeMethod() throws Throwable { - if (isRpc) { + if (isRpc && SET_CALL_ID_FOR_TEST.get()) { Client.setCallIdAndRetryCount(callId, counters.retries, retryInvocationHandler.asyncCallHandler); } diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/retry/RetryPolicies.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/retry/RetryPolicies.java index d7693f868eb30..e14459123016b 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/retry/RetryPolicies.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/retry/RetryPolicies.java @@ -181,15 +181,20 @@ public static final RetryPolicy retryByRemoteException( } /** - * A retry policy for exceptions other than RemoteException. + *

    + * A retry policy where RemoteException and SaslException are not retried, other individual + * exception types can have RetryPolicy overrides, and any other exception type without an + * override is not retried. + *

    + * * @param defaultPolicy defaultPolicy. * @param exceptionToPolicyMap exceptionToPolicyMap. * @return RetryPolicy. */ - public static final RetryPolicy retryOtherThanRemoteException( + public static final RetryPolicy retryOtherThanRemoteAndSaslException( RetryPolicy defaultPolicy, Map, RetryPolicy> exceptionToPolicyMap) { - return new OtherThanRemoteExceptionDependentRetry(defaultPolicy, + return new OtherThanRemoteAndSaslExceptionDependentRetry(defaultPolicy, exceptionToPolicyMap); } @@ -589,12 +594,12 @@ public RetryAction shouldRetry(Exception e, int retries, int failovers, } } - static class OtherThanRemoteExceptionDependentRetry implements RetryPolicy { + static class OtherThanRemoteAndSaslExceptionDependentRetry implements RetryPolicy { private RetryPolicy defaultPolicy; private Map, RetryPolicy> exceptionToPolicyMap; - public OtherThanRemoteExceptionDependentRetry(RetryPolicy defaultPolicy, + OtherThanRemoteAndSaslExceptionDependentRetry(RetryPolicy defaultPolicy, Map, RetryPolicy> exceptionToPolicyMap) { this.defaultPolicy = defaultPolicy; @@ -605,10 +610,8 @@ public OtherThanRemoteExceptionDependentRetry(RetryPolicy defaultPolicy, public RetryAction shouldRetry(Exception e, int retries, int failovers, boolean isIdempotentOrAtMostOnce) throws Exception { RetryPolicy policy = null; - // ignore Remote Exception - if (e instanceof RemoteException) { - // do nothing - } else { + // ignore RemoteException and SaslException + if (!(e instanceof RemoteException || isSaslFailure(e))) { policy = exceptionToPolicyMap.get(e.getClass()); } if (policy == null) { diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/AlignmentContext.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/AlignmentContext.java index 3d309235fe891..8d43fd74a843c 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/AlignmentContext.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/AlignmentContext.java @@ -46,7 +46,7 @@ public interface AlignmentContext { void updateResponseState(RpcResponseHeaderProto.Builder header); /** - * This is the intended client method call to implement to recieve state info + * This is the intended client method call to implement to receive state info * during RPC response processing. * * @param header The RPC response header. diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/CallerContext.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/CallerContext.java index dbd9184a2b91e..98d7e82c70e0a 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/CallerContext.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/CallerContext.java @@ -47,6 +47,8 @@ public final class CallerContext { // field names public static final String CLIENT_IP_STR = "clientIp"; public static final String CLIENT_PORT_STR = "clientPort"; + public static final String CLIENT_ID_STR = "clientId"; + public static final String CLIENT_CALL_ID_STR = "clientCallId"; /** The caller context. * diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Client.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Client.java index 2fe8aca85ed9a..20fc9efe57e0a 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Client.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Client.java @@ -61,6 +61,7 @@ import javax.net.SocketFactory; import javax.security.sasl.Sasl; +import javax.security.sasl.SaslException; import java.io.*; import java.net.*; import java.nio.ByteBuffer; @@ -418,7 +419,7 @@ public synchronized Writable getRpcResponse() { * socket: responses may be delivered out of order. */ private class Connection extends Thread { private InetSocketAddress server; // server ip:port - private final ConnectionId remoteId; // connection id + private final ConnectionId remoteId; // connection id private AuthMethod authMethod; // authentication method private AuthProtocol authProtocol; private int serviceClass; @@ -644,6 +645,9 @@ private synchronized boolean updateAddress() throws IOException { LOG.warn("Address change detected. Old: " + server.toString() + " New: " + currentAddr.toString()); server = currentAddr; + // Update the remote address so that reconnections are with the updated address. + // This avoids thrashing. + remoteId.setAddress(currentAddr); UserGroupInformation ticket = remoteId.getTicket(); this.setName("IPC Client (" + socketFactory.hashCode() + ") connection to " + server.toString() + " from " @@ -1620,7 +1624,8 @@ private Writable getRpcResponse(final Call call, final Connection connection, } if (call.error != null) { - if (call.error instanceof RemoteException) { + if (call.error instanceof RemoteException || + call.error instanceof SaslException) { call.error.fillInStackTrace(); throw call.error; } else { // local exception @@ -1698,9 +1703,9 @@ private Connection getConnection(ConnectionId remoteId, @InterfaceAudience.LimitedPrivate({"HDFS", "MapReduce"}) @InterfaceStability.Evolving public static class ConnectionId { - InetSocketAddress address; - UserGroupInformation ticket; - final Class protocol; + private InetSocketAddress address; + private final UserGroupInformation ticket; + private final Class protocol; private static final int PRIME = 16777619; private final int rpcTimeout; private final int maxIdleTime; //connections will be culled if it was idle for @@ -1715,8 +1720,8 @@ public static class ConnectionId { private final int pingInterval; // how often sends ping to the server in msecs private String saslQop; // here for testing private final Configuration conf; // used to get the expected kerberos principal name - - ConnectionId(InetSocketAddress address, Class protocol, + + public ConnectionId(InetSocketAddress address, Class protocol, UserGroupInformation ticket, int rpcTimeout, RetryPolicy connectionRetryPolicy, Configuration conf) { this.protocol = protocol; @@ -1751,7 +1756,28 @@ public static class ConnectionId { InetSocketAddress getAddress() { return address; } - + + /** + * This is used to update the remote address when an address change is detected. This method + * ensures that the {@link #hashCode()} won't change. + * + * @param address the updated address + * @throws IllegalArgumentException if the hostname or port doesn't match + * @see Connection#updateAddress() + */ + void setAddress(InetSocketAddress address) { + if (!Objects.equals(this.address.getHostName(), address.getHostName())) { + throw new IllegalArgumentException("Hostname must match: " + this.address + " vs " + + address); + } + if (this.address.getPort() != address.getPort()) { + throw new IllegalArgumentException("Port must match: " + this.address + " vs " + address); + } + + this.address = address; + } + + Class getProtocol() { return protocol; } @@ -1760,7 +1786,7 @@ UserGroupInformation getTicket() { return ticket; } - private int getRpcTimeout() { + int getRpcTimeout() { return rpcTimeout; } @@ -1794,6 +1820,10 @@ boolean getDoPing() { int getPingInterval() { return pingInterval; } + + RetryPolicy getRetryPolicy() { + return connectionRetryPolicy; + } @VisibleForTesting String getSaslQop() { @@ -1858,7 +1888,11 @@ && isEqual(this.protocol, that.protocol) @Override public int hashCode() { int result = connectionRetryPolicy.hashCode(); - result = PRIME * result + ((address == null) ? 0 : address.hashCode()); + // We calculate based on the host name and port without the IP address, since the hashCode + // must be stable even if the IP address is updated. + result = PRIME * result + ((address == null || address.getHostName() == null) ? 0 : + address.getHostName().hashCode()); + result = PRIME * result + ((address == null) ? 0 : address.getPort()); result = PRIME * result + (doPing ? 1231 : 1237); result = PRIME * result + maxIdleTime; result = PRIME * result + pingInterval; diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/ProtobufRpcEngine.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/ProtobufRpcEngine.java index e53f57b1fc9dd..df0f734d08016 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/ProtobufRpcEngine.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/ProtobufRpcEngine.java @@ -77,6 +77,16 @@ public static AsyncGet getAsyncReturnMessage() { return ASYNC_RETURN_MESSAGE.get(); } + @Override + @SuppressWarnings("unchecked") + public ProtocolProxy getProxy(Class protocol, long clientVersion, + ConnectionId connId, Configuration conf, SocketFactory factory, + AlignmentContext alignmentContext) throws IOException { + final Invoker invoker = new Invoker(protocol, connId, conf, factory, alignmentContext); + return new ProtocolProxy(protocol, (T) Proxy.newProxyInstance( + protocol.getClassLoader(), new Class[] {protocol}, invoker), false); + } + public ProtocolProxy getProxy(Class protocol, long clientVersion, InetSocketAddress addr, UserGroupInformation ticket, Configuration conf, SocketFactory factory, int rpcTimeout) throws IOException { @@ -116,7 +126,7 @@ public ProtocolProxy getProtocolMetaInfoProxy( return new ProtocolProxy(protocol, (ProtocolMetaInfoPB) Proxy.newProxyInstance(protocol.getClassLoader(), new Class[] { protocol }, new Invoker(protocol, connId, conf, - factory)), false); + factory, null)), false); } protected static class Invoker implements RpcInvocationHandler { @@ -137,9 +147,8 @@ protected Invoker(Class protocol, InetSocketAddress addr, throws IOException { this(protocol, Client.ConnectionId.getConnectionId( addr, protocol, ticket, rpcTimeout, connectionRetryPolicy, conf), - conf, factory); + conf, factory, alignmentContext); this.fallbackToSimpleAuth = fallbackToSimpleAuth; - this.alignmentContext = alignmentContext; } /** @@ -148,14 +157,16 @@ protected Invoker(Class protocol, InetSocketAddress addr, * @param connId input connId. * @param conf input Configuration. * @param factory input factory. + * @param alignmentContext Alignment context */ protected Invoker(Class protocol, Client.ConnectionId connId, - Configuration conf, SocketFactory factory) { + Configuration conf, SocketFactory factory, AlignmentContext alignmentContext) { this.remoteId = connId; this.client = CLIENTS.getClient(conf, factory, RpcWritable.Buffer.class); this.protocolName = RPC.getProtocolName(protocol); this.clientProtocolVersion = RPC .getProtocolVersion(protocol); + this.alignmentContext = alignmentContext; } private RequestHeaderProto constructRpcRequestHeader(Method method) { diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/ProtobufRpcEngine2.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/ProtobufRpcEngine2.java index 3a8c6275820c4..bedecc8851d6a 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/ProtobufRpcEngine2.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/ProtobufRpcEngine2.java @@ -100,6 +100,16 @@ public ProtocolProxy getProxy( rpcTimeout, connectionRetryPolicy, null, null); } + @Override + @SuppressWarnings("unchecked") + public ProtocolProxy getProxy(Class protocol, long clientVersion, + ConnectionId connId, Configuration conf, SocketFactory factory, + AlignmentContext alignmentContext) throws IOException { + final Invoker invoker = new Invoker(protocol, connId, conf, factory, alignmentContext); + return new ProtocolProxy(protocol, (T) Proxy.newProxyInstance( + protocol.getClassLoader(), new Class[] {protocol}, invoker), false); + } + @Override @SuppressWarnings("unchecked") public ProtocolProxy getProxy(Class protocol, long clientVersion, @@ -123,7 +133,7 @@ public ProtocolProxy getProtocolMetaInfoProxy( return new ProtocolProxy(protocol, (ProtocolMetaInfoPB) Proxy.newProxyInstance(protocol.getClassLoader(), new Class[]{protocol}, new Invoker(protocol, connId, conf, - factory)), false); + factory, null)), false); } protected static class Invoker implements RpcInvocationHandler { @@ -144,9 +154,8 @@ protected Invoker(Class protocol, InetSocketAddress addr, throws IOException { this(protocol, Client.ConnectionId.getConnectionId( addr, protocol, ticket, rpcTimeout, connectionRetryPolicy, conf), - conf, factory); + conf, factory, alignmentContext); this.fallbackToSimpleAuth = fallbackToSimpleAuth; - this.alignmentContext = alignmentContext; } /** @@ -156,14 +165,16 @@ protected Invoker(Class protocol, InetSocketAddress addr, * @param connId input connId. * @param conf input Configuration. * @param factory input factory. + * @param alignmentContext Alignment context */ protected Invoker(Class protocol, Client.ConnectionId connId, - Configuration conf, SocketFactory factory) { + Configuration conf, SocketFactory factory, AlignmentContext alignmentContext) { this.remoteId = connId; this.client = CLIENTS.getClient(conf, factory, RpcWritable.Buffer.class); this.protocolName = RPC.getProtocolName(protocol); this.clientProtocolVersion = RPC .getProtocolVersion(protocol); + this.alignmentContext = alignmentContext; } private RequestHeaderProto constructRpcRequestHeader(Method method) { diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/RPC.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/RPC.java index 818305b316984..fc562b525ad6a 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/RPC.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/RPC.java @@ -541,6 +541,50 @@ public static ProtocolProxy getProtocolProxy(Class protocol, return getProtocolProxy(protocol, clientVersion, addr, ticket, conf, factory, getRpcTimeout(conf), null); } + + /** + * Get a protocol proxy that contains a proxy connection to a remote server + * and a set of methods that are supported by the server. + * + * @param Generics Type T + * @param protocol protocol class + * @param clientVersion client's version + * @param connId client connection identifier + * @param conf configuration + * @param factory socket factory + * @return the protocol proxy + * @throws IOException if the far end through a RemoteException + */ + public static ProtocolProxy getProtocolProxy(Class protocol, + long clientVersion, ConnectionId connId, Configuration conf, + SocketFactory factory) throws IOException { + return getProtocolProxy(protocol, clientVersion, connId, conf, + factory, null); + } + + /** + * Get a protocol proxy that contains a proxy connection to a remote server + * and a set of methods that are supported by the server. + * + * @param Generics Type T + * @param protocol protocol class + * @param clientVersion client's version + * @param connId client connection identifier + * @param conf configuration + * @param factory socket factory + * @param alignmentContext StateID alignment context + * @return the protocol proxy + * @throws IOException if the far end through a RemoteException + */ + public static ProtocolProxy getProtocolProxy(Class protocol, + long clientVersion, ConnectionId connId, Configuration conf, + SocketFactory factory, AlignmentContext alignmentContext) throws IOException { + if (UserGroupInformation.isSecurityEnabled()) { + SaslRpcServer.init(conf); + } + return getProtocolEngine(protocol, conf).getProxy( + protocol, clientVersion, connId, conf, factory, alignmentContext); + } /** * Construct a client-side proxy that implements the named protocol, diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/RetryCache.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/RetryCache.java index 3d64a84bfb46f..624cc08ac25be 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/RetryCache.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/RetryCache.java @@ -55,14 +55,14 @@ public static class CacheEntry implements LightWeightCache.Entry { /** * Processing state of the requests. */ - private static byte INPROGRESS = 0; - private static byte SUCCESS = 1; - private static byte FAILED = 2; + private static final byte INPROGRESS = 0; + private static final byte SUCCESS = 1; + private static final byte FAILED = 2; private byte state = INPROGRESS; // Store uuid as two long for better memory utilization - private final long clientIdMsb; // Most signficant bytes + private final long clientIdMsb; // Most significant bytes private final long clientIdLsb; // Least significant bytes private final int callId; @@ -140,8 +140,8 @@ public long getExpirationTime() { @Override public String toString() { - return (new UUID(this.clientIdMsb, this.clientIdLsb)).toString() + ":" - + this.callId + ":" + this.state; + return String.format("%s:%s:%s", new UUID(this.clientIdMsb, this.clientIdLsb), + this.callId, this.state); } } @@ -183,7 +183,7 @@ public Object getPayload() { private final LightWeightGSet set; private final long expirationTime; - private String cacheName; + private final String cacheName; private final ReentrantLock lock = new ReentrantLock(); @@ -195,7 +195,7 @@ public Object getPayload() { */ public RetryCache(String cacheName, double percentage, long expirationTime) { int capacity = LightWeightGSet.computeCapacity(percentage, cacheName); - capacity = capacity > MAX_CAPACITY ? capacity : MAX_CAPACITY; + capacity = Math.max(capacity, MAX_CAPACITY); this.set = new LightWeightCache(capacity, capacity, expirationTime, 0); this.expirationTime = expirationTime; @@ -203,11 +203,11 @@ public RetryCache(String cacheName, double percentage, long expirationTime) { this.retryCacheMetrics = RetryCacheMetrics.create(this); } - private static boolean skipRetryCache() { + private static boolean skipRetryCache(byte[] clientId, int callId) { // Do not track non RPC invocation or RPC requests with // invalid callId or clientId in retry cache - return !Server.isRpcInvocation() || Server.getCallId() < 0 - || Arrays.equals(Server.getClientId(), RpcConstants.DUMMY_CLIENT_ID); + return !Server.isRpcInvocation() || callId < 0 + || Arrays.equals(clientId, RpcConstants.DUMMY_CLIENT_ID); } public void lock() { @@ -332,43 +332,51 @@ public void addCacheEntryWithPayload(byte[] clientId, int callId, retryCacheMetrics.incrCacheUpdated(); } - private static CacheEntry newEntry(long expirationTime) { - return new CacheEntry(Server.getClientId(), Server.getCallId(), + private static CacheEntry newEntry(long expirationTime, + byte[] clientId, int callId) { + return new CacheEntry(clientId, callId, System.nanoTime() + expirationTime); } private static CacheEntryWithPayload newEntry(Object payload, - long expirationTime) { - return new CacheEntryWithPayload(Server.getClientId(), Server.getCallId(), + long expirationTime, byte[] clientId, int callId) { + return new CacheEntryWithPayload(clientId, callId, payload, System.nanoTime() + expirationTime); } /** * Static method that provides null check for retryCache. * @param cache input Cache. + * @param clientId client id of this request + * @param callId client call id of this request * @return CacheEntry. */ - public static CacheEntry waitForCompletion(RetryCache cache) { - if (skipRetryCache()) { + public static CacheEntry waitForCompletion(RetryCache cache, + byte[] clientId, int callId) { + if (skipRetryCache(clientId, callId)) { return null; } return cache != null ? cache - .waitForCompletion(newEntry(cache.expirationTime)) : null; + .waitForCompletion(newEntry(cache.expirationTime, + clientId, callId)) : null; } /** * Static method that provides null check for retryCache. * @param cache input cache. * @param payload input payload. + * @param clientId client id of this request + * @param callId client call id of this request * @return CacheEntryWithPayload. */ public static CacheEntryWithPayload waitForCompletion(RetryCache cache, - Object payload) { - if (skipRetryCache()) { + Object payload, byte[] clientId, int callId) { + if (skipRetryCache(clientId, callId)) { return null; } return (CacheEntryWithPayload) (cache != null ? cache - .waitForCompletion(newEntry(payload, cache.expirationTime)) : null); + .waitForCompletion(newEntry(payload, cache.expirationTime, + clientId, callId)) : null); } public static void setState(CacheEntry e, boolean success) { diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/RpcEngine.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/RpcEngine.java index afc9d035b097c..f322f6eb98abb 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/RpcEngine.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/RpcEngine.java @@ -57,6 +57,24 @@ ProtocolProxy getProxy(Class protocol, SocketFactory factory, int rpcTimeout, RetryPolicy connectionRetryPolicy) throws IOException; + /** + * Construct a client-side proxy object with a ConnectionId. + * + * @param Generics Type T. + * @param protocol input protocol. + * @param clientVersion input clientVersion. + * @param connId input ConnectionId. + * @param conf input Configuration. + * @param factory input factory. + * @param alignmentContext Alignment context + * @throws IOException raised on errors performing I/O. + * @return ProtocolProxy. + */ + ProtocolProxy getProxy(Class protocol, long clientVersion, + Client.ConnectionId connId, Configuration conf, SocketFactory factory, + AlignmentContext alignmentContext) + throws IOException; + /** * Construct a client-side proxy object. * diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java index e79612f7a5a0f..17366eb9569f1 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java @@ -287,11 +287,8 @@ public static void registerProtocolEngine(RPC.RpcKind rpcKind, throw new IllegalArgumentException("ReRegistration of rpcKind: " + rpcKind); } - if (LOG.isDebugEnabled()) { - LOG.debug("rpcKind=" + rpcKind + - ", rpcRequestWrapperClass=" + rpcRequestWrapperClass + - ", rpcInvoker=" + rpcInvoker); - } + LOG.debug("rpcKind={}, rpcRequestWrapperClass={}, rpcInvoker={}.", + rpcKind, rpcRequestWrapperClass, rpcInvoker); } public Class getRpcRequestWrapper( @@ -928,7 +925,7 @@ public static class Call implements Schedulable, private volatile String detailedMetricsName = ""; final int callId; // the client's call id final int retryCount; // the retry count of the call - long timestampNanos; // time the call was received + private final long timestampNanos; // time the call was received long responseTimestampNanos; // time the call was served private AtomicInteger responseWaitCount = new AtomicInteger(1); final RPC.RpcKind rpcKind; @@ -940,6 +937,9 @@ public static class Call implements Schedulable, // the priority level assigned by scheduler, 0 by default private long clientStateId; private boolean isCallCoordinated; + // Serialized RouterFederatedStateProto message to + // store last seen states for multiple namespaces. + private ByteString federatedNamespaceState; Call() { this(RpcConstants.INVALID_CALL_ID, RpcConstants.INVALID_RETRY_COUNT, @@ -997,6 +997,14 @@ public ProcessingDetails getProcessingDetails() { return processingDetails; } + public void setFederatedNamespaceState(ByteString federatedNamespaceState) { + this.federatedNamespaceState = federatedNamespaceState; + } + + public ByteString getFederatedNamespaceState() { + return this.federatedNamespaceState; + } + @Override public String toString() { return "Call#" + callId + " Retry#" + retryCount; @@ -1110,6 +1118,10 @@ public void setDeferredResponse(Writable response) { public void setDeferredError(Throwable t) { } + + public long getTimestampNanos() { + return timestampNanos; + } } /** A RPC extended call queued for handling. */ @@ -1191,7 +1203,7 @@ public Void run() throws Exception { try { value = call( - rpcKind, connection.protocolName, rpcRequest, timestampNanos); + rpcKind, connection.protocolName, rpcRequest, getTimestampNanos()); } catch (Throwable e) { populateResponseParamsOnError(e, responseParams); } @@ -1212,9 +1224,7 @@ public Void run() throws Exception { deltaNanos = Time.monotonicNowNanos() - startNanos; details.set(Timing.RESPONSE, deltaNanos, TimeUnit.NANOSECONDS); } else { - if (LOG.isDebugEnabled()) { - LOG.debug("Deferring response for callId: " + this.callId); - } + LOG.debug("Deferring response for callId: {}", this.callId); } return null; } @@ -1711,9 +1721,7 @@ private void doRunLoop() { // If there were some calls that have not been sent out for a // long time, discard them. // - if(LOG.isDebugEnabled()) { - LOG.debug("Checking for old call responses."); - } + LOG.debug("Checking for old call responses."); ArrayList calls; // get the list of channels from list of keys. @@ -1813,9 +1821,8 @@ private boolean processResponse(LinkedList responseQueue, // call = responseQueue.removeFirst(); SocketChannel channel = call.connection.channel; - if (LOG.isDebugEnabled()) { - LOG.debug(Thread.currentThread().getName() + ": responding to " + call); - } + + LOG.debug("{}: responding to {}.", Thread.currentThread().getName(), call); // // Send as much data as we can in the non-blocking fashion // @@ -1832,10 +1839,8 @@ private boolean processResponse(LinkedList responseQueue, } else { done = false; // more calls pending to be sent. } - if (LOG.isDebugEnabled()) { - LOG.debug(Thread.currentThread().getName() + ": responding to " + call - + " Wrote " + numBytes + " bytes."); - } + LOG.debug("{}: responding to {} Wrote {} bytes.", + Thread.currentThread().getName(), call, numBytes); } else { // // If we were unable to write the entire response out, then @@ -1860,10 +1865,8 @@ private boolean processResponse(LinkedList responseQueue, decPending(); } } - if (LOG.isDebugEnabled()) { - LOG.debug(Thread.currentThread().getName() + ": responding to " + call - + " Wrote partial " + numBytes + " bytes."); - } + LOG.debug("{}: responding to {} Wrote partial {} bytes.", + Thread.currentThread().getName(), call, numBytes); } error = false; // everything went off well } @@ -2209,13 +2212,11 @@ private void saslProcess(RpcSaslProto saslMessage) if (saslServer != null && saslServer.isComplete()) { if (LOG.isDebugEnabled()) { - LOG.debug("SASL server context established. Negotiated QoP is " - + saslServer.getNegotiatedProperty(Sasl.QOP)); + LOG.debug("SASL server context established. Negotiated QoP is {}.", + saslServer.getNegotiatedProperty(Sasl.QOP)); } user = getAuthorizedUgi(saslServer.getAuthorizationID()); - if (LOG.isDebugEnabled()) { - LOG.debug("SASL server successfully authenticated client: " + user); - } + LOG.debug("SASL server successfully authenticated client: {}.", user); rpcMetrics.incrAuthenticationSuccesses(); AUDITLOG.info(AUTH_SUCCESSFUL_FOR + user + " from " + toString()); saslContextEstablished = true; @@ -2320,10 +2321,8 @@ private RpcSaslProto processSaslToken(RpcSaslProto saslMessage) throw new SaslException("Client did not send a token"); } byte[] saslToken = saslMessage.getToken().toByteArray(); - if (LOG.isDebugEnabled()) { - LOG.debug("Have read input token of size " + saslToken.length - + " for processing by saslServer.evaluateResponse()"); - } + LOG.debug("Have read input token of size {} for processing by saslServer.evaluateResponse()", + saslToken.length); saslToken = saslServer.evaluateResponse(saslToken); return buildSaslResponse( saslServer.isComplete() ? SaslState.SUCCESS : SaslState.CHALLENGE, @@ -2338,9 +2337,8 @@ private void switchToSimple() { private RpcSaslProto buildSaslResponse(SaslState state, byte[] replyToken) { if (LOG.isDebugEnabled()) { - LOG.debug("Will send " + state + " token of size " - + ((replyToken != null) ? replyToken.length : null) - + " from saslServer."); + LOG.debug("Will send {} token of size {} from saslServer.", state, + ((replyToken != null) ? replyToken.length : null)); } RpcSaslProto.Builder response = RpcSaslProto.newBuilder(); response.setState(state); @@ -2664,10 +2662,8 @@ private void processConnectionContext(RpcWritable.Buffer buffer) */ private void unwrapPacketAndProcessRpcs(byte[] inBuf) throws IOException, InterruptedException { - if (LOG.isDebugEnabled()) { - LOG.debug("Have read input token of size " + inBuf.length - + " for processing by saslServer.unwrap()"); - } + LOG.debug("Have read input token of size {} for processing by saslServer.unwrap()", + inBuf.length); inBuf = saslServer.unwrap(inBuf, 0, inBuf.length); ReadableByteChannel ch = Channels.newChannel(new ByteArrayInputStream( inBuf)); @@ -2729,9 +2725,7 @@ private void processOneRpc(ByteBuffer bb) getMessage(RpcRequestHeaderProto.getDefaultInstance(), buffer); callId = header.getCallId(); retry = header.getRetryCount(); - if (LOG.isDebugEnabled()) { - LOG.debug(" got #" + callId); - } + LOG.debug(" got #{}", callId); checkRpcHeaders(header); if (callId < 0) { // callIds typically used during connection setup @@ -2746,11 +2740,8 @@ private void processOneRpc(ByteBuffer bb) } catch (RpcServerException rse) { // inform client of error, but do not rethrow else non-fatal // exceptions will close connection! - if (LOG.isDebugEnabled()) { - LOG.debug(Thread.currentThread().getName() + - ": processOneRpc from client " + this + - " threw exception [" + rse + "]"); - } + LOG.debug("{}: processOneRpc from client {} threw exception [{}]", + Thread.currentThread().getName(), this, rse); // use the wrapped exception if there is one. Throwable t = (rse.getCause() != null) ? rse.getCause() : rse; final RpcCall call = new RpcCall(this, callId, retry); @@ -2888,6 +2879,9 @@ private void processRpcRequest(RpcRequestHeaderProto header, stateId = alignmentContext.receiveRequestState( header, getMaxIdleTime()); call.setClientStateId(stateId); + if (header.hasRouterFederatedState()) { + call.setFederatedNamespaceState(header.getRouterFederatedState()); + } } } catch (IOException ioe) { throw new RpcServerException("Processing RPC request caught ", ioe); @@ -2962,9 +2956,7 @@ private void authorizeConnection() throws RpcServerException { ProxyUsers.authorize(user, this.getHostAddress()); } authorize(user, protocolName, getHostInetAddress()); - if (LOG.isDebugEnabled()) { - LOG.debug("Successfully authorized " + connectionContext); - } + LOG.debug("Successfully authorized {}.", connectionContext); rpcMetrics.incrAuthorizationSuccesses(); } catch (AuthorizationException ae) { LOG.info("Connection from " + this @@ -3081,7 +3073,7 @@ public Handler(int instanceNumber) { @Override public void run() { - LOG.debug(Thread.currentThread().getName() + ": starting"); + LOG.debug("{}: starting", Thread.currentThread().getName()); SERVER.set(Server.this); while (running) { TraceScope traceScope = null; @@ -3115,9 +3107,7 @@ public void run() { call = null; continue; } - if (LOG.isDebugEnabled()) { - LOG.debug(Thread.currentThread().getName() + ": " + call + " for RpcKind " + call.rpcKind); - } + LOG.debug("{}: {} for RpcKind {}.", Thread.currentThread().getName(), call, call.rpcKind); CurCall.set(call); if (call.span != null) { traceScope = tracer.activateSpan(call.span); @@ -3152,21 +3142,21 @@ public void run() { IOUtils.cleanupWithLogger(LOG, traceScope); if (call != null) { updateMetrics(call, startTimeNanos, connDropped); - ProcessingDetails.LOG.debug( - "Served: [{}]{} name={} user={} details={}", + ProcessingDetails.LOG.debug("Served: [{}]{} name={} user={} details={}", call, (call.isResponseDeferred() ? ", deferred" : ""), call.getDetailedMetricsName(), call.getRemoteUser(), call.getProcessingDetails()); } } } - LOG.debug(Thread.currentThread().getName() + ": exiting"); + LOG.debug("{}: exiting", Thread.currentThread().getName()); } private void requeueCall(Call call) throws IOException, InterruptedException { try { internalQueueCall(call, false); + rpcMetrics.incrRequeueCalls(); } catch (RpcServerException rse) { call.doResponse(rse.getCause(), rse.getRpcStatusProto()); } @@ -3389,14 +3379,13 @@ private List getAuthMethods(SecretManager secretManager, " authentication requires a secret manager"); } } else if (secretManager != null) { - LOG.debug(AuthenticationMethod.TOKEN + - " authentication enabled for secret manager"); + LOG.debug("{} authentication enabled for secret manager", AuthenticationMethod.TOKEN); // most preferred, go to the front of the line! authMethods.add(AuthenticationMethod.TOKEN.getAuthMethod()); } authMethods.add(confAuthenticationMethod.getAuthMethod()); - LOG.debug("Server accepts auth methods:" + authMethods); + LOG.debug("Server accepts auth methods:{}", authMethods); return authMethods; } @@ -3556,9 +3545,7 @@ private void wrapWithSasl(RpcCall call) throws IOException { synchronized (call.connection.saslServer) { token = call.connection.saslServer.wrap(token, 0, token.length); } - if (LOG.isDebugEnabled()) - LOG.debug("Adding saslServer wrapped token of size " + token.length - + " as call response."); + LOG.debug("Adding saslServer wrapped token of size {} as call response.", token.length); // rebuild with sasl header and payload RpcResponseHeaderProto saslHeader = RpcResponseHeaderProto.newBuilder() .setCallId(AuthProtocol.SASL.callId) @@ -4004,11 +3991,8 @@ Connection register(SocketChannel channel, int ingressPort, Connection connection = new Connection(channel, Time.now(), ingressPort, isOnAuxiliaryPort); add(connection); - if (LOG.isDebugEnabled()) { - LOG.debug("Server connection from " + connection + - "; # active connections: " + size() + - "; # queued calls: " + callQueue.size()); - } + LOG.debug("Server connection from {}; # active connections: {}; # queued calls: {}.", + connection, size(), callQueue.size()); return connection; } @@ -4016,9 +4000,8 @@ boolean close(Connection connection) { boolean exists = remove(connection); if (exists) { if (LOG.isDebugEnabled()) { - LOG.debug(Thread.currentThread().getName() + - ": disconnecting client " + connection + - ". Number of active connections: "+ size()); + LOG.debug("{}: disconnecting client {}. Number of active connections: {}.", + Thread.currentThread().getName(), connection, size()); } // only close if actually removed to avoid double-closing due // to possible races @@ -4080,9 +4063,7 @@ public void run() { if (!running) { return; } - if (LOG.isDebugEnabled()) { - LOG.debug(Thread.currentThread().getName()+": task running"); - } + LOG.debug("{}: task running", Thread.currentThread().getName()); try { closeIdle(false); } finally { @@ -4131,4 +4112,18 @@ public synchronized void run() { } } + @VisibleForTesting + CallQueueManager getCallQueue() { + return callQueue; + } + + @VisibleForTesting + void setCallQueue(CallQueueManager callQueue) { + this.callQueue = callQueue; + } + + @VisibleForTesting + void setRpcRequestClass(Class rpcRequestClass) { + this.rpcRequestClass = rpcRequestClass; + } } diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/WritableRpcEngine.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/WritableRpcEngine.java index 21181f860d98a..d92bcea5d2eff 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/WritableRpcEngine.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/WritableRpcEngine.java @@ -306,6 +306,29 @@ public ProtocolProxy getProxy(Class protocol, long clientVersion, rpcTimeout, connectionRetryPolicy, null, null); } + /** + * Construct a client-side proxy object with a ConnectionId. + * + * @param Generics Type T. + * @param protocol input protocol. + * @param clientVersion input clientVersion. + * @param connId input ConnectionId. + * @param conf input Configuration. + * @param factory input factory. + * @param alignmentContext Alignment context + * @throws IOException raised on errors performing I/O. + * @return ProtocolProxy. + */ + @Override + public ProtocolProxy getProxy(Class protocol, long clientVersion, + Client.ConnectionId connId, Configuration conf, SocketFactory factory, + AlignmentContext alignmentContext) + throws IOException { + return getProxy(protocol, clientVersion, connId.getAddress(), + connId.getTicket(), conf, factory, connId.getRpcTimeout(), + connId.getRetryPolicy(), null, alignmentContext); + } + /** * Construct a client-side proxy object that implements the named protocol, * talking to a server at the named address. diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/metrics/RpcMetrics.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/metrics/RpcMetrics.java index bf21e3865fa8a..282eca3cf8373 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/metrics/RpcMetrics.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/metrics/RpcMetrics.java @@ -128,6 +128,8 @@ public static RpcMetrics create(Server server, Configuration conf) { MutableCounterLong rpcClientBackoff; @Metric("Number of Slow RPC calls") MutableCounterLong rpcSlowCalls; + @Metric("Number of requeue calls") + MutableCounterLong rpcRequeueCalls; @Metric("Number of open connections") public int numOpenConnections() { return server.getNumOpenConnections(); @@ -304,6 +306,13 @@ public void incrSlowRpc() { rpcSlowCalls.incr(); } + /** + * Increments the Requeue Calls counter. + */ + public void incrRequeueCalls() { + rpcRequeueCalls.incr(); + } + /** * Returns a MutableRate Counter. * @return Mutable Rate @@ -344,6 +353,15 @@ public long getRpcSlowCalls() { return rpcSlowCalls.value(); } + /** + * Returns the number of requeue calls. + * @return long + */ + @VisibleForTesting + public long getRpcRequeueCalls() { + return rpcRequeueCalls.value(); + } + public MutableRate getDeferredRpcProcessingTime() { return deferredRpcProcessingTime; } @@ -364,4 +382,9 @@ public double getDeferredRpcProcessingStdDev() { public MetricsTag getTag(String tagName) { return registry.getTag(tagName); } + + @VisibleForTesting + public MutableCounterLong getRpcAuthorizationSuccesses() { + return rpcAuthorizationSuccesses; + } } diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/impl/MetricsRecordImpl.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/impl/MetricsRecordImpl.java index 9ffceaaa0ddda..b11f775a73db3 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/impl/MetricsRecordImpl.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/impl/MetricsRecordImpl.java @@ -22,12 +22,14 @@ import static org.apache.hadoop.util.Preconditions.*; +import org.apache.hadoop.classification.VisibleForTesting; import org.apache.hadoop.metrics2.MetricsInfo; import org.apache.hadoop.metrics2.AbstractMetric; import org.apache.hadoop.metrics2.MetricsTag; import static org.apache.hadoop.metrics2.util.Contracts.*; -class MetricsRecordImpl extends AbstractMetricsRecord { +@VisibleForTesting +public class MetricsRecordImpl extends AbstractMetricsRecord { protected static final String DEFAULT_CONTEXT = "default"; private final long timestamp; diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/lib/MutableRatesWithAggregation.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/lib/MutableRatesWithAggregation.java index 4c5f0a844aaab..60b33a84b519b 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/lib/MutableRatesWithAggregation.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/lib/MutableRatesWithAggregation.java @@ -72,7 +72,7 @@ public void init(Class protocol) { return; } protocolCache.add(protocol); - for (Method method : protocol.getDeclaredMethods()) { + for (Method method : protocol.getMethods()) { String name = method.getName(); LOG.debug(name); addMetricIfNotExists(name); diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/sink/GraphiteSink.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/sink/GraphiteSink.java index ea1bde3a75e03..e07260c99936f 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/sink/GraphiteSink.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/sink/GraphiteSink.java @@ -21,6 +21,7 @@ import org.apache.commons.configuration2.SubsetConfiguration; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.classification.VisibleForTesting; import org.apache.hadoop.metrics2.AbstractMetric; import org.apache.hadoop.metrics2.MetricsException; import org.apache.hadoop.metrics2.MetricsRecord; @@ -37,171 +38,173 @@ import java.nio.charset.StandardCharsets; /** - * A metrics sink that writes to a Graphite server + * A metrics sink that writes to a Graphite server. */ @InterfaceAudience.Public @InterfaceStability.Evolving public class GraphiteSink implements MetricsSink, Closeable { - private static final Logger LOG = - LoggerFactory.getLogger(GraphiteSink.class); - private static final String SERVER_HOST_KEY = "server_host"; - private static final String SERVER_PORT_KEY = "server_port"; - private static final String METRICS_PREFIX = "metrics_prefix"; - private String metricsPrefix = null; - private Graphite graphite = null; - - @Override - public void init(SubsetConfiguration conf) { - // Get Graphite host configurations. - final String serverHost = conf.getString(SERVER_HOST_KEY); - final int serverPort = Integer.parseInt(conf.getString(SERVER_PORT_KEY)); - - // Get Graphite metrics graph prefix. - metricsPrefix = conf.getString(METRICS_PREFIX); - if (metricsPrefix == null) - metricsPrefix = ""; - - graphite = new Graphite(serverHost, serverPort); - graphite.connect(); + private static final Logger LOG = + LoggerFactory.getLogger(GraphiteSink.class); + private static final String SERVER_HOST_KEY = "server_host"; + private static final String SERVER_PORT_KEY = "server_port"; + private static final String METRICS_PREFIX = "metrics_prefix"; + private String metricsPrefix = null; + private Graphite graphite = null; + + @Override + public void init(SubsetConfiguration conf) { + // Get Graphite host configurations. + final String serverHost = conf.getString(SERVER_HOST_KEY); + final int serverPort = Integer.parseInt(conf.getString(SERVER_PORT_KEY)); + + // Get Graphite metrics graph prefix. + metricsPrefix = conf.getString(METRICS_PREFIX); + if (metricsPrefix == null) { + metricsPrefix = ""; } - @Override - public void putMetrics(MetricsRecord record) { - StringBuilder lines = new StringBuilder(); - StringBuilder metricsPathPrefix = new StringBuilder(); - - // Configure the hierarchical place to display the graph. - metricsPathPrefix.append(metricsPrefix).append(".") - .append(record.context()).append(".").append(record.name()); - - for (MetricsTag tag : record.tags()) { - if (tag.value() != null) { - metricsPathPrefix.append(".") - .append(tag.name()) - .append("=") - .append(tag.value()); - } - } - - // The record timestamp is in milliseconds while Graphite expects an epoc time in seconds. - long timestamp = record.timestamp() / 1000L; + graphite = new Graphite(serverHost, serverPort); + graphite.connect(); + } + + @Override + public void putMetrics(MetricsRecord record) { + StringBuilder lines = new StringBuilder(); + StringBuilder metricsPathPrefix = new StringBuilder(); + + // Configure the hierarchical place to display the graph. + metricsPathPrefix.append(metricsPrefix).append(".") + .append(record.context()).append(".").append(record.name()); + + for (MetricsTag tag : record.tags()) { + if (tag.value() != null) { + metricsPathPrefix.append(".") + .append(tag.name()) + .append("=") + .append(tag.value()); + } + } - // Collect datapoints. - for (AbstractMetric metric : record.metrics()) { - lines.append( - metricsPathPrefix.toString() + "." - + metric.name().replace(' ', '.')).append(" ") - .append(metric.value()).append(" ").append(timestamp) - .append("\n"); - } + // The record timestamp is in milliseconds while Graphite expects an epoc time in seconds. + long timestamp = record.timestamp() / 1000L; - try { - graphite.write(lines.toString()); - } catch (Exception e) { - LOG.warn("Error sending metrics to Graphite", e); - try { - graphite.close(); - } catch (Exception e1) { - throw new MetricsException("Error closing connection to Graphite", e1); - } - } + // Collect datapoints. + for (AbstractMetric metric : record.metrics()) { + lines.append(metricsPathPrefix + "." + metric.name().replace(' ', '.')).append(" ") + .append(metric.value()).append(" ").append(timestamp) + .append("\n"); } - @Override - public void flush() { + try { + graphite.write(lines.toString()); + } catch (Exception e) { + LOG.warn("Error sending metrics to Graphite.", e); try { - graphite.flush(); - } catch (Exception e) { - LOG.warn("Error flushing metrics to Graphite", e); - try { - graphite.close(); - } catch (Exception e1) { - throw new MetricsException("Error closing connection to Graphite", e1); - } + graphite.close(); + } catch (Exception e1) { + throw new MetricsException("Error closing connection to Graphite", e1); } } - - @Override - public void close() throws IOException { - graphite.close(); + } + + @Override + public void flush() { + try { + graphite.flush(); + } catch (Exception e) { + LOG.warn("Error flushing metrics to Graphite.", e); + try { + graphite.close(); + } catch (Exception e1) { + throw new MetricsException("Error closing connection to Graphite.", e1); + } } + } - public static class Graphite { - private final static int MAX_CONNECTION_FAILURES = 5; + @Override + public void close() throws IOException { + graphite.close(); + } - private String serverHost; - private int serverPort; - private Writer writer = null; - private Socket socket = null; - private int connectionFailures = 0; + public static class Graphite { + private final static int MAX_CONNECTION_FAILURES = 5; - public Graphite(String serverHost, int serverPort) { - this.serverHost = serverHost; - this.serverPort = serverPort; - } + private String serverHost; + private int serverPort; + private Writer writer = null; + private Socket socket = null; + private int connectionFailures = 0; - public void connect() { - if (isConnected()) { - throw new MetricsException("Already connected to Graphite"); - } - if (tooManyConnectionFailures()) { - // return silently (there was ERROR in logs when we reached limit for the first time) - return; - } - try { + public Graphite(String serverHost, int serverPort) { + this.serverHost = serverHost; + this.serverPort = serverPort; + } + + public void connect() { + if (isConnected()) { + throw new MetricsException("Already connected to Graphite"); + } + if (tooManyConnectionFailures()) { + // return silently (there was ERROR in logs when we reached limit for the first time) + return; + } + try { // Open a connection to Graphite server. - socket = new Socket(serverHost, serverPort); + socket = new Socket(serverHost, serverPort); writer = new OutputStreamWriter(socket.getOutputStream(), StandardCharsets.UTF_8); - } catch (Exception e) { - connectionFailures++; - if (tooManyConnectionFailures()) { - // first time when connection limit reached, report to logs - LOG.error("Too many connection failures, would not try to connect again."); - } - throw new MetricsException("Error creating connection, " - + serverHost + ":" + serverPort, e); + } catch (Exception e) { + connectionFailures++; + if (tooManyConnectionFailures()) { + // first time when connection limit reached, report to logs + LOG.error("Too many connection failures, would not try to connect again."); } + throw new MetricsException("Error creating connection, " + + serverHost + ":" + serverPort, e); } + } - public void write(String msg) throws IOException { - if (!isConnected()) { - connect(); - } - if (isConnected()) { - writer.write(msg); - } + public void write(String msg) throws IOException { + if (!isConnected()) { + connect(); } - - public void flush() throws IOException { - if (isConnected()) { - writer.flush(); - } + if (isConnected()) { + writer.write(msg); } + } - public boolean isConnected() { - return socket != null && socket.isConnected() && !socket.isClosed(); + public void flush() throws IOException { + if (isConnected()) { + writer.flush(); } + } - public void close() throws IOException { - try { - if (writer != null) { - writer.close(); - } - } catch (IOException ex) { - if (socket != null) { - socket.close(); - } - } finally { - socket = null; - writer = null; - } - } + public boolean isConnected() { + return socket != null && socket.isConnected() && !socket.isClosed(); + } - private boolean tooManyConnectionFailures() { - return connectionFailures > MAX_CONNECTION_FAILURES; + public void close() throws IOException { + try { + if (writer != null) { + writer.close(); + } + } catch (IOException ex) { + if (socket != null) { + socket.close(); + } + } finally { + socket = null; + writer = null; } + } + private boolean tooManyConnectionFailures() { + return connectionFailures > MAX_CONNECTION_FAILURES; } + } + @VisibleForTesting + void setGraphite(Graphite graphite) { + this.graphite = graphite; + } } diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/sink/StatsDSink.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/sink/StatsDSink.java index d1ec47fdecb31..4f41c0b0057ce 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/sink/StatsDSink.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/sink/StatsDSink.java @@ -28,6 +28,7 @@ import org.apache.commons.configuration2.SubsetConfiguration; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.classification.VisibleForTesting; import org.apache.hadoop.metrics2.AbstractMetric; import org.apache.hadoop.metrics2.MetricType; import org.apache.hadoop.metrics2.MetricsException; @@ -214,5 +215,8 @@ public void close() throws IOException { } } - + @VisibleForTesting + void setStatsd(StatsD statsd) { + this.statsd = statsd; + } } diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/sink/ganglia/AbstractGangliaSink.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/sink/ganglia/AbstractGangliaSink.java index d3d794fa74a91..620f2f2faa679 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/sink/ganglia/AbstractGangliaSink.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/sink/ganglia/AbstractGangliaSink.java @@ -21,6 +21,7 @@ import java.io.IOException; import java.net.*; import java.nio.charset.StandardCharsets; +import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -78,6 +79,10 @@ public abstract class AbstractGangliaSink implements MetricsSink { private int offset; private boolean supportSparseMetrics = SUPPORT_SPARSE_METRICS_DEFAULT; + public List getMetricsServers() { + return metricsServers; + } + /** * Used for visiting Metrics */ @@ -133,8 +138,11 @@ public void init(SubsetConfiguration conf) { } // load the gannglia servers from properties - metricsServers = Servers.parse(conf.getString(SERVERS_PROPERTY), - DEFAULT_PORT); + List serversFromConf = + conf.getList(String.class, SERVERS_PROPERTY, new ArrayList()); + metricsServers = + Servers.parse(serversFromConf.size() > 0 ? String.join(",", serversFromConf) : null, + DEFAULT_PORT); multicastEnabled = conf.getBoolean(MULTICAST_ENABLED_PROPERTY, DEFAULT_MULTICAST_ENABLED); multicastTtl = conf.getInt(MULTICAST_TTL_PROPERTY, DEFAULT_MULTICAST_TTL); diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/util/SampleStat.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/util/SampleStat.java index 23abfc4bedc31..dec7033424fbe 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/util/SampleStat.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/util/SampleStat.java @@ -27,33 +27,29 @@ public class SampleStat { private final MinMax minmax = new MinMax(); private long numSamples = 0; - private double a0, a1, s0, s1, total; + private double mean, s; /** * Construct a new running sample stat */ public SampleStat() { - a0 = s0 = 0.0; - total = 0.0; + mean = 0.0; + s = 0.0; } public void reset() { numSamples = 0; - a0 = s0 = 0.0; - total = 0.0; + mean = 0.0; + s = 0.0; minmax.reset(); } // We want to reuse the object, sometimes. - void reset(long numSamples, double a0, double a1, double s0, double s1, - double total, MinMax minmax) { - this.numSamples = numSamples; - this.a0 = a0; - this.a1 = a1; - this.s0 = s0; - this.s1 = s1; - this.total = total; - this.minmax.reset(minmax); + void reset(long numSamples1, double mean1, double s1, MinMax minmax1) { + numSamples = numSamples1; + mean = mean1; + s = s1; + minmax.reset(minmax1); } /** @@ -61,7 +57,7 @@ void reset(long numSamples, double a0, double a1, double s0, double s1, * @param other the destination to hold our values */ public void copyTo(SampleStat other) { - other.reset(numSamples, a0, a1, s0, s1, total, minmax); + other.reset(numSamples, mean, s, minmax); } /** @@ -78,24 +74,22 @@ public SampleStat add(double x) { * Add some sample and a partial sum to the running stat. * Note, min/max is not evaluated using this method. * @param nSamples number of samples - * @param x the partial sum + * @param xTotal the partial sum * @return self */ - public SampleStat add(long nSamples, double x) { + public SampleStat add(long nSamples, double xTotal) { numSamples += nSamples; - total += x; - if (numSamples == 1) { - a0 = a1 = x; - s0 = 0.0; - } - else { - // The Welford method for numerical stability - a1 = a0 + (x - a0) / numSamples; - s1 = s0 + (x - a0) * (x - a1); - a0 = a1; - s0 = s1; - } + // use the weighted incremental version of Welford's algorithm to get + // numerical stability while treating the samples as being weighted + // by nSamples + // see https://en.wikipedia.org/wiki/Algorithms_for_calculating_variance + + double x = xTotal / nSamples; + double meanOld = mean; + + mean += ((double) nSamples / numSamples) * (x - meanOld); + s += nSamples * (x - meanOld) * (x - mean); return this; } @@ -110,21 +104,21 @@ public long numSamples() { * @return the total of all samples added */ public double total() { - return total; + return mean * numSamples; } /** * @return the arithmetic mean of the samples */ public double mean() { - return numSamples > 0 ? (total / numSamples) : 0.0; + return numSamples > 0 ? mean : 0.0; } /** * @return the variance of the samples */ public double variance() { - return numSamples > 1 ? s1 / (numSamples - 1) : 0.0; + return numSamples > 1 ? s / (numSamples - 1) : 0.0; } /** diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/DNSDomainNameResolver.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/DNSDomainNameResolver.java index 5866e2960fe1b..ce962bf9e8c6a 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/DNSDomainNameResolver.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/DNSDomainNameResolver.java @@ -18,6 +18,10 @@ package org.apache.hadoop.net; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import javax.naming.NamingException; import java.net.InetAddress; import java.net.UnknownHostException; @@ -27,6 +31,10 @@ * fully qualified domain names belonging to the IPs from this host name */ public class DNSDomainNameResolver implements DomainNameResolver { + + private final static Logger LOG = + LoggerFactory.getLogger(DNSDomainNameResolver.class.getName()); + @Override public InetAddress[] getAllByDomainName(String domainName) throws UnknownHostException { @@ -40,6 +48,16 @@ public String getHostnameByIP(InetAddress address) { && host.charAt(host.length()-1) == '.') { host = host.substring(0, host.length()-1); } + // Protect against the Java behaviour of returning the IP address as a string from a cache + // instead of performing a reverse lookup. + if (host != null && host.equals(address.getHostAddress())) { + LOG.debug("IP address returned for FQDN detected: {}", address.getHostAddress()); + try { + return DNS.reverseDns(address, null); + } catch (NamingException lookupFailure) { + LOG.warn("Failed to perform reverse lookup: {}", address); + } + } return host; } diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetworkTopology.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetworkTopology.java index ebb354e7db3cb..50be1ab759f36 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetworkTopology.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetworkTopology.java @@ -522,8 +522,7 @@ protected Node chooseRandom(final String scope, String excludedScope, } } if (numOfDatanodes <= 0) { - LOG.debug("Failed to find datanode (scope=\"{}\" excludedScope=\"{}\")." - + " numOfDatanodes={}", + LOG.debug("Failed to find datanode (scope=\"{}\" excludedScope=\"{}\"). numOfDatanodes={}", scope, excludedScope, numOfDatanodes); return null; } @@ -539,10 +538,12 @@ protected Node chooseRandom(final String scope, String excludedScope, netlock.readLock().unlock(); } } - LOG.debug("Choosing random from {} available nodes on node {}," - + " scope={}, excludedScope={}, excludeNodes={}. numOfDatanodes={}.", - availableNodes, innerNode, scope, excludedScope, excludedNodes, - numOfDatanodes); + if (LOG.isDebugEnabled()) { + LOG.debug("Choosing random from {} available nodes on node {}, scope={}," + + " excludedScope={}, excludeNodes={}. numOfDatanodes={}.", + availableNodes, innerNode, scope, excludedScope, excludedNodes, + numOfDatanodes); + } Node ret = null; if (availableNodes > 0) { ret = chooseRandom(innerNode, node, excludedNodes, numOfDatanodes, diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/LdapGroupsMapping.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/LdapGroupsMapping.java index 4eb3d865ec78e..f0ff5bd700877 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/LdapGroupsMapping.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/LdapGroupsMapping.java @@ -30,6 +30,7 @@ import java.security.GeneralSecurityException; import java.security.KeyStore; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collections; import java.util.Hashtable; import java.util.Iterator; @@ -59,6 +60,7 @@ import javax.net.ssl.TrustManager; import javax.net.ssl.TrustManagerFactory; +import org.apache.hadoop.classification.VisibleForTesting; import org.apache.hadoop.thirdparty.com.google.common.collect.Iterators; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; @@ -251,6 +253,10 @@ public class LdapGroupsMapping public static final String POSIX_GID_ATTR_KEY = LDAP_CONFIG_PREFIX + ".posix.attr.gid.name"; public static final String POSIX_GID_ATTR_DEFAULT = "gidNumber"; + public static final String GROUP_SEARCH_FILTER_PATTERN = + LDAP_CONFIG_PREFIX + ".group.search.filter.pattern"; + public static final String GROUP_SEARCH_FILTER_PATTERN_DEFAULT = ""; + /* * Posix attributes */ @@ -336,6 +342,7 @@ public class LdapGroupsMapping private int numAttempts; private volatile int numAttemptsBeforeFailover; private volatile String ldapCtxFactoryClassName; + private volatile String[] groupSearchFilterParams; /** * Returns list of groups for a user. @@ -428,15 +435,22 @@ private NamingEnumeration lookupPosixGroup(SearchResult result, * @return a list of strings representing group names of the user. * @throws NamingException if unable to find group names */ - private Set lookupGroup(SearchResult result, DirContext c, + @VisibleForTesting + Set lookupGroup(SearchResult result, DirContext c, int goUpHierarchy) throws NamingException { Set groups = new LinkedHashSet<>(); Set groupDNs = new HashSet<>(); NamingEnumeration groupResults; - // perform the second LDAP query - if (isPosix) { + + String[] resolved = resolveCustomGroupFilterArgs(result); + // If custom group filter argument is supplied, use that!!! + if (resolved != null) { + groupResults = + c.search(groupbaseDN, groupSearchFilter, resolved, SEARCH_CONTROLS); + } else if (isPosix) { + // perform the second LDAP query groupResults = lookupPosixGroup(result, c); } else { String userDn = result.getNameInNamespace(); @@ -460,6 +474,25 @@ private Set lookupGroup(SearchResult result, DirContext c, return groups; } + private String[] resolveCustomGroupFilterArgs(SearchResult result) + throws NamingException { + if (groupSearchFilterParams != null) { + String[] filterElems = new String[groupSearchFilterParams.length]; + for (int i = 0; i < groupSearchFilterParams.length; i++) { + // Specific handling for userDN. + if (groupSearchFilterParams[i].equalsIgnoreCase("userDN")) { + filterElems[i] = result.getNameInNamespace(); + } else { + filterElems[i] = + result.getAttributes().get(groupSearchFilterParams[i]).get() + .toString(); + } + } + return filterElems; + } + return null; + } + /** * Perform LDAP queries to get group names of a user. * @@ -510,6 +543,8 @@ Set doGetGroups(String user, int goUpHierarchy) } } catch (NamingException e) { // If the first lookup failed, fall back to the typical scenario. + // In order to force the fallback, we need to reset groups collection. + groups.clear(); LOG.info("Failed to get groups from the first lookup. Initiating " + "the second LDAP query using the user's DN.", e); } @@ -777,6 +812,12 @@ public synchronized void setConf(Configuration conf) { conf.get(POSIX_UID_ATTR_KEY, POSIX_UID_ATTR_DEFAULT); posixGidAttr = conf.get(POSIX_GID_ATTR_KEY, POSIX_GID_ATTR_DEFAULT); + String groupSearchFilterParamCSV = conf.get(GROUP_SEARCH_FILTER_PATTERN, + GROUP_SEARCH_FILTER_PATTERN_DEFAULT); + if(groupSearchFilterParamCSV!=null && !groupSearchFilterParamCSV.isEmpty()) { + LOG.debug("Using custom group search filters: {}", groupSearchFilterParamCSV); + groupSearchFilterParams = groupSearchFilterParamCSV.split(","); + } int dirSearchTimeout = conf.getInt(DIRECTORY_SEARCH_TIMEOUT, DIRECTORY_SEARCH_TIMEOUT_DEFAULT); @@ -791,7 +832,16 @@ public synchronized void setConf(Configuration conf) { returningAttributes = new String[] { groupNameAttr, posixUidAttr, posixGidAttr}; } - SEARCH_CONTROLS.setReturningAttributes(returningAttributes); + + // If custom group filter is being used, fetch attributes in the filter + // as well. + ArrayList customAttributes = new ArrayList<>(); + if (groupSearchFilterParams != null) { + customAttributes.addAll(Arrays.asList(groupSearchFilterParams)); + } + customAttributes.addAll(Arrays.asList(returningAttributes)); + SEARCH_CONTROLS + .setReturningAttributes(customAttributes.toArray(new String[0])); // LDAP_CTX_FACTORY_CLASS_DEFAULT is not open to unnamed modules // in Java 11+, so the default value is set to null to avoid diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/SaslRpcClient.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/SaslRpcClient.java index e5d62389abab7..ce7878480e22c 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/SaslRpcClient.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/SaslRpcClient.java @@ -237,7 +237,14 @@ private SaslClient createSaslClient(SaslAuth authType) LOG.debug("client isn't using kerberos"); return null; } - String serverPrincipal = getServerPrincipal(authType); + final String serverPrincipal; + try { + serverPrincipal = getServerPrincipal(authType); + } catch (IllegalArgumentException ex) { + // YARN-11210: getServerPrincipal can throw IllegalArgumentException if Kerberos + // configuration is bad, this is surfaced as a non-retryable SaslException + throw new SaslException("Bad Kerberos server principal configuration", ex); + } if (serverPrincipal == null) { LOG.debug("protocol doesn't use kerberos"); return null; diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/SecurityUtil.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/SecurityUtil.java index 2b9822a3d4817..3369869bde24d 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/SecurityUtil.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/SecurityUtil.java @@ -44,6 +44,8 @@ import org.apache.hadoop.fs.CommonConfigurationKeys; import org.apache.hadoop.io.Text; import org.apache.hadoop.net.DNS; +import org.apache.hadoop.net.DomainNameResolver; +import org.apache.hadoop.net.DomainNameResolverFactory; import org.apache.hadoop.net.NetUtils; import org.apache.hadoop.security.UserGroupInformation.AuthenticationMethod; import org.apache.hadoop.security.token.Token; @@ -81,6 +83,8 @@ private SecurityUtil() { @VisibleForTesting static HostResolver hostResolver; + private static DomainNameResolver domainNameResolver; + private static boolean logSlowLookups; private static int slowLookupThresholdMs; @@ -112,6 +116,9 @@ private static void setConfigurationInternal(Configuration conf) { .HADOOP_SECURITY_DNS_LOG_SLOW_LOOKUPS_THRESHOLD_MS_KEY, CommonConfigurationKeys .HADOOP_SECURITY_DNS_LOG_SLOW_LOOKUPS_THRESHOLD_MS_DEFAULT); + + domainNameResolver = DomainNameResolverFactory.newInstance(conf, + CommonConfigurationKeys.HADOOP_SECURITY_RESOLVER_IMPL); } /** @@ -212,7 +219,7 @@ public static String getServerPrincipal(String principalConfig, throw new IOException("Can't replace " + HOSTNAME_PATTERN + " pattern since client address is null"); } - return replacePattern(components, addr.getCanonicalHostName()); + return replacePattern(components, domainNameResolver.getHostnameByIP(addr)); } } diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/ssl/FileBasedKeyStoresFactory.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/ssl/FileBasedKeyStoresFactory.java index 520047b3a0414..4fbb3c8aea6fc 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/ssl/FileBasedKeyStoresFactory.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/ssl/FileBasedKeyStoresFactory.java @@ -319,8 +319,10 @@ String getPassword(Configuration conf, String alias, String defaultPass) { */ @Override public synchronized void destroy() { - if (trustManager != null) { + if (fileMonitoringTimer != null) { fileMonitoringTimer.cancel(); + } + if (trustManager != null) { trustManager = null; keyManagers = null; trustManagers = null; diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/ZKDelegationTokenSecretManager.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/ZKDelegationTokenSecretManager.java index d0630e38b4ddb..fb9a2951f598a 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/ZKDelegationTokenSecretManager.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/ZKDelegationTokenSecretManager.java @@ -42,7 +42,6 @@ import org.apache.curator.framework.recipes.shared.SharedCount; import org.apache.curator.framework.recipes.shared.VersionedValue; import org.apache.curator.retry.RetryNTimes; -import org.apache.curator.utils.EnsurePath; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.classification.InterfaceStability.Unstable; @@ -60,6 +59,7 @@ import org.apache.zookeeper.client.ZKClientConfig; import org.apache.zookeeper.data.ACL; import org.apache.zookeeper.data.Id; +import org.apache.zookeeper.data.Stat; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -134,6 +134,11 @@ public static void setCurator(CuratorFramework curator) { CURATOR_TL.set(curator); } + @VisibleForTesting + protected static CuratorFramework getCurator() { + return CURATOR_TL.get(); + } + private final boolean isExternalClient; protected final CuratorFramework zkClient; private SharedCount delTokSeqCounter; @@ -260,10 +265,12 @@ public void startThreads() throws IOException { // If namespace parents are implicitly created, they won't have ACLs. // So, let's explicitly create them. CuratorFramework nullNsFw = zkClient.usingNamespace(null); - EnsurePath ensureNs = - nullNsFw.newNamespaceAwareEnsurePath("/" + zkClient.getNamespace()); try { - ensureNs.ensure(nullNsFw.getZookeeperClient()); + String nameSpace = "/" + zkClient.getNamespace(); + Stat stat = nullNsFw.checkExists().forPath(nameSpace); + if (stat == null) { + nullNsFw.create().creatingParentContainersIfNeeded().forPath(nameSpace); + } } catch (Exception e) { throw new IOException("Could not create namespace", e); } diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/ExitUtil.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/ExitUtil.java index 32f4b5b7a72d4..dd47aeeefac2c 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/ExitUtil.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/ExitUtil.java @@ -17,6 +17,8 @@ */ package org.apache.hadoop.util; +import java.util.concurrent.atomic.AtomicReference; + import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; import org.slf4j.Logger; @@ -36,8 +38,10 @@ public final class ExitUtil { LOG = LoggerFactory.getLogger(ExitUtil.class.getName()); private static volatile boolean systemExitDisabled = false; private static volatile boolean systemHaltDisabled = false; - private static volatile ExitException firstExitException; - private static volatile HaltException firstHaltException; + private static final AtomicReference FIRST_EXIT_EXCEPTION = + new AtomicReference<>(); + private static final AtomicReference FIRST_HALT_EXCEPTION = + new AtomicReference<>(); /** Message raised from an exit exception if none were provided: {@value}. */ public static final String EXIT_EXCEPTION_MESSAGE = "ExitException"; /** Message raised from a halt exception if none were provided: {@value}. */ @@ -159,28 +163,29 @@ public static void disableSystemHalt() { */ public static boolean terminateCalled() { // Either we set this member or we actually called System#exit - return firstExitException != null; + return FIRST_EXIT_EXCEPTION.get() != null; } /** * @return true if halt has been called. */ public static boolean haltCalled() { - return firstHaltException != null; + // Either we set this member or we actually called Runtime#halt + return FIRST_HALT_EXCEPTION.get() != null; } /** - * @return the first ExitException thrown, null if none thrown yet. + * @return the first {@code ExitException} thrown, null if none thrown yet. */ public static ExitException getFirstExitException() { - return firstExitException; + return FIRST_EXIT_EXCEPTION.get(); } /** * @return the first {@code HaltException} thrown, null if none thrown yet. */ public static HaltException getFirstHaltException() { - return firstHaltException; + return FIRST_HALT_EXCEPTION.get(); } /** @@ -188,63 +193,136 @@ public static HaltException getFirstHaltException() { * where one test in the suite expects an exit but others do not. */ public static void resetFirstExitException() { - firstExitException = null; + FIRST_EXIT_EXCEPTION.set(null); } + /** + * Reset the tracking of process termination. This is for use in unit tests + * where one test in the suite expects a halt but others do not. + */ public static void resetFirstHaltException() { - firstHaltException = null; + FIRST_HALT_EXCEPTION.set(null); } /** + * Suppresses if legit and returns the first non-null of the two. Legit means + * suppressor if neither null nor suppressed. + * @param suppressor Throwable that suppresses suppressed + * @param suppressed Throwable that is suppressed by suppressor + * @return suppressor if not null, suppressed otherwise + */ + private static T addSuppressed(T suppressor, T suppressed) { + if (suppressor == null) { + return suppressed; + } + if (suppressor != suppressed) { + suppressor.addSuppressed(suppressed); + } + return suppressor; + } + + /** + * Exits the JVM if exit is enabled, rethrow provided exception or any raised error otherwise. * Inner termination: either exit with the exception's exit code, * or, if system exits are disabled, rethrow the exception. * @param ee exit exception + * @throws ExitException if {@link System#exit(int)} is disabled and not suppressed by an Error + * @throws Error if {@link System#exit(int)} is disabled and one Error arise, suppressing + * anything else, even ee */ - public static synchronized void terminate(ExitException ee) - throws ExitException { - int status = ee.getExitCode(); - String msg = ee.getMessage(); + public static void terminate(final ExitException ee) throws ExitException { + final int status = ee.getExitCode(); + Error caught = null; if (status != 0) { - //exit indicates a problem, log it - LOG.debug("Exiting with status {}: {}", status, msg, ee); - LOG.info("Exiting with status {}: {}", status, msg); + try { + // exit indicates a problem, log it + String msg = ee.getMessage(); + LOG.debug("Exiting with status {}: {}", status, msg, ee); + LOG.info("Exiting with status {}: {}", status, msg); + } catch (Error e) { + // errors have higher priority than HaltException, it may be re-thrown. + // OOM and ThreadDeath are 2 examples of Errors to re-throw + caught = e; + } catch (Throwable t) { + // all other kind of throwables are suppressed + addSuppressed(ee, t); + } } if (systemExitDisabled) { - LOG.error("Terminate called", ee); - if (!terminateCalled()) { - firstExitException = ee; + try { + LOG.error("Terminate called", ee); + } catch (Error e) { + // errors have higher priority again, if it's a 2nd error, the 1st one suprpesses it + caught = addSuppressed(caught, e); + } catch (Throwable t) { + // all other kind of throwables are suppressed + addSuppressed(ee, t); } + FIRST_EXIT_EXCEPTION.compareAndSet(null, ee); + if (caught != null) { + caught.addSuppressed(ee); + throw caught; + } + // not suppressed by a higher prority error throw ee; + } else { + // when exit is enabled, whatever Throwable happened, we exit the VM + System.exit(status); } - System.exit(status); } /** - * Forcibly terminates the currently running Java virtual machine. - * The exception argument is rethrown if JVM halting is disabled. - * @param ee the exception containing the status code, message and any stack + * Halts the JVM if halt is enabled, rethrow provided exception or any raised error otherwise. + * If halt is disabled, this method throws either the exception argument if no + * error arise, the first error if at least one arise, suppressing he. + * If halt is enabled, all throwables are caught, even errors. + * + * @param he the exception containing the status code, message and any stack * trace. - * @throws HaltException if {@link Runtime#halt(int)} is disabled. + * @throws HaltException if {@link Runtime#halt(int)} is disabled and not suppressed by an Error + * @throws Error if {@link Runtime#halt(int)} is disabled and one Error arise, suppressing + * anyuthing else, even he */ - public static synchronized void halt(HaltException ee) throws HaltException { - int status = ee.getExitCode(); - String msg = ee.getMessage(); - try { - if (status != 0) { - //exit indicates a problem, log it - LOG.info("Halt with status {}: {}", status, msg, ee); + public static void halt(final HaltException he) throws HaltException { + final int status = he.getExitCode(); + Error caught = null; + if (status != 0) { + try { + // exit indicates a problem, log it + String msg = he.getMessage(); + LOG.info("Halt with status {}: {}", status, msg, he); + } catch (Error e) { + // errors have higher priority than HaltException, it may be re-thrown. + // OOM and ThreadDeath are 2 examples of Errors to re-throw + caught = e; + } catch (Throwable t) { + // all other kind of throwables are suppressed + addSuppressed(he, t); } - } catch (Exception ignored) { - // ignore exceptions here, as it may be due to an out of memory situation } + // systemHaltDisabled is volatile and not used in scenario nheding atomicty, + // thus it does not nhed a synchronized access nor a atomic access if (systemHaltDisabled) { - LOG.error("Halt called", ee); - if (!haltCalled()) { - firstHaltException = ee; + try { + LOG.error("Halt called", he); + } catch (Error e) { + // errors have higher priority again, if it's a 2nd error, the 1st one suprpesses it + caught = addSuppressed(caught, e); + } catch (Throwable t) { + // all other kind of throwables are suppressed + addSuppressed(he, t); } - throw ee; + FIRST_HALT_EXCEPTION.compareAndSet(null, he); + if (caught != null) { + caught.addSuppressed(he); + throw caught; + } + // not suppressed by a higher prority error + throw he; + } else { + // when halt is enabled, whatever Throwable happened, we halt the VM + Runtime.getRuntime().halt(status); } - Runtime.getRuntime().halt(status); } /** diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Shell.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Shell.java index bd6bcb08d9c6d..65978f3c5f59c 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Shell.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Shell.java @@ -60,7 +60,7 @@ public abstract class Shell { * {@value} */ private static final String WINDOWS_PROBLEMS = - "https://wiki.apache.org/hadoop/WindowsProblems"; + "https://cwiki.apache.org/confluence/display/HADOOP2/WindowsProblems"; /** * Name of the windows utils binary: {@value}. diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/functional/Function4RaisingIOE.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/functional/Function4RaisingIOE.java new file mode 100644 index 0000000000000..f0cd5c08c572b --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/functional/Function4RaisingIOE.java @@ -0,0 +1,43 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.util.functional; + +import java.io.IOException; + +/** + * Function of arity 4 which may raise an IOException. + * @param type of arg1. + * @param type of arg2. + * @param type of arg3. + * @param type of arg4. + * @param return type. + */ +public interface Function4RaisingIOE { + + /** + * Apply the function. + * @param i1 argument 1. + * @param i2 argument 2. + * @param i3 argument 3. + * @param i4 argument 4. + * @return return value. + * @throws IOException any IOE. + */ + R apply(I1 i1, I2 i2, I3 i3, I4 i4) throws IOException; +} diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/functional/TaskPool.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/functional/TaskPool.java index 0abaab211de04..c9e6d0b78ac11 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/functional/TaskPool.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/functional/TaskPool.java @@ -37,6 +37,7 @@ import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.fs.RemoteIterator; +import org.apache.hadoop.fs.statistics.IOStatisticsContext; import static java.util.Objects.requireNonNull; import static org.apache.hadoop.util.functional.RemoteIterators.remoteIteratorFromIterable; @@ -136,6 +137,15 @@ public static class Builder { private boolean stopAbortsOnFailure = false; private int sleepInterval = SLEEP_INTERVAL_AWAITING_COMPLETION; + /** + * IOStatisticsContext to switch to in all threads + * taking part in the commit operation. + * This ensures that the IOStatistics collected in the + * worker threads will be aggregated into the total statistics + * of the thread calling the committer commit/abort methods. + */ + private IOStatisticsContext ioStatisticsContext = null; + /** * Create the builder. * @param items items to process @@ -242,7 +252,7 @@ public Builder stopAbortsOnFailure() { * @param value new value * @return the builder */ - public Builder sleepInterval(final int value) { + public Builder sleepInterval(final int value) { sleepInterval = value; return this; } @@ -364,6 +374,8 @@ private boolean runSingleThreaded(Task task) /** * Parallel execution. + * All tasks run within the same IOStatisticsContext as the + * thread calling this method. * @param task task to execute * @param exception which may be raised in execution. * @return true if the operation executed successfully @@ -379,64 +391,70 @@ private boolean runParallel(final Task task) final AtomicBoolean revertFailed = new AtomicBoolean(false); List> futures = new ArrayList<>(); + ioStatisticsContext = IOStatisticsContext.getCurrentIOStatisticsContext(); IOException iteratorIOE = null; final RemoteIterator iterator = this.items; try { - while(iterator.hasNext()) { + while (iterator.hasNext()) { final I item = iterator.next(); // submit a task for each item that will either run or abort the task futures.add(service.submit(() -> { - if (!(stopOnFailure && taskFailed.get())) { - // run the task - boolean threw = true; - try { - LOG.debug("Executing task"); - task.run(item); - succeeded.add(item); - LOG.debug("Task succeeded"); - - threw = false; - - } catch (Exception e) { - taskFailed.set(true); - exceptions.add(e); - LOG.info("Task failed {}", e.toString()); - LOG.debug("Task failed", e); - - if (onFailure != null) { - try { - onFailure.run(item, e); - } catch (Exception failException) { - LOG.warn("Failed to clean up on failure", e); - // swallow the exception - } - } - } finally { - if (threw) { + setStatisticsContext(); + try { + if (!(stopOnFailure && taskFailed.get())) { + // prepare and run the task + boolean threw = true; + try { + LOG.debug("Executing task"); + task.run(item); + succeeded.add(item); + LOG.debug("Task succeeded"); + + threw = false; + + } catch (Exception e) { taskFailed.set(true); + exceptions.add(e); + LOG.info("Task failed {}", e.toString()); + LOG.debug("Task failed", e); + + if (onFailure != null) { + try { + onFailure.run(item, e); + } catch (Exception failException) { + LOG.warn("Failed to clean up on failure", e); + // swallow the exception + } + } + } finally { + if (threw) { + taskFailed.set(true); + } } - } - } else if (abortTask != null) { - // abort the task instead of running it - if (stopAbortsOnFailure && abortFailed.get()) { - return; - } + } else if (abortTask != null) { + // abort the task instead of running it + if (stopAbortsOnFailure && abortFailed.get()) { + return; + } - boolean failed = true; - try { - LOG.info("Aborting task"); - abortTask.run(item); - failed = false; - } catch (Exception e) { - LOG.error("Failed to abort task", e); - // swallow the exception - } finally { - if (failed) { - abortFailed.set(true); + boolean failed = true; + try { + LOG.info("Aborting task"); + abortTask.run(item); + failed = false; + } catch (Exception e) { + LOG.error("Failed to abort task", e); + // swallow the exception + } finally { + if (failed) { + abortFailed.set(true); + } } } + } finally { + resetStatisticsContext(); } })); } @@ -447,7 +465,6 @@ private boolean runParallel(final Task task) // mark as a task failure so all submitted tasks will halt/abort taskFailed.set(true); } - // let the above tasks complete (or abort) waitFor(futures, sleepInterval); int futureCount = futures.size(); @@ -464,6 +481,7 @@ private boolean runParallel(final Task task) } boolean failed = true; + setStatisticsContext(); try { revertTask.run(item); failed = false; @@ -474,6 +492,7 @@ private boolean runParallel(final Task task) if (failed) { revertFailed.set(true); } + resetStatisticsContext(); } })); } @@ -498,6 +517,26 @@ private boolean runParallel(final Task task) // return true if all tasks succeeded. return !taskFailed.get(); } + + /** + * Set the statistics context for this thread. + */ + private void setStatisticsContext() { + if (ioStatisticsContext != null) { + IOStatisticsContext.setThreadIOStatisticsContext(ioStatisticsContext); + } + } + + /** + * Reset the statistics context if it was set earlier. + * This unbinds the current thread from any statistics + * context. + */ + private void resetStatisticsContext() { + if (ioStatisticsContext != null) { + IOStatisticsContext.setThreadIOStatisticsContext(null); + } + } } /** diff --git a/hadoop-common-project/hadoop-common/src/main/proto/RpcHeader.proto b/hadoop-common-project/hadoop-common/src/main/proto/RpcHeader.proto index 042928c2aee18..d9becf722e982 100644 --- a/hadoop-common-project/hadoop-common/src/main/proto/RpcHeader.proto +++ b/hadoop-common-project/hadoop-common/src/main/proto/RpcHeader.proto @@ -91,6 +91,10 @@ message RpcRequestHeaderProto { // the header for the RpcRequest optional RPCTraceInfoProto traceInfo = 6; // tracing info optional RPCCallerContextProto callerContext = 7; // call context optional int64 stateId = 8; // The last seen Global State ID + // Alignment context info for use with routers. + // The client should not interpret these bytes, but only forward bytes + // received from RpcResponseHeaderProto.routerFederatedState. + optional bytes routerFederatedState = 9; } @@ -157,6 +161,10 @@ message RpcResponseHeaderProto { optional bytes clientId = 7; // Globally unique client ID optional sint32 retryCount = 8 [default = -1]; optional int64 stateId = 9; // The last written Global State ID + // Alignment context info for use with routers. + // The client should not interpret these bytes, but only + // forward them to the router using RpcRequestHeaderProto.routerFederatedState. + optional bytes routerFederatedState = 10; } message RpcSaslProto { diff --git a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml index 5a1c09f0141ed..17cd228dc1ba0 100644 --- a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml +++ b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml @@ -130,6 +130,14 @@ + + hadoop.security.resolver.impl + org.apache.hadoop.net.DNSDomainNameResolver + + The resolver implementation used to resolve FQDN for Kerberos + + + hadoop.security.dns.log-slow-lookups.enabled false @@ -577,6 +585,18 @@ + + hadoop.security.group.mapping.ldap.group.search.filter.pattern + + + Comma separated values that needs to be substituted in the group search + filter during group lookup. The values are substituted in the order they + appear in the list, the first value will replace {0} the second {1} and + so on. + + + + hadoop.security.group.mapping.providers diff --git a/hadoop-common-project/hadoop-common/src/site/markdown/GroupsMapping.md b/hadoop-common-project/hadoop-common/src/site/markdown/GroupsMapping.md index 03759d80092cd..cd6e6fecb1389 100644 --- a/hadoop-common-project/hadoop-common/src/site/markdown/GroupsMapping.md +++ b/hadoop-common-project/hadoop-common/src/site/markdown/GroupsMapping.md @@ -85,6 +85,14 @@ This is the limit for each ldap query. If `hadoop.security.group.mapping.ldap.s `hadoop.security.group.mapping.ldap.base` configures how far to walk up the groups hierarchy when resolving groups. By default, with a limit of 0, in order to be considered a member of a group, the user must be an explicit member in LDAP. Otherwise, it will traverse the group hierarchy `hadoop.security.group.mapping.ldap.search.group.hierarchy.levels` levels up. +It is possible to have custom group search filters with different arguments using +the configuration `hadoop.security.group.mapping.ldap.group.search.filter.pattern`, we can configure comma separated values here and the values configured will be fetched from the LDAP attributes and will be replaced in the group +search filter in the order they appear here, say if the first entry here is uid, so uid will be fetched from the attributes and the value fetched +will be used in place of {0} in the group search filter, similarly the second value configured will replace {1} and so on. + +Note: If `hadoop.security.group.mapping.ldap.group.search.filter.pattern` is configured, the group search will always be done assuming this group +search filter pattern irrespective of any other parameters. + ### Bind user(s) ### If the LDAP server does not support anonymous binds, set the distinguished name of the user to bind in `hadoop.security.group.mapping.ldap.bind.user`. diff --git a/hadoop-common-project/hadoop-common/src/site/markdown/filesystem/fsdatainputstream.md b/hadoop-common-project/hadoop-common/src/site/markdown/filesystem/fsdatainputstream.md index 197b999c81f66..f64a2bd03b63b 100644 --- a/hadoop-common-project/hadoop-common/src/site/markdown/filesystem/fsdatainputstream.md +++ b/hadoop-common-project/hadoop-common/src/site/markdown/filesystem/fsdatainputstream.md @@ -454,6 +454,13 @@ Also, clients are encouraged to use `WeakReferencedElasticByteBufferPool` for allocating buffers such that even direct buffers are garbage collected when they are no longer referenced. +The position returned by `getPos()` after `readVectored()` is undefined. + +If a file is changed while the `readVectored()` operation is in progress, the output is +undefined. Some ranges may have old data, some may have new, and some may have both. + +While a `readVectored()` operation is in progress, normal read api calls may block. + Note: Don't use direct buffers for reading from ChecksumFileSystem as that may lead to memory fragmentation explained in HADOOP-18296. diff --git a/hadoop-common-project/hadoop-common/src/site/markdown/release/3.2.4/CHANGELOG.3.2.4.md b/hadoop-common-project/hadoop-common/src/site/markdown/release/3.2.4/CHANGELOG.3.2.4.md new file mode 100644 index 0000000000000..fc0079d1c9bd8 --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/site/markdown/release/3.2.4/CHANGELOG.3.2.4.md @@ -0,0 +1,213 @@ + + +# Apache Hadoop Changelog + +## Release 3.2.4 - 2022-07-12 + + + +### NEW FEATURES: + +| JIRA | Summary | Priority | Component | Reporter | Contributor | +|:---- |:---- | :--- |:---- |:---- |:---- | +| [HDFS-16337](https://issues.apache.org/jira/browse/HDFS-16337) | Show start time of Datanode on Web | Minor | . | Tao Li | Tao Li | + + +### IMPROVEMENTS: + +| JIRA | Summary | Priority | Component | Reporter | Contributor | +|:---- |:---- | :--- |:---- |:---- |:---- | +| [HDFS-15075](https://issues.apache.org/jira/browse/HDFS-15075) | Remove process command timing from BPServiceActor | Major | . | Íñigo Goiri | Xiaoqiao He | +| [HDFS-15150](https://issues.apache.org/jira/browse/HDFS-15150) | Introduce read write lock to Datanode | Major | datanode | Stephen O'Donnell | Stephen O'Donnell | +| [HDFS-16175](https://issues.apache.org/jira/browse/HDFS-16175) | Improve the configurable value of Server #PURGE\_INTERVAL\_NANOS | Major | ipc | JiangHua Zhu | JiangHua Zhu | +| [HDFS-16173](https://issues.apache.org/jira/browse/HDFS-16173) | Improve CopyCommands#Put#executor queue configurability | Major | fs | JiangHua Zhu | JiangHua Zhu | +| [HADOOP-17897](https://issues.apache.org/jira/browse/HADOOP-17897) | Allow nested blocks in switch case in checkstyle settings | Minor | build | Masatake Iwasaki | Masatake Iwasaki | +| [HADOOP-17857](https://issues.apache.org/jira/browse/HADOOP-17857) | Check real user ACLs in addition to proxied user ACLs | Major | . | Eric Payne | Eric Payne | +| [HDFS-14997](https://issues.apache.org/jira/browse/HDFS-14997) | BPServiceActor processes commands from NameNode asynchronously | Major | datanode | Xiaoqiao He | Xiaoqiao He | +| [HADOOP-17926](https://issues.apache.org/jira/browse/HADOOP-17926) | Maven-eclipse-plugin is no longer needed since Eclipse can import Maven projects by itself. | Minor | documentation | Rintaro Ikeda | Rintaro Ikeda | +| [YARN-10935](https://issues.apache.org/jira/browse/YARN-10935) | AM Total Queue Limit goes below per-user AM Limit if parent is full. | Major | capacity scheduler, capacityscheduler | Eric Payne | Eric Payne | +| [HDFS-16241](https://issues.apache.org/jira/browse/HDFS-16241) | Standby close reconstruction thread | Major | . | zhanghuazong | zhanghuazong | +| [YARN-1115](https://issues.apache.org/jira/browse/YARN-1115) | Provide optional means for a scheduler to check real user ACLs | Major | capacity scheduler, scheduler | Eric Payne | | +| [HDFS-16279](https://issues.apache.org/jira/browse/HDFS-16279) | Print detail datanode info when process first storage report | Minor | . | Tao Li | Tao Li | +| [HDFS-16294](https://issues.apache.org/jira/browse/HDFS-16294) | Remove invalid DataNode#CONFIG\_PROPERTY\_SIMULATED | Major | datanode | JiangHua Zhu | JiangHua Zhu | +| [HDFS-16299](https://issues.apache.org/jira/browse/HDFS-16299) | Fix bug for TestDataNodeVolumeMetrics#verifyDataNodeVolumeMetrics | Minor | . | Tao Li | Tao Li | +| [HDFS-16301](https://issues.apache.org/jira/browse/HDFS-16301) | Improve BenchmarkThroughput#SIZE naming standardization | Minor | benchmarks, test | JiangHua Zhu | JiangHua Zhu | +| [YARN-10997](https://issues.apache.org/jira/browse/YARN-10997) | Revisit allocation and reservation logging | Major | . | Andras Gyori | Andras Gyori | +| [HDFS-16315](https://issues.apache.org/jira/browse/HDFS-16315) | Add metrics related to Transfer and NativeCopy for DataNode | Major | . | Tao Li | Tao Li | +| [HADOOP-17998](https://issues.apache.org/jira/browse/HADOOP-17998) | Allow get command to run with multi threads. | Major | fs | Chengwei Wang | Chengwei Wang | +| [HDFS-16345](https://issues.apache.org/jira/browse/HDFS-16345) | Fix test cases fail in TestBlockStoragePolicy | Major | build | guophilipse | guophilipse | +| [HADOOP-18035](https://issues.apache.org/jira/browse/HADOOP-18035) | Skip unit test failures to run all the unit tests | Major | build | Akira Ajisaka | Akira Ajisaka | +| [HADOOP-18040](https://issues.apache.org/jira/browse/HADOOP-18040) | Use maven.test.failure.ignore instead of ignoreTestFailure | Major | build | Akira Ajisaka | Akira Ajisaka | +| [HDFS-16352](https://issues.apache.org/jira/browse/HDFS-16352) | return the real datanode numBlocks in #getDatanodeStorageReport | Major | . | qinyuren | qinyuren | +| [HDFS-16386](https://issues.apache.org/jira/browse/HDFS-16386) | Reduce DataNode load when FsDatasetAsyncDiskService is working | Major | datanode | JiangHua Zhu | JiangHua Zhu | +| [HDFS-16391](https://issues.apache.org/jira/browse/HDFS-16391) | Avoid evaluation of LOG.debug statement in NameNodeHeartbeatService | Trivial | . | wangzhaohui | wangzhaohui | +| [YARN-8234](https://issues.apache.org/jira/browse/YARN-8234) | Improve RM system metrics publisher's performance by pushing events to timeline server in batch | Critical | resourcemanager, timelineserver | Hu Ziqian | Ashutosh Gupta | +| [HDFS-16430](https://issues.apache.org/jira/browse/HDFS-16430) | Validate maximum blocks in EC group when adding an EC policy | Minor | ec, erasure-coding | daimin | daimin | +| [HDFS-16403](https://issues.apache.org/jira/browse/HDFS-16403) | Improve FUSE IO performance by supporting FUSE parameter max\_background | Minor | fuse-dfs | daimin | daimin | +| [HADOOP-18136](https://issues.apache.org/jira/browse/HADOOP-18136) | Verify FileUtils.unTar() handling of missing .tar files | Minor | test, util | Steve Loughran | Steve Loughran | +| [HDFS-16529](https://issues.apache.org/jira/browse/HDFS-16529) | Remove unnecessary setObserverRead in TestConsistentReadsObserver | Trivial | test | wangzhaohui | wangzhaohui | +| [HDFS-16530](https://issues.apache.org/jira/browse/HDFS-16530) | setReplication debug log creates a new string even if debug is disabled | Major | namenode | Stephen O'Donnell | Stephen O'Donnell | +| [HDFS-16427](https://issues.apache.org/jira/browse/HDFS-16427) | Add debug log for BlockManager#chooseExcessRedundancyStriped | Minor | erasure-coding | Tao Li | Tao Li | +| [HDFS-16389](https://issues.apache.org/jira/browse/HDFS-16389) | Improve NNThroughputBenchmark test mkdirs | Major | benchmarks, namenode | JiangHua Zhu | JiangHua Zhu | +| [MAPREDUCE-7373](https://issues.apache.org/jira/browse/MAPREDUCE-7373) | Building MapReduce NativeTask fails on Fedora 34+ | Major | build, nativetask | Kengo Seki | Kengo Seki | +| [HDFS-16355](https://issues.apache.org/jira/browse/HDFS-16355) | Improve the description of dfs.block.scanner.volume.bytes.per.second | Minor | documentation, hdfs | guophilipse | guophilipse | +| [HADOOP-18088](https://issues.apache.org/jira/browse/HADOOP-18088) | Replace log4j 1.x with reload4j | Major | . | Wei-Chiu Chuang | Wei-Chiu Chuang | +| [HDFS-16501](https://issues.apache.org/jira/browse/HDFS-16501) | Print the exception when reporting a bad block | Major | datanode | qinyuren | qinyuren | +| [YARN-11116](https://issues.apache.org/jira/browse/YARN-11116) | Migrate Times util from SimpleDateFormat to thread-safe DateTimeFormatter class | Minor | . | Jonathan Turner Eagles | Jonathan Turner Eagles | +| [YARN-10080](https://issues.apache.org/jira/browse/YARN-10080) | Support show app id on localizer thread pool | Major | nodemanager | zhoukang | Ashutosh Gupta | +| [HADOOP-18240](https://issues.apache.org/jira/browse/HADOOP-18240) | Upgrade Yetus to 0.14.0 | Major | build | Akira Ajisaka | Ashutosh Gupta | +| [HDFS-16585](https://issues.apache.org/jira/browse/HDFS-16585) | Add @VisibleForTesting in Dispatcher.java after HDFS-16268 | Trivial | . | Wei-Chiu Chuang | Ashutosh Gupta | +| [HDFS-16610](https://issues.apache.org/jira/browse/HDFS-16610) | Make fsck read timeout configurable | Major | hdfs-client | Stephen O'Donnell | Stephen O'Donnell | + + +### BUG FIXES: + +| JIRA | Summary | Priority | Component | Reporter | Contributor | +|:---- |:---- | :--- |:---- |:---- |:---- | +| [HDFS-13983](https://issues.apache.org/jira/browse/HDFS-13983) | TestOfflineImageViewer crashes in windows | Major | . | Vinayakumar B | Vinayakumar B | +| [YARN-9744](https://issues.apache.org/jira/browse/YARN-9744) | RollingLevelDBTimelineStore.getEntityByTime fails with NPE | Major | timelineserver | Prabhu Joseph | Prabhu Joseph | +| [HDFS-15113](https://issues.apache.org/jira/browse/HDFS-15113) | Missing IBR when NameNode restart if open processCommand async feature | Blocker | datanode | Xiaoqiao He | Xiaoqiao He | +| [HADOOP-16985](https://issues.apache.org/jira/browse/HADOOP-16985) | Handle release package related issues | Major | . | Vinayakumar B | Vinayakumar B | +| [HADOOP-17116](https://issues.apache.org/jira/browse/HADOOP-17116) | Skip Retry INFO logging on first failover from a proxy | Major | ha | Hanisha Koneru | Hanisha Koneru | +| [HDFS-15651](https://issues.apache.org/jira/browse/HDFS-15651) | Client could not obtain block when DN CommandProcessingThread exit | Major | . | Yiqun Lin | Mingxiang Li | +| [HDFS-15963](https://issues.apache.org/jira/browse/HDFS-15963) | Unreleased volume references cause an infinite loop | Critical | datanode | Shuyan Zhang | Shuyan Zhang | +| [HDFS-14575](https://issues.apache.org/jira/browse/HDFS-14575) | LeaseRenewer#daemon threads leak in DFSClient | Major | . | Tao Yang | Renukaprasad C | +| [HADOOP-17796](https://issues.apache.org/jira/browse/HADOOP-17796) | Upgrade jetty version to 9.4.43 | Major | . | Wei-Chiu Chuang | Renukaprasad C | +| [HDFS-15175](https://issues.apache.org/jira/browse/HDFS-15175) | Multiple CloseOp shared block instance causes the standby namenode to crash when rolling editlog | Critical | . | Yicong Cai | Wan Chang | +| [HDFS-16177](https://issues.apache.org/jira/browse/HDFS-16177) | Bug fix for Util#receiveFile | Minor | . | Tao Li | Tao Li | +| [YARN-10814](https://issues.apache.org/jira/browse/YARN-10814) | YARN shouldn't start with empty hadoop.http.authentication.signature.secret.file | Major | . | Benjamin Teke | Tamas Domok | +| [HADOOP-17874](https://issues.apache.org/jira/browse/HADOOP-17874) | ExceptionsHandler to add terse/suppressed Exceptions in thread-safe manner | Major | . | Viraj Jasani | Viraj Jasani | +| [HADOOP-15129](https://issues.apache.org/jira/browse/HADOOP-15129) | Datanode caches namenode DNS lookup failure and cannot startup | Minor | ipc | Karthik Palaniappan | Chris Nauroth | +| [YARN-10901](https://issues.apache.org/jira/browse/YARN-10901) | Permission checking error on an existing directory in LogAggregationFileController#verifyAndCreateRemoteLogDir | Major | nodemanager | Tamas Domok | Tamas Domok | +| [HDFS-16207](https://issues.apache.org/jira/browse/HDFS-16207) | Remove NN logs stack trace for non-existent xattr query | Major | namenode | Ahmed Hussein | Ahmed Hussein | +| [HDFS-16187](https://issues.apache.org/jira/browse/HDFS-16187) | SnapshotDiff behaviour with Xattrs and Acls is not consistent across NN restarts with checkpointing | Major | snapshots | Srinivasu Majeti | Shashikant Banerjee | +| [HDFS-16198](https://issues.apache.org/jira/browse/HDFS-16198) | Short circuit read leaks Slot objects when InvalidToken exception is thrown | Major | . | Eungsop Yoo | Eungsop Yoo | +| [YARN-10870](https://issues.apache.org/jira/browse/YARN-10870) | Missing user filtering check -\> yarn.webapp.filter-entity-list-by-user for RM Scheduler page | Major | yarn | Siddharth Ahuja | Gergely Pollák | +| [HADOOP-17919](https://issues.apache.org/jira/browse/HADOOP-17919) | Fix command line example in Hadoop Cluster Setup documentation | Minor | documentation | Rintaro Ikeda | Rintaro Ikeda | +| [HDFS-16235](https://issues.apache.org/jira/browse/HDFS-16235) | Deadlock in LeaseRenewer for static remove method | Major | hdfs | angerszhu | angerszhu | +| [HDFS-16181](https://issues.apache.org/jira/browse/HDFS-16181) | [SBN Read] Fix metric of RpcRequestCacheMissAmount can't display when tailEditLog form JN | Critical | . | wangzhaohui | wangzhaohui | +| [HADOOP-17925](https://issues.apache.org/jira/browse/HADOOP-17925) | BUILDING.txt should not encourage to activate docs profile on building binary artifacts | Minor | documentation | Rintaro Ikeda | Masatake Iwasaki | +| [HADOOP-16532](https://issues.apache.org/jira/browse/HADOOP-16532) | Fix TestViewFsTrash to use the correct homeDir. | Minor | test, viewfs | Steve Loughran | Xing Lin | +| [HDFS-16268](https://issues.apache.org/jira/browse/HDFS-16268) | Balancer stuck when moving striped blocks due to NPE | Major | balancer & mover, erasure-coding | Leon Gao | Leon Gao | +| [HDFS-7612](https://issues.apache.org/jira/browse/HDFS-7612) | TestOfflineEditsViewer.testStored() uses incorrect default value for cacheDir | Major | test | Konstantin Shvachko | Michael Kuchenbecker | +| [HDFS-16311](https://issues.apache.org/jira/browse/HDFS-16311) | Metric metadataOperationRate calculation error in DataNodeVolumeMetrics | Major | . | Tao Li | Tao Li | +| [HDFS-16182](https://issues.apache.org/jira/browse/HDFS-16182) | numOfReplicas is given the wrong value in BlockPlacementPolicyDefault$chooseTarget can cause DataStreamer to fail with Heterogeneous Storage | Major | namanode | Max Xie | Max Xie | +| [HADOOP-17999](https://issues.apache.org/jira/browse/HADOOP-17999) | No-op implementation of setWriteChecksum and setVerifyChecksum in ViewFileSystem | Major | . | Abhishek Das | Abhishek Das | +| [HDFS-16329](https://issues.apache.org/jira/browse/HDFS-16329) | Fix log format for BlockManager | Minor | . | Tao Li | Tao Li | +| [HDFS-16330](https://issues.apache.org/jira/browse/HDFS-16330) | Fix incorrect placeholder for Exception logs in DiskBalancer | Major | . | Viraj Jasani | Viraj Jasani | +| [HDFS-16328](https://issues.apache.org/jira/browse/HDFS-16328) | Correct disk balancer param desc | Minor | documentation, hdfs | guophilipse | guophilipse | +| [HDFS-16343](https://issues.apache.org/jira/browse/HDFS-16343) | Add some debug logs when the dfsUsed are not used during Datanode startup | Major | datanode | Mukul Kumar Singh | Mukul Kumar Singh | +| [YARN-10991](https://issues.apache.org/jira/browse/YARN-10991) | Fix to ignore the grouping "[]" for resourcesStr in parseResourcesString method | Minor | distributed-shell | Ashutosh Gupta | Ashutosh Gupta | +| [HADOOP-17975](https://issues.apache.org/jira/browse/HADOOP-17975) | Fallback to simple auth does not work for a secondary DistributedFileSystem instance | Major | ipc | István Fajth | István Fajth | +| [HDFS-16350](https://issues.apache.org/jira/browse/HDFS-16350) | Datanode start time should be set after RPC server starts successfully | Minor | . | Viraj Jasani | Viraj Jasani | +| [YARN-11007](https://issues.apache.org/jira/browse/YARN-11007) | Correct words in YARN documents | Minor | documentation | guophilipse | guophilipse | +| [HDFS-16332](https://issues.apache.org/jira/browse/HDFS-16332) | Expired block token causes slow read due to missing handling in sasl handshake | Major | datanode, dfs, dfsclient | Shinya Yoshida | Shinya Yoshida | +| [YARN-9063](https://issues.apache.org/jira/browse/YARN-9063) | ATS 1.5 fails to start if RollingLevelDb files are corrupt or missing | Major | timelineserver, timelineservice | Tarun Parimi | Ashutosh Gupta | +| [HDFS-16333](https://issues.apache.org/jira/browse/HDFS-16333) | fix balancer bug when transfer an EC block | Major | balancer & mover, erasure-coding | qinyuren | qinyuren | +| [HDFS-16373](https://issues.apache.org/jira/browse/HDFS-16373) | Fix MiniDFSCluster restart in case of multiple namenodes | Major | . | Ayush Saxena | Ayush Saxena | +| [HDFS-16377](https://issues.apache.org/jira/browse/HDFS-16377) | Should CheckNotNull before access FsDatasetSpi | Major | . | Tao Li | Tao Li | +| [YARN-6862](https://issues.apache.org/jira/browse/YARN-6862) | Nodemanager resource usage metrics sometimes are negative | Major | nodemanager | YunFan Zhou | Benjamin Teke | +| [YARN-10178](https://issues.apache.org/jira/browse/YARN-10178) | Global Scheduler async thread crash caused by 'Comparison method violates its general contract | Major | capacity scheduler | tuyu | Andras Gyori | +| [HDFS-16395](https://issues.apache.org/jira/browse/HDFS-16395) | Remove useless NNThroughputBenchmark#dummyActionNoSynch() | Major | benchmarks, namenode | JiangHua Zhu | JiangHua Zhu | +| [HADOOP-18063](https://issues.apache.org/jira/browse/HADOOP-18063) | Remove unused import AbstractJavaKeyStoreProvider in Shell class | Minor | . | JiangHua Zhu | JiangHua Zhu | +| [HDFS-16409](https://issues.apache.org/jira/browse/HDFS-16409) | Fix typo: testHasExeceptionsReturnsCorrectValue -\> testHasExceptionsReturnsCorrectValue | Trivial | . | Ashutosh Gupta | Ashutosh Gupta | +| [HDFS-16408](https://issues.apache.org/jira/browse/HDFS-16408) | Ensure LeaseRecheckIntervalMs is greater than zero | Major | namenode | Jingxuan Fu | Jingxuan Fu | +| [YARN-11055](https://issues.apache.org/jira/browse/YARN-11055) | In cgroups-operations.c some fprintf format strings don't end with "\\n" | Minor | nodemanager | Gera Shegalov | Gera Shegalov | +| [HDFS-16303](https://issues.apache.org/jira/browse/HDFS-16303) | Losing over 100 datanodes in state decommissioning results in full blockage of all datanode decommissioning | Major | . | Kevin Wikant | Kevin Wikant | +| [HDFS-16443](https://issues.apache.org/jira/browse/HDFS-16443) | Fix edge case where DatanodeAdminDefaultMonitor doubly enqueues a DatanodeDescriptor on exception | Major | hdfs | Kevin Wikant | Kevin Wikant | +| [HDFS-16449](https://issues.apache.org/jira/browse/HDFS-16449) | Fix hadoop web site release notes and changelog not available | Minor | documentation | guophilipse | guophilipse | +| [HADOOP-18192](https://issues.apache.org/jira/browse/HADOOP-18192) | Fix multiple\_bindings warning about slf4j-reload4j | Major | . | Masatake Iwasaki | Masatake Iwasaki | +| [HDFS-16479](https://issues.apache.org/jira/browse/HDFS-16479) | EC: NameNode should not send a reconstruction work when the source datanodes are insufficient | Critical | ec, erasure-coding | Yuanbo Liu | Takanobu Asanuma | +| [HDFS-16509](https://issues.apache.org/jira/browse/HDFS-16509) | Fix decommission UnsupportedOperationException: Remove unsupported | Major | namenode | daimin | daimin | +| [HDFS-16456](https://issues.apache.org/jira/browse/HDFS-16456) | EC: Decommission a rack with only on dn will fail when the rack number is equal with replication | Critical | ec, namenode | caozhiqiang | caozhiqiang | +| [HDFS-16437](https://issues.apache.org/jira/browse/HDFS-16437) | ReverseXML processor doesn't accept XML files without the SnapshotDiffSection. | Critical | hdfs | yanbin.zhang | yanbin.zhang | +| [HDFS-16507](https://issues.apache.org/jira/browse/HDFS-16507) | [SBN read] Avoid purging edit log which is in progress | Critical | . | Tao Li | Tao Li | +| [YARN-10720](https://issues.apache.org/jira/browse/YARN-10720) | YARN WebAppProxyServlet should support connection timeout to prevent proxy server from hanging | Critical | . | Qi Zhu | Qi Zhu | +| [HDFS-16428](https://issues.apache.org/jira/browse/HDFS-16428) | Source path with storagePolicy cause wrong typeConsumed while rename | Major | hdfs, namenode | lei w | lei w | +| [YARN-11014](https://issues.apache.org/jira/browse/YARN-11014) | YARN incorrectly validates maximum capacity resources on the validation API | Major | . | Benjamin Teke | Benjamin Teke | +| [YARN-11075](https://issues.apache.org/jira/browse/YARN-11075) | Explicitly declare serialVersionUID in LogMutation class | Major | . | Benjamin Teke | Benjamin Teke | +| [HDFS-11041](https://issues.apache.org/jira/browse/HDFS-11041) | Unable to unregister FsDatasetState MBean if DataNode is shutdown twice | Trivial | datanode | Wei-Chiu Chuang | Wei-Chiu Chuang | +| [HDFS-16538](https://issues.apache.org/jira/browse/HDFS-16538) | EC decoding failed due to not enough valid inputs | Major | erasure-coding | qinyuren | qinyuren | +| [HDFS-16544](https://issues.apache.org/jira/browse/HDFS-16544) | EC decoding failed due to invalid buffer | Major | erasure-coding | qinyuren | qinyuren | +| [HDFS-16546](https://issues.apache.org/jira/browse/HDFS-16546) | Fix UT TestOfflineImageViewer#testReverseXmlWithoutSnapshotDiffSection to branch branch-3.2 | Major | test | daimin | daimin | +| [HDFS-16552](https://issues.apache.org/jira/browse/HDFS-16552) | Fix NPE for TestBlockManager | Major | . | Tao Li | Tao Li | +| [MAPREDUCE-7246](https://issues.apache.org/jira/browse/MAPREDUCE-7246) | In MapredAppMasterRest#Mapreduce\_Application\_Master\_Info\_API, the datatype of appId should be "string". | Major | documentation | jenny | Ashutosh Gupta | +| [YARN-10187](https://issues.apache.org/jira/browse/YARN-10187) | Removing hadoop-yarn-project/hadoop-yarn/README as it is no longer maintained. | Minor | documentation | N Sanketh Reddy | Ashutosh Gupta | +| [HDFS-16185](https://issues.apache.org/jira/browse/HDFS-16185) | Fix comment in LowRedundancyBlocks.java | Minor | documentation | Akira Ajisaka | Ashutosh Gupta | +| [HADOOP-17479](https://issues.apache.org/jira/browse/HADOOP-17479) | Fix the examples of hadoop config prefix | Minor | documentation | Akira Ajisaka | Ashutosh Gupta | +| [HDFS-16579](https://issues.apache.org/jira/browse/HDFS-16579) | Fix build failure for TestBlockManager on branch-3.2 | Major | . | Tao Li | Tao Li | +| [YARN-11092](https://issues.apache.org/jira/browse/YARN-11092) | Upgrade jquery ui to 1.13.1 | Major | . | D M Murali Krishna Reddy | Ashutosh Gupta | +| [YARN-11133](https://issues.apache.org/jira/browse/YARN-11133) | YarnClient gets the wrong EffectiveMinCapacity value | Major | api | Zilong Zhu | Zilong Zhu | +| [YARN-10850](https://issues.apache.org/jira/browse/YARN-10850) | TimelineService v2 lists containers for all attempts when filtering for one | Major | timelinereader | Benjamin Teke | Benjamin Teke | +| [YARN-11126](https://issues.apache.org/jira/browse/YARN-11126) | ZKConfigurationStore Java deserialisation vulnerability | Major | yarn | Tamas Domok | Tamas Domok | +| [YARN-11162](https://issues.apache.org/jira/browse/YARN-11162) | Set the zk acl for nodes created by ZKConfigurationStore. | Major | resourcemanager | Owen O'Malley | Owen O'Malley | +| [HDFS-16586](https://issues.apache.org/jira/browse/HDFS-16586) | Purge FsDatasetAsyncDiskService threadgroup; it causes BPServiceActor$CommandProcessingThread IllegalThreadStateException 'fatal exception and exit' | Major | datanode | Michael Stack | Michael Stack | +| [HADOOP-18251](https://issues.apache.org/jira/browse/HADOOP-18251) | Fix failure of extracting JIRA id from commit message in git\_jira\_fix\_version\_check.py | Minor | build | Masatake Iwasaki | Masatake Iwasaki | +| [HDFS-16583](https://issues.apache.org/jira/browse/HDFS-16583) | DatanodeAdminDefaultMonitor can get stuck in an infinite loop | Major | . | Stephen O'Donnell | Stephen O'Donnell | +| [HDFS-16623](https://issues.apache.org/jira/browse/HDFS-16623) | IllegalArgumentException in LifelineSender | Major | . | ZanderXu | ZanderXu | +| [HDFS-16064](https://issues.apache.org/jira/browse/HDFS-16064) | Determine when to invalidate corrupt replicas based on number of usable replicas | Major | datanode, namenode | Kevin Wikant | Kevin Wikant | +| [HADOOP-18100](https://issues.apache.org/jira/browse/HADOOP-18100) | Change scope of inner classes in InodeTree to make them accessible outside package | Major | . | Abhishek Das | Abhishek Das | +| [HADOOP-18334](https://issues.apache.org/jira/browse/HADOOP-18334) | Fix create-release to address removal of GPG\_AGENT\_INFO in branch-3.2 | Major | build | Masatake Iwasaki | Masatake Iwasaki | + + +### TESTS: + +| JIRA | Summary | Priority | Component | Reporter | Contributor | +|:---- |:---- | :--- |:---- |:---- |:---- | +| [MAPREDUCE-7342](https://issues.apache.org/jira/browse/MAPREDUCE-7342) | Stop RMService in TestClientRedirect.testRedirect() | Minor | . | Zhengxi Li | Zhengxi Li | +| [MAPREDUCE-7311](https://issues.apache.org/jira/browse/MAPREDUCE-7311) | Fix non-idempotent test in TestTaskProgressReporter | Minor | . | Zhengxi Li | Zhengxi Li | +| [HDFS-15862](https://issues.apache.org/jira/browse/HDFS-15862) | Make TestViewfsWithNfs3.testNfsRenameSingleNN() idempotent | Minor | nfs | Zhengxi Li | Zhengxi Li | + + +### SUB-TASKS: + +| JIRA | Summary | Priority | Component | Reporter | Contributor | +|:---- |:---- | :--- |:---- |:---- |:---- | +| [HDFS-15457](https://issues.apache.org/jira/browse/HDFS-15457) | TestFsDatasetImpl fails intermittently | Major | hdfs | Ahmed Hussein | Ahmed Hussein | +| [HDFS-15818](https://issues.apache.org/jira/browse/HDFS-15818) | Fix TestFsDatasetImpl.testReadLockCanBeDisabledByConfig | Minor | test | Leon Gao | Leon Gao | +| [YARN-10503](https://issues.apache.org/jira/browse/YARN-10503) | Support queue capacity in terms of absolute resources with custom resourceType. | Critical | . | Qi Zhu | Qi Zhu | +| [HADOOP-17126](https://issues.apache.org/jira/browse/HADOOP-17126) | implement non-guava Precondition checkNotNull | Major | . | Ahmed Hussein | Ahmed Hussein | +| [HADOOP-17929](https://issues.apache.org/jira/browse/HADOOP-17929) | implement non-guava Precondition checkArgument | Major | . | Ahmed Hussein | Ahmed Hussein | +| [HADOOP-17947](https://issues.apache.org/jira/browse/HADOOP-17947) | Provide alternative to Guava VisibleForTesting | Major | . | Viraj Jasani | Viraj Jasani | +| [HADOOP-17930](https://issues.apache.org/jira/browse/HADOOP-17930) | implement non-guava Precondition checkState | Major | . | Ahmed Hussein | Ahmed Hussein | +| [HADOOP-17374](https://issues.apache.org/jira/browse/HADOOP-17374) | AliyunOSS: support ListObjectsV2 | Major | fs/oss | wujinhu | wujinhu | +| [HDFS-16336](https://issues.apache.org/jira/browse/HDFS-16336) | De-flake TestRollingUpgrade#testRollback | Minor | hdfs, test | Kevin Wikant | Viraj Jasani | +| [HDFS-16171](https://issues.apache.org/jira/browse/HDFS-16171) | De-flake testDecommissionStatus | Major | . | Viraj Jasani | Viraj Jasani | +| [HDFS-16169](https://issues.apache.org/jira/browse/HDFS-16169) | Fix TestBlockTokenWithDFSStriped#testEnd2End failure | Major | test | Hui Fei | secfree | +| [HDFS-16484](https://issues.apache.org/jira/browse/HDFS-16484) | [SPS]: Fix an infinite loop bug in SPSPathIdProcessor thread | Major | . | qinyuren | qinyuren | +| [HADOOP-16663](https://issues.apache.org/jira/browse/HADOOP-16663) | Backport "HADOOP-16560 [YARN] use protobuf-maven-plugin to generate protobuf classes" to all active branches | Major | . | Duo Zhang | Duo Zhang | +| [HADOOP-16664](https://issues.apache.org/jira/browse/HADOOP-16664) | Backport "HADOOP-16561 [MAPREDUCE] use protobuf-maven-plugin to generate protobuf classes" to all active branches | Major | . | Duo Zhang | Duo Zhang | + + +### OTHER: + +| JIRA | Summary | Priority | Component | Reporter | Contributor | +|:---- |:---- | :--- |:---- |:---- |:---- | +| [HDFS-16298](https://issues.apache.org/jira/browse/HDFS-16298) | Improve error msg for BlockMissingException | Minor | . | Tao Li | Tao Li | +| [HDFS-16312](https://issues.apache.org/jira/browse/HDFS-16312) | Fix typo for DataNodeVolumeMetrics and ProfilingFileIoEvents | Minor | . | Tao Li | Tao Li | +| [HDFS-16326](https://issues.apache.org/jira/browse/HDFS-16326) | Simplify the code for DiskBalancer | Minor | . | Tao Li | Tao Li | +| [HDFS-16339](https://issues.apache.org/jira/browse/HDFS-16339) | Show the threshold when mover threads quota is exceeded | Minor | . | Tao Li | Tao Li | +| [YARN-10820](https://issues.apache.org/jira/browse/YARN-10820) | Make GetClusterNodesRequestPBImpl thread safe | Major | client | Prabhu Joseph | SwathiChandrashekar | +| [HADOOP-13464](https://issues.apache.org/jira/browse/HADOOP-13464) | update GSON to 2.7+ | Minor | build | Sean Busbey | Igor Dvorzhak | +| [HADOOP-18191](https://issues.apache.org/jira/browse/HADOOP-18191) | Log retry count while handling exceptions in RetryInvocationHandler | Minor | . | Viraj Jasani | Viraj Jasani | +| [HDFS-16551](https://issues.apache.org/jira/browse/HDFS-16551) | Backport HADOOP-17588 to 3.3 and other active old branches. | Major | . | Renukaprasad C | Renukaprasad C | +| [HDFS-16618](https://issues.apache.org/jira/browse/HDFS-16618) | sync\_file\_range error should include more volume and file info | Minor | . | Viraj Jasani | Viraj Jasani | +| [HADOOP-18300](https://issues.apache.org/jira/browse/HADOOP-18300) | Update Gson to 2.9.0 | Minor | build | Igor Dvorzhak | Igor Dvorzhak | + + diff --git a/hadoop-common-project/hadoop-common/src/site/markdown/release/3.2.4/RELEASENOTES.3.2.4.md b/hadoop-common-project/hadoop-common/src/site/markdown/release/3.2.4/RELEASENOTES.3.2.4.md new file mode 100644 index 0000000000000..fac976d655da1 --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/site/markdown/release/3.2.4/RELEASENOTES.3.2.4.md @@ -0,0 +1,55 @@ + + +# Apache Hadoop 3.2.4 Release Notes + +These release notes cover new developer and user-facing incompatibilities, important issues, features, and major improvements. + + +--- + +* [YARN-10820](https://issues.apache.org/jira/browse/YARN-10820) | *Major* | **Make GetClusterNodesRequestPBImpl thread safe** + +Added syncronization so that the "yarn node list" command does not fail intermittently + + +--- + +* [YARN-8234](https://issues.apache.org/jira/browse/YARN-8234) | *Critical* | **Improve RM system metrics publisher's performance by pushing events to timeline server in batch** + +When Timeline Service V1 or V1.5 is used, if "yarn.resourcemanager.system-metrics-publisher.timeline-server-v1.enable-batch" is set to true, ResourceManager sends timeline events in batch. The default value is false. If this functionality is enabled, the maximum number that events published in batch is configured by "yarn.resourcemanager.system-metrics-publisher.timeline-server-v1.batch-size". The default value is 1000. The interval of publishing events can be configured by "yarn.resourcemanager.system-metrics-publisher.timeline-server-v1.interval-seconds". By default, it is set to 60 seconds. + + +--- + +* [HADOOP-18088](https://issues.apache.org/jira/browse/HADOOP-18088) | *Major* | **Replace log4j 1.x with reload4j** + +log4j 1 was replaced with reload4j which is fork of log4j 1.2.17 with the goal of fixing pressing security issues. + +If you are depending on the hadoop artifacts in your build were explicitly excluding log4 artifacts, and now want to exclude the reload4j files, you will need to update your exclusion lists +\ + \org.slf4j\ + \slf4j-reload4j\ +\ +\ + \ch.qos.reload4j\ + \reload4j\ +\ + + + diff --git a/hadoop-common-project/hadoop-common/src/site/markdown/release/3.3.4/CHANGELOG.3.3.4.md b/hadoop-common-project/hadoop-common/src/site/markdown/release/3.3.4/CHANGELOG.3.3.4.md new file mode 100644 index 0000000000000..78b805240c78e --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/site/markdown/release/3.3.4/CHANGELOG.3.3.4.md @@ -0,0 +1,56 @@ + + +# Apache Hadoop Changelog + +## Release 3.3.4 - 2022-07-29 + + + +### IMPROVEMENTS: + +| JIRA | Summary | Priority | Component | Reporter | Contributor | +|:---- |:---- | :--- |:---- |:---- |:---- | +| [HADOOP-18044](https://issues.apache.org/jira/browse/HADOOP-18044) | Hadoop - Upgrade to JQuery 3.6.0 | Major | . | Yuan Luo | Yuan Luo | +| [YARN-11195](https://issues.apache.org/jira/browse/YARN-11195) | Document how to configure NUMA in YARN | Major | documentation | Prabhu Joseph | Samrat Deb | +| [HADOOP-18332](https://issues.apache.org/jira/browse/HADOOP-18332) | Remove rs-api dependency by downgrading jackson to 2.12.7 | Major | build | PJ Fanning | PJ Fanning | +| [HADOOP-18354](https://issues.apache.org/jira/browse/HADOOP-18354) | Upgrade reload4j to 1.2.22 due to XXE vulnerability | Major | . | PJ Fanning | PJ Fanning | + + +### BUG FIXES: + +| JIRA | Summary | Priority | Component | Reporter | Contributor | +|:---- |:---- | :--- |:---- |:---- |:---- | +| [HADOOP-18085](https://issues.apache.org/jira/browse/HADOOP-18085) | S3 SDK Upgrade causes AccessPoint ARN endpoint mistranslation | Major | fs/s3, test | Bogdan Stolojan | Bogdan Stolojan | +| [YARN-11092](https://issues.apache.org/jira/browse/YARN-11092) | Upgrade jquery ui to 1.13.1 | Major | . | D M Murali Krishna Reddy | groot | +| [HDFS-16453](https://issues.apache.org/jira/browse/HDFS-16453) | Upgrade okhttp from 2.7.5 to 4.9.3 | Major | hdfs-client | Ivan Viaznikov | groot | +| [YARN-10974](https://issues.apache.org/jira/browse/YARN-10974) | CS UI: queue filter and openQueues param do not work as expected | Major | capacity scheduler | Chengbing Liu | Chengbing Liu | +| [HADOOP-18237](https://issues.apache.org/jira/browse/HADOOP-18237) | Upgrade Apache Xerces Java to 2.12.2 | Major | build | groot | groot | +| [HADOOP-18074](https://issues.apache.org/jira/browse/HADOOP-18074) | Partial/Incomplete groups list can be returned in LDAP groups lookup | Major | security | Philippe Lanoe | Larry McCay | +| [HADOOP-18079](https://issues.apache.org/jira/browse/HADOOP-18079) | Upgrade Netty to 4.1.77.Final | Major | build | Renukaprasad C | Wei-Chiu Chuang | + + +### SUB-TASKS: + +| JIRA | Summary | Priority | Component | Reporter | Contributor | +|:---- |:---- | :--- |:---- |:---- |:---- | +| [HADOOP-18068](https://issues.apache.org/jira/browse/HADOOP-18068) | Upgrade AWS SDK to 1.12.132 | Major | build, fs/s3 | Steve Loughran | Steve Loughran | +| [HADOOP-18307](https://issues.apache.org/jira/browse/HADOOP-18307) | remove hadoop-cos as a dependency of hadoop-cloud-storage | Major | bulid, fs | Steve Loughran | Steve Loughran | +| [HADOOP-18344](https://issues.apache.org/jira/browse/HADOOP-18344) | AWS SDK update to 1.12.262 to address jackson CVE-2018-7489 | Major | fs/s3 | Steve Loughran | Steve Loughran | + + diff --git a/hadoop-common-project/hadoop-common/src/site/markdown/release/3.3.4/RELEASENOTES.3.3.4.md b/hadoop-common-project/hadoop-common/src/site/markdown/release/3.3.4/RELEASENOTES.3.3.4.md new file mode 100644 index 0000000000000..79573880423d6 --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/site/markdown/release/3.3.4/RELEASENOTES.3.3.4.md @@ -0,0 +1,66 @@ + + +# Apache Hadoop 3.3.4 Release Notes + +These release notes cover new developer and user-facing incompatibilities, important issues, features, and major improvements. + + +--- + +* [HDFS-16453](https://issues.apache.org/jira/browse/HDFS-16453) | *Major* | **Upgrade okhttp from 2.7.5 to 4.9.3** + +okhttp has been updated to address CVE-2021-0341 + + +--- + +* [HADOOP-18237](https://issues.apache.org/jira/browse/HADOOP-18237) | *Major* | **Upgrade Apache Xerces Java to 2.12.2** + +Apache Xerces has been updated to 2.12.2 to fix CVE-2022-23437 + + +--- + +* [HADOOP-18307](https://issues.apache.org/jira/browse/HADOOP-18307) | *Major* | **remove hadoop-cos as a dependency of hadoop-cloud-storage** + +We have recently become aware that libraries which include a shaded apache httpclient libraries (hadoop-client-runtime.jar, aws-java-sdk-bundle.jar, gcs-connector-shaded.jar, cos\_api-bundle-5.6.19.jar) all load and use the unshaded resource mozilla/public-suffix-list.txt. If an out of date version of this is found on the classpath first, attempts to negotiate TLS connections may fail with the error "Certificate doesn't match any of the subject alternative names". This release does not declare the hadoop-cos library to be a dependency of the hadoop-cloud-storage POM, so applications depending on that module are no longer exposed to this issue. If an application requires use of the hadoop-cos module, please declare an explicit dependency. + + +--- + +* [HADOOP-18332](https://issues.apache.org/jira/browse/HADOOP-18332) | *Major* | **Remove rs-api dependency by downgrading jackson to 2.12.7** + +Downgrades Jackson from 2.13.2 to 2.12.7 to fix class conflicts in downstream projects. This version of jackson does contain the fix for CVE-2020-36518. + + +--- + +* [HADOOP-18079](https://issues.apache.org/jira/browse/HADOOP-18079) | *Major* | **Upgrade Netty to 4.1.77.Final** + +Netty has been updated to address CVE-2019-20444, CVE-2019-20445 and CVE-2022-24823 + + +--- + +* [HADOOP-18344](https://issues.apache.org/jira/browse/HADOOP-18344) | *Major* | **AWS SDK update to 1.12.262 to address jackson CVE-2018-7489** + +The AWS SDK has been updated to 1.12.262 to address jackson CVE-2018-7489 + + + diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestLocalDirAllocator.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestLocalDirAllocator.java index acda898ea1342..939881f39df6d 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestLocalDirAllocator.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestLocalDirAllocator.java @@ -26,6 +26,7 @@ import java.util.NoSuchElementException; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.test.LambdaTestUtils; import org.apache.hadoop.util.DiskChecker.DiskErrorException; import org.apache.hadoop.util.Shell; @@ -532,4 +533,20 @@ public void testGetLocalPathForWriteForInvalidPaths() throws Exception { } } + /** + * Test to verify LocalDirAllocator log details to provide diagnostics when file creation fails. + * + * @throws Exception + */ + @Test(timeout = 30000) + public void testGetLocalPathForWriteForLessSpace() throws Exception { + String dir0 = buildBufferDir(ROOT, 0); + String dir1 = buildBufferDir(ROOT, 1); + conf.set(CONTEXT, dir0 + "," + dir1); + LambdaTestUtils.intercept(DiskErrorException.class, + String.format("Could not find any valid local directory for %s with requested size %s", + "p1/x", Long.MAX_VALUE - 1), "Expect a DiskErrorException.", + () -> dirAllocator.getLocalPathForWrite("p1/x", Long.MAX_VALUE - 1, conf)); + } } + diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestLocalFileSystem.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestLocalFileSystem.java index 29ef6ca6c7afd..38e16221a4518 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestLocalFileSystem.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestLocalFileSystem.java @@ -24,7 +24,6 @@ import org.apache.hadoop.io.IOUtils; import org.apache.hadoop.test.GenericTestUtils; import org.apache.hadoop.test.LambdaTestUtils; -import org.apache.hadoop.test.Whitebox; import org.apache.hadoop.util.StringUtils; import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.IO_FILE_BUFFER_SIZE_DEFAULT; @@ -650,7 +649,8 @@ public void testFileStatusPipeFile() throws Exception { RawLocalFileSystem fs = spy(origFs); Configuration conf = mock(Configuration.class); fs.setConf(conf); - Whitebox.setInternalState(fs, "useDeprecatedFileStatus", false); + + RawLocalFileSystem.setUseDeprecatedFileStatus(false); Path path = new Path("/foo"); File pipe = mock(File.class); when(pipe.isFile()).thenReturn(false); diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestVectoredReadUtils.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestVectoredReadUtils.java index 5d08b02e113d5..ebf0e14053bba 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestVectoredReadUtils.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestVectoredReadUtils.java @@ -35,6 +35,8 @@ import org.apache.hadoop.test.HadoopTestBase; import static org.apache.hadoop.fs.VectoredReadUtils.sortRanges; +import static org.apache.hadoop.fs.VectoredReadUtils.validateNonOverlappingAndReturnSortedRanges; +import static org.apache.hadoop.test.LambdaTestUtils.intercept; import static org.apache.hadoop.test.MoreAsserts.assertFutureCompletedSuccessfully; import static org.apache.hadoop.test.MoreAsserts.assertFutureFailedExceptionally; @@ -231,6 +233,36 @@ public void testSortAndMergeMoreCases() throws Exception { } + @Test + public void testValidateOverlappingRanges() throws Exception { + List input = Arrays.asList( + FileRange.createFileRange(100, 100), + FileRange.createFileRange(200, 100), + FileRange.createFileRange(250, 100) + ); + + intercept(UnsupportedOperationException.class, + () -> validateNonOverlappingAndReturnSortedRanges(input)); + + List input1 = Arrays.asList( + FileRange.createFileRange(100, 100), + FileRange.createFileRange(500, 100), + FileRange.createFileRange(1000, 100), + FileRange.createFileRange(1000, 100) + ); + + intercept(UnsupportedOperationException.class, + () -> validateNonOverlappingAndReturnSortedRanges(input1)); + + List input2 = Arrays.asList( + FileRange.createFileRange(100, 100), + FileRange.createFileRange(200, 100), + FileRange.createFileRange(300, 100) + ); + // consecutive ranges should pass. + validateNonOverlappingAndReturnSortedRanges(input2); + } + @Test public void testMaxSizeZeroDisablesMering() throws Exception { List randomRanges = Arrays.asList( @@ -354,17 +386,31 @@ public void testReadVectored() throws Exception { List input = Arrays.asList(FileRange.createFileRange(0, 100), FileRange.createFileRange(100_000, 100), FileRange.createFileRange(200_000, 100)); + runAndValidateVectoredRead(input); + } + + @Test + public void testReadVectoredZeroBytes() throws Exception { + List input = Arrays.asList(FileRange.createFileRange(0, 0), + FileRange.createFileRange(100_000, 100), + FileRange.createFileRange(200_000, 0)); + runAndValidateVectoredRead(input); + } + + + private void runAndValidateVectoredRead(List input) + throws Exception { Stream stream = Mockito.mock(Stream.class); Mockito.doAnswer(invocation -> { fillBuffer(invocation.getArgument(1)); return null; }).when(stream).readFully(ArgumentMatchers.anyLong(), - ArgumentMatchers.any(ByteBuffer.class)); + ArgumentMatchers.any(ByteBuffer.class)); // should not merge the ranges VectoredReadUtils.readVectored(stream, input, ByteBuffer::allocate); Mockito.verify(stream, Mockito.times(3)) - .readFully(ArgumentMatchers.anyLong(), ArgumentMatchers.any(ByteBuffer.class)); - for(int b=0; b < input.size(); ++b) { + .readFully(ArgumentMatchers.anyLong(), ArgumentMatchers.any(ByteBuffer.class)); + for (int b = 0; b < input.size(); ++b) { validateBuffer("buffer " + b, input.get(b).getData().get(), 0); } } diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/AbstractContractVectoredReadTest.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/AbstractContractVectoredReadTest.java index 77bcc496ff4a2..86b645b9ec9c5 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/AbstractContractVectoredReadTest.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/AbstractContractVectoredReadTest.java @@ -24,11 +24,10 @@ import java.util.Arrays; import java.util.List; import java.util.concurrent.CompletableFuture; -import java.util.concurrent.ExecutionException; +import java.util.concurrent.TimeUnit; import java.util.function.IntFunction; import org.assertj.core.api.Assertions; -import org.junit.Assert; import org.junit.Test; import org.junit.runner.RunWith; import org.junit.runners.Parameterized; @@ -43,13 +42,14 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.impl.FutureIOSupport; import org.apache.hadoop.io.WeakReferencedElasticByteBufferPool; -import org.apache.hadoop.test.LambdaTestUtils; import static org.apache.hadoop.fs.contract.ContractTestUtils.assertCapabilities; import static org.apache.hadoop.fs.contract.ContractTestUtils.assertDatasetEquals; import static org.apache.hadoop.fs.contract.ContractTestUtils.createFile; import static org.apache.hadoop.fs.contract.ContractTestUtils.returnBuffersToPoolPostRead; import static org.apache.hadoop.fs.contract.ContractTestUtils.validateVectoredReadResult; +import static org.apache.hadoop.test.LambdaTestUtils.intercept; +import static org.apache.hadoop.test.LambdaTestUtils.interceptFuture; @RunWith(Parameterized.class) public abstract class AbstractContractVectoredReadTest extends AbstractFSContractTestBase { @@ -84,6 +84,10 @@ public IntFunction getAllocate() { return allocate; } + public WeakReferencedElasticByteBufferPool getPool() { + return pool; + } + @Override public void setup() throws Exception { super.setup(); @@ -268,6 +272,11 @@ public void testConsecutiveRanges() throws Exception { } } + /** + * Test to validate EOF ranges. Default implementation fails with EOFException + * while reading the ranges. Some implementation like s3, checksum fs fail fast + * as they already have the file length calculated. + */ @Test public void testEOFRanges() throws Exception { FileSystem fs = getFileSystem(); @@ -277,16 +286,11 @@ public void testEOFRanges() throws Exception { in.readVectored(fileRanges, allocate); for (FileRange res : fileRanges) { CompletableFuture data = res.getData(); - try { - ByteBuffer buffer = data.get(); - // Shouldn't reach here. - Assert.fail("EOFException must be thrown while reading EOF"); - } catch (ExecutionException ex) { - // ignore as expected. - } catch (Exception ex) { - LOG.error("Exception while running vectored read ", ex); - Assert.fail("Exception while running vectored read " + ex); - } + interceptFuture(EOFException.class, + "", + ContractTestUtils.VECTORED_READ_OPERATION_TEST_TIMEOUT_SECONDS, + TimeUnit.SECONDS, + data); } } } @@ -382,6 +386,13 @@ protected List getSampleOverlappingRanges() { return fileRanges; } + protected List getConsecutiveRanges() { + List fileRanges = new ArrayList<>(); + fileRanges.add(FileRange.createFileRange(100, 500)); + fileRanges.add(FileRange.createFileRange(600, 500)); + return fileRanges; + } + /** * Validate that exceptions must be thrown during a vectored * read operation with specific input ranges. @@ -399,7 +410,7 @@ protected void verifyExceptionalVectoredRead( fs.openFile(path(VECTORED_READ_FILE_NAME)) .build(); try (FSDataInputStream in = builder.get()) { - LambdaTestUtils.intercept(clazz, + intercept(clazz, () -> in.readVectored(fileRanges, allocate)); } } diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/localfs/TestLocalFSContractVectoredRead.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/localfs/TestLocalFSContractVectoredRead.java index 5d6ca3f8f0c90..5ee888015315c 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/localfs/TestLocalFSContractVectoredRead.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/localfs/TestLocalFSContractVectoredRead.java @@ -18,6 +18,7 @@ package org.apache.hadoop.fs.contract.localfs; +import java.io.EOFException; import java.util.ArrayList; import java.util.List; import java.util.concurrent.CompletableFuture; @@ -30,6 +31,7 @@ import org.apache.hadoop.fs.FSDataInputStream; import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.fs.FileRange; +import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.LocalFileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.contract.AbstractContractVectoredReadTest; @@ -52,9 +54,33 @@ protected AbstractFSContract createContract(Configuration conf) { @Test public void testChecksumValidationDuringVectoredRead() throws Exception { - Path testPath = path("big_range_checksum"); + Path testPath = path("big_range_checksum_file"); + List someRandomRanges = new ArrayList<>(); + someRandomRanges.add(FileRange.createFileRange(10, 1024)); + someRandomRanges.add(FileRange.createFileRange(1025, 1024)); + validateCheckReadException(testPath, DATASET_LEN, someRandomRanges); + } + + + /** + * Test for file size less than checksum chunk size. + * {@code ChecksumFileSystem#bytesPerChecksum}. + */ + @Test + public void testChecksumValidationDuringVectoredReadSmallFile() throws Exception { + Path testPath = path("big_range_checksum_file"); + final int length = 471; + List smallFileRanges = new ArrayList<>(); + smallFileRanges.add(FileRange.createFileRange(10, 50)); + smallFileRanges.add(FileRange.createFileRange(100, 20)); + validateCheckReadException(testPath, length, smallFileRanges); + } + + private void validateCheckReadException(Path testPath, + int length, + List ranges) throws Exception { LocalFileSystem localFs = (LocalFileSystem) getFileSystem(); - final byte[] datasetCorrect = ContractTestUtils.dataset(DATASET_LEN, 'a', 32); + final byte[] datasetCorrect = ContractTestUtils.dataset(length, 'a', 32); try (FSDataOutputStream out = localFs.create(testPath, true)){ out.write(datasetCorrect); } @@ -63,24 +89,55 @@ public void testChecksumValidationDuringVectoredRead() throws Exception { .describedAs("Checksum file should be present") .isTrue(); CompletableFuture fis = localFs.openFile(testPath).build(); - List someRandomRanges = new ArrayList<>(); - someRandomRanges.add(FileRange.createFileRange(10, 1024)); - someRandomRanges.add(FileRange.createFileRange(1025, 1024)); try (FSDataInputStream in = fis.get()){ - in.readVectored(someRandomRanges, getAllocate()); - validateVectoredReadResult(someRandomRanges, datasetCorrect); + in.readVectored(ranges, getAllocate()); + validateVectoredReadResult(ranges, datasetCorrect); } - final byte[] datasetCorrupted = ContractTestUtils.dataset(DATASET_LEN, 'a', 64); + final byte[] datasetCorrupted = ContractTestUtils.dataset(length, 'a', 64); try (FSDataOutputStream out = localFs.getRaw().create(testPath, true)){ out.write(datasetCorrupted); } CompletableFuture fisN = localFs.openFile(testPath).build(); try (FSDataInputStream in = fisN.get()){ - in.readVectored(someRandomRanges, getAllocate()); + in.readVectored(ranges, getAllocate()); // Expect checksum exception when data is updated directly through // raw local fs instance. intercept(ChecksumException.class, - () -> validateVectoredReadResult(someRandomRanges, datasetCorrupted)); + () -> validateVectoredReadResult(ranges, datasetCorrupted)); + } + } + @Test + public void tesChecksumVectoredReadBoundaries() throws Exception { + Path testPath = path("boundary_range_checksum_file"); + final int length = 1071; + LocalFileSystem localFs = (LocalFileSystem) getFileSystem(); + final byte[] datasetCorrect = ContractTestUtils.dataset(length, 'a', 32); + try (FSDataOutputStream out = localFs.create(testPath, true)){ + out.write(datasetCorrect); + } + Path checksumPath = localFs.getChecksumFile(testPath); + Assertions.assertThat(localFs.exists(checksumPath)) + .describedAs("Checksum file should be present at {} ", checksumPath) + .isTrue(); + CompletableFuture fis = localFs.openFile(testPath).build(); + List smallRange = new ArrayList<>(); + smallRange.add(FileRange.createFileRange(1000, 71)); + try (FSDataInputStream in = fis.get()){ + in.readVectored(smallRange, getAllocate()); + validateVectoredReadResult(smallRange, datasetCorrect); } } + + + /** + * Overriding in checksum fs as vectored read api fails fast + * in case of EOF requested range. + */ + @Override + public void testEOFRanges() throws Exception { + FileSystem fs = getFileSystem(); + List fileRanges = new ArrayList<>(); + fileRanges.add(FileRange.createFileRange(DATASET_LEN, 100)); + verifyExceptionalVectoredRead(fs, fileRanges, EOFException.class); + } } diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/impl/prefetch/ExceptionAsserts.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/impl/prefetch/ExceptionAsserts.java new file mode 100644 index 0000000000000..82348d97798ea --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/impl/prefetch/ExceptionAsserts.java @@ -0,0 +1,63 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hadoop.fs.impl.prefetch; + +import org.apache.hadoop.test.LambdaTestUtils; + +import static org.apache.hadoop.test.LambdaTestUtils.intercept; + +public final class ExceptionAsserts { + + private ExceptionAsserts() { + } + + + /** + * Asserts that the given code throws an exception of the given type + * and that the exception message contains the given sub-message. + * + * Usage: + * + * ExceptionAsserts.assertThrows( + * IllegalArgumentException.class, + * "'nullArg' must not be null", + * () -> Preconditions.checkNotNull(null, "nullArg")); + * + * Note: JUnit 5 has similar functionality but it will be a long time before + * we move to that framework because of significant differences and lack of + * backward compatibility for some JUnit rules. + */ + public static void assertThrows( + Class expectedExceptionClass, + String partialMessage, + LambdaTestUtils.VoidCallable code) throws Exception { + + intercept(expectedExceptionClass, partialMessage, code); + + } + + public static void assertThrows( + Class expectedExceptionClass, + LambdaTestUtils.VoidCallable code) throws Exception { + + intercept(expectedExceptionClass, code); + + } +} diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/impl/prefetch/SampleDataForTests.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/impl/prefetch/SampleDataForTests.java new file mode 100644 index 0000000000000..b6f744582d3e2 --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/impl/prefetch/SampleDataForTests.java @@ -0,0 +1,71 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hadoop.fs.impl.prefetch; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +/** + * Frequently used test data items. + */ +public final class SampleDataForTests { + + private SampleDataForTests() { + } + + + // Array data. + public static final Object[] NULL_ARRAY = null; + + public static final Object[] EMPTY_ARRAY = new Object[0]; + + public static final Object[] NON_EMPTY_ARRAY = new Object[1]; + + public static final byte[] NULL_BYTE_ARRAY = null; + + public static final byte[] EMPTY_BYTE_ARRAY = new byte[0]; + + public static final byte[] NON_EMPTY_BYTE_ARRAY = new byte[1]; + + public static final short[] NULL_SHORT_ARRAY = null; + + public static final short[] EMPTY_SHORT_ARRAY = new short[0]; + + public static final short[] NON_EMPTY_SHORT_ARRAY = new short[1]; + + public static final int[] NULL_INT_ARRAY = null; + + public static final int[] EMPTY_INT_ARRAY = new int[0]; + + public static final int[] NON_EMPTY_INT_ARRAY = new int[1]; + + public static final long[] NULL_LONG_ARRAY = null; + + public static final long[] EMPTY_LONG_ARRAY = new long[0]; + + public static final long[] NON_EMPTY_LONG_ARRAY = new long[1]; + + public static final List NULL_LIST = null; + + public static final List EMPTY_LIST = new ArrayList(); + + public static final List VALID_LIST = Arrays.asList(new Object[1]); +} diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/impl/prefetch/TestBlockCache.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/impl/prefetch/TestBlockCache.java new file mode 100644 index 0000000000000..2ea041283a7e1 --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/impl/prefetch/TestBlockCache.java @@ -0,0 +1,98 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hadoop.fs.impl.prefetch; + +import java.nio.ByteBuffer; + +import org.junit.Test; + +import org.apache.hadoop.test.AbstractHadoopTestBase; + +import static org.apache.hadoop.test.LambdaTestUtils.intercept; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNotSame; +import static org.junit.Assert.assertTrue; + +public class TestBlockCache extends AbstractHadoopTestBase { + + private static final int BUFFER_SIZE = 16; + + @Test + public void testArgChecks() throws Exception { + // Should not throw. + BlockCache cache = + new SingleFilePerBlockCache(EmptyPrefetchingStatistics.getInstance()); + + ByteBuffer buffer = ByteBuffer.allocate(16); + + // Verify it throws correctly. + intercept(IllegalArgumentException.class, "'buffer' must not be null", + () -> cache.put(42, null)); + + + intercept(NullPointerException.class, null, + () -> new SingleFilePerBlockCache(null)); + + } + + + @Test + public void testPutAndGet() throws Exception { + BlockCache cache = + new SingleFilePerBlockCache(EmptyPrefetchingStatistics.getInstance()); + + ByteBuffer buffer1 = ByteBuffer.allocate(BUFFER_SIZE); + for (byte i = 0; i < BUFFER_SIZE; i++) { + buffer1.put(i); + } + + assertEquals(0, cache.size()); + assertFalse(cache.containsBlock(0)); + cache.put(0, buffer1); + assertEquals(1, cache.size()); + assertTrue(cache.containsBlock(0)); + ByteBuffer buffer2 = ByteBuffer.allocate(BUFFER_SIZE); + cache.get(0, buffer2); + assertNotSame(buffer1, buffer2); + assertBuffersEqual(buffer1, buffer2); + + assertEquals(1, cache.size()); + assertFalse(cache.containsBlock(1)); + cache.put(1, buffer1); + assertEquals(2, cache.size()); + assertTrue(cache.containsBlock(1)); + ByteBuffer buffer3 = ByteBuffer.allocate(BUFFER_SIZE); + cache.get(1, buffer3); + assertNotSame(buffer1, buffer3); + assertBuffersEqual(buffer1, buffer3); + } + + private void assertBuffersEqual(ByteBuffer buffer1, ByteBuffer buffer2) { + assertNotNull(buffer1); + assertNotNull(buffer2); + assertEquals(buffer1.limit(), buffer2.limit()); + assertEquals(BUFFER_SIZE, buffer1.limit()); + for (int i = 0; i < BUFFER_SIZE; i++) { + assertEquals(buffer1.get(i), buffer2.get(i)); + } + } +} diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/impl/prefetch/TestBlockData.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/impl/prefetch/TestBlockData.java new file mode 100644 index 0000000000000..50ce220f6527e --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/impl/prefetch/TestBlockData.java @@ -0,0 +1,159 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hadoop.fs.impl.prefetch; + +import org.junit.Test; + +import org.apache.hadoop.test.AbstractHadoopTestBase; + +import static org.apache.hadoop.test.LambdaTestUtils.intercept; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +public class TestBlockData extends AbstractHadoopTestBase { + + @Test + public void testArgChecks() throws Exception { + // Should not throw. + new BlockData(10, 5); + new BlockData(5, 10); + new BlockData(0, 10); + + // Verify it throws correctly. + + + intercept(IllegalArgumentException.class, "'fileSize' must not be negative", + () -> new BlockData(-1, 2)); + + intercept(IllegalArgumentException.class, + "'blockSize' must be a positive integer", + () -> new BlockData(10, 0)); + + intercept(IllegalArgumentException.class, + "'blockSize' must be a positive integer", + () -> new BlockData(10, -2)); + + intercept(IllegalArgumentException.class, + "'blockNumber' (-1) must be within the range [0, 3]", + () -> new BlockData(10, 3).isLastBlock( + -1)); + + intercept(IllegalArgumentException.class, + "'blockNumber' (11) must be within the range [0, 3]", + () -> new BlockData(10, 3).isLastBlock( + 11)); + + } + + @Test + public void testComputedFields() throws Exception { + testComputedFieldsHelper(0, 10); + testComputedFieldsHelper(1, 10); + testComputedFieldsHelper(10, 1); + testComputedFieldsHelper(10, 2); + testComputedFieldsHelper(10, 3); + } + + private void testComputedFieldsHelper(long fileSize, int blockSize) + throws Exception { + BlockData bd = new BlockData(fileSize, blockSize); + + if (fileSize == 0) { + assertFalse(bd.isLastBlock(0)); + assertFalse(bd.isLastBlock(1)); + assertFalse(bd.isValidOffset(0)); + assertEquals(0, bd.getSize(0)); + assertEquals("", bd.getStateString()); + + ExceptionAsserts.assertThrows( + IllegalArgumentException.class, + "'offset' (0) must be within the range [0, -1]", + () -> bd.getBlockNumber(0)); + + ExceptionAsserts.assertThrows( + IllegalArgumentException.class, + "'blockNumber' (0) must be within the range [0, -1]", + () -> bd.getStartOffset(0)); + + ExceptionAsserts.assertThrows( + IllegalArgumentException.class, + "'offset' (0) must be within the range [0, -1]", + () -> bd.getRelativeOffset(0, 0)); + + ExceptionAsserts.assertThrows( + IllegalArgumentException.class, + "'blockNumber' (0) must be within the range [0, -1]", + () -> bd.getState(0)); + + ExceptionAsserts.assertThrows( + IllegalArgumentException.class, + "'blockNumber' (0) must be within the range [0, -1]", + () -> bd.setState(0, BlockData.State.READY)); + + return; + } + + assertEquals(fileSize, bd.getFileSize()); + assertEquals(blockSize, bd.getBlockSize()); + + int expectedNumBlocks = (int) (fileSize / blockSize); + if (fileSize % blockSize > 0) { + expectedNumBlocks++; + } + assertEquals(expectedNumBlocks, bd.getNumBlocks()); + + int lastBlockNumber = expectedNumBlocks - 1; + for (int b = 0; b < lastBlockNumber; b++) { + assertFalse(bd.isLastBlock(b)); + assertEquals(blockSize, bd.getSize(b)); + } + assertTrue(bd.isLastBlock(lastBlockNumber)); + int lastBlockSize = (int) (fileSize - blockSize * (expectedNumBlocks - 1)); + assertEquals(lastBlockSize, bd.getSize(lastBlockNumber)); + + // Offset related methods. + for (long offset = 0; offset < fileSize; offset++) { + int expectedBlockNumber = (int) (offset / blockSize); + assertEquals(expectedBlockNumber, bd.getBlockNumber(offset)); + + for (int b = 0; b < expectedNumBlocks - 1; b++) { + long expectedStartOffset = b * blockSize; + assertEquals(expectedStartOffset, bd.getStartOffset(b)); + + int expectedRelativeOffset = (int) (offset - expectedStartOffset); + assertEquals(expectedRelativeOffset, bd.getRelativeOffset(b, offset)); + } + } + + + // State methods. + for (int b = 0; b < expectedNumBlocks; b++) { + assertEquals(b * blockSize, bd.getStartOffset(b)); + assertEquals(BlockData.State.NOT_READY, bd.getState(b)); + bd.setState(b, BlockData.State.QUEUED); + assertEquals(BlockData.State.QUEUED, bd.getState(b)); + bd.setState(b, BlockData.State.READY); + assertEquals(BlockData.State.READY, bd.getState(b)); + bd.setState(b, BlockData.State.CACHED); + assertEquals(BlockData.State.CACHED, bd.getState(b)); + } + } +} diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/impl/prefetch/TestBlockOperations.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/impl/prefetch/TestBlockOperations.java new file mode 100644 index 0000000000000..703041379ab6e --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/impl/prefetch/TestBlockOperations.java @@ -0,0 +1,106 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hadoop.fs.impl.prefetch; + +import java.lang.reflect.Method; + +import org.junit.Test; + +import org.apache.hadoop.test.AbstractHadoopTestBase; + +import static org.apache.hadoop.test.LambdaTestUtils.intercept; +import static org.junit.Assert.assertTrue; + +public class TestBlockOperations extends AbstractHadoopTestBase { + + @Test + public void testArgChecks() throws Exception { + // Should not throw. + BlockOperations ops = new BlockOperations(); + + // Verify it throws correctly. + + intercept(IllegalArgumentException.class, + "'blockNumber' must not be negative", + () -> ops.getPrefetched(-1)); + + intercept(IllegalArgumentException.class, + "'blockNumber' must not be negative", + () -> ops.getCached(-1)); + + intercept(IllegalArgumentException.class, + "'blockNumber' must not be negative", + () -> ops.getRead(-1)); + + intercept(IllegalArgumentException.class, + "'blockNumber' must not be negative", + () -> ops.release(-1)); + + intercept(IllegalArgumentException.class, + "'blockNumber' must not be negative", + () -> ops.requestPrefetch(-1)); + + intercept(IllegalArgumentException.class, + "'blockNumber' must not be negative", + () -> ops.requestCaching(-1)); + + } + + @Test + public void testGetSummary() throws Exception { + verifySummary("getPrefetched", "GP"); + verifySummary("getCached", "GC"); + verifySummary("getRead", "GR"); + verifySummary("release", "RL"); + verifySummary("requestPrefetch", "RP"); + verifySummary("prefetch", "PF"); + verifySummary("requestCaching", "RC"); + verifySummary("addToCache", "C+"); + + verifySummaryNoArg("cancelPrefetches", "CP"); + verifySummaryNoArg("close", "CX"); + } + + private void verifySummary(String methodName, String shortName) + throws Exception { + int blockNumber = 42; + BlockOperations ops = new BlockOperations(); + Method method = ops.getClass().getDeclaredMethod(methodName, int.class); + BlockOperations.Operation op = + (BlockOperations.Operation) method.invoke(ops, blockNumber); + ops.end(op); + String summary = ops.getSummary(false); + String opSummary = String.format("%s(%d)", shortName, blockNumber); + String expectedSummary = String.format("%s;E%s;", opSummary, opSummary); + assertTrue(summary.startsWith(expectedSummary)); + } + + private void verifySummaryNoArg(String methodName, String shortName) + throws Exception { + BlockOperations ops = new BlockOperations(); + Method method = ops.getClass().getDeclaredMethod(methodName); + BlockOperations.Operation op = + (BlockOperations.Operation) method.invoke(ops); + ops.end(op); + String summary = ops.getSummary(false); + String expectedSummary = String.format("%s;E%s;", shortName, shortName); + assertTrue(summary.startsWith(expectedSummary)); + } +} diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/impl/prefetch/TestBoundedResourcePool.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/impl/prefetch/TestBoundedResourcePool.java new file mode 100644 index 0000000000000..fc29e1b725405 --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/impl/prefetch/TestBoundedResourcePool.java @@ -0,0 +1,147 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hadoop.fs.impl.prefetch; + +import java.nio.ByteBuffer; +import java.util.Collections; +import java.util.IdentityHashMap; +import java.util.Set; + +import org.junit.Test; + +import org.apache.hadoop.test.AbstractHadoopTestBase; + +import static org.apache.hadoop.test.LambdaTestUtils.intercept; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertSame; +import static org.junit.Assert.assertTrue; + +public class TestBoundedResourcePool extends AbstractHadoopTestBase { + + static class BufferPool extends BoundedResourcePool { + + BufferPool(int size) { + super(size); + } + + @Override + protected ByteBuffer createNew() { + return ByteBuffer.allocate(10); + } + } + + @Test + public void testArgChecks() throws Exception { + + // Should not throw. + BufferPool pool = new BufferPool(5); + + // Verify it throws correctly. + + intercept(IllegalArgumentException.class, + "'size' must be a positive integer", + () -> new BufferPool(-1)); + + intercept(IllegalArgumentException.class, + "'size' must be a positive integer", + () -> new BufferPool(0)); + + intercept(IllegalArgumentException.class, "'item' must not be null", + () -> pool.release(null)); + + intercept(IllegalArgumentException.class, + "This item is not a part of this pool", + () -> pool.release(ByteBuffer.allocate(4))); + + } + + @Test + public void testAcquireReleaseSingle() { + final int numBuffers = 5; + BufferPool pool = new BufferPool(numBuffers); + + assertEquals(0, pool.numCreated()); + assertEquals(numBuffers, pool.numAvailable()); + + ByteBuffer buffer1 = pool.acquire(); + assertNotNull(buffer1); + assertEquals(1, pool.numCreated()); + assertEquals(numBuffers - 1, pool.numAvailable()); + + // Release and immediately reacquire => should not end up creating new buffer. + pool.release(buffer1); + assertEquals(1, pool.numCreated()); + + ByteBuffer buffer2 = pool.acquire(); + assertNotNull(buffer2); + assertSame(buffer1, buffer2); + assertEquals(1, pool.numCreated()); + } + + @Test + public void testAcquireReleaseMultiple() { + final int numBuffers = 5; + BufferPool pool = new BufferPool(numBuffers); + Set buffers = + Collections.newSetFromMap(new IdentityHashMap()); + + assertEquals(0, pool.numCreated()); + + // Acquire all one by one. + for (int i = 0; i < numBuffers; i++) { + assertEquals(numBuffers - i, pool.numAvailable()); + ByteBuffer buffer = pool.acquire(); + assertNotNull(buffer); + assertFalse(buffers.contains(buffer)); + buffers.add(buffer); + assertEquals(i + 1, pool.numCreated()); + } + + assertEquals(numBuffers, pool.numCreated()); + assertEquals(0, pool.numAvailable()); + + int releaseCount = 0; + + // Release all one by one. + for (ByteBuffer buffer : buffers) { + assertEquals(releaseCount, pool.numAvailable()); + releaseCount++; + pool.release(buffer); + assertEquals(releaseCount, pool.numAvailable()); + + // Releasing the same buffer again should not have any ill effect. + pool.release(buffer); + assertEquals(releaseCount, pool.numAvailable()); + pool.release(buffer); + assertEquals(releaseCount, pool.numAvailable()); + } + + // Acquire all one by one again to ensure that they are the same ones we got earlier. + for (int i = 0; i < numBuffers; i++) { + ByteBuffer buffer = pool.acquire(); + assertTrue(buffers.contains(buffer)); + } + + assertEquals(numBuffers, pool.numCreated()); + assertEquals(0, pool.numAvailable()); + } +} diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/impl/prefetch/TestBufferData.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/impl/prefetch/TestBufferData.java new file mode 100644 index 0000000000000..ee5f95ca6bbb6 --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/impl/prefetch/TestBufferData.java @@ -0,0 +1,244 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hadoop.fs.impl.prefetch; + +import java.nio.ByteBuffer; +import java.nio.ReadOnlyBufferException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.concurrent.CompletableFuture; + +import org.junit.Test; + +import org.apache.hadoop.test.AbstractHadoopTestBase; + +import static org.apache.hadoop.test.LambdaTestUtils.intercept; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNotSame; +import static org.junit.Assert.assertSame; + +public class TestBufferData extends AbstractHadoopTestBase { + + @Test + public void testArgChecks() throws Exception { + // Should not throw. + ByteBuffer buffer = ByteBuffer.allocate(1); + BufferData data = new BufferData(1, buffer); + + // Verify it throws correctly. + + intercept(IllegalArgumentException.class, + "'blockNumber' must not be negative", + () -> new BufferData(-1, buffer)); + + intercept(IllegalArgumentException.class, "'buffer' must not be null", + () -> new BufferData(1, null)); + + intercept(IllegalArgumentException.class, "'actionFuture' must not be null", + () -> data.setPrefetch(null)); + + intercept(IllegalArgumentException.class, "'actionFuture' must not be null", + () -> data.setCaching(null)); + + intercept(IllegalArgumentException.class, "'states' must not be null", + () -> data.throwIfStateIncorrect((BufferData.State[]) null)); + + intercept(IllegalStateException.class, + "Expected buffer state to be 'READY or CACHING' but found", + () -> data.throwIfStateIncorrect(BufferData.State.READY, + BufferData.State.CACHING)); + + } + + @Test + public void testValidStateUpdates() { + ByteBuffer buffer = ByteBuffer.allocate(1); + BufferData data = new BufferData(1, buffer); + + assertEquals(BufferData.State.BLANK, data.getState()); + + CompletableFuture actionFuture = new CompletableFuture<>(); + actionFuture.complete(null); + data.setPrefetch(actionFuture); + assertEquals(BufferData.State.PREFETCHING, data.getState()); + assertNotNull(data.getActionFuture()); + assertSame(actionFuture, data.getActionFuture()); + + CompletableFuture actionFuture2 = new CompletableFuture<>(); + data.setCaching(actionFuture2); + assertEquals(BufferData.State.CACHING, data.getState()); + assertNotNull(data.getActionFuture()); + assertSame(actionFuture2, data.getActionFuture()); + assertNotSame(actionFuture, actionFuture2); + + List states = Arrays.asList( + BufferData.State.BLANK, + BufferData.State.PREFETCHING, + BufferData.State.CACHING, + BufferData.State.READY + ); + + BufferData data2 = new BufferData(1, buffer); + BufferData.State prevState = null; + for (BufferData.State state : states) { + if (prevState != null) { + assertEquals(prevState, data2.getState()); + data2.updateState(state, prevState); + assertEquals(state, data2.getState()); + } + prevState = state; + } + } + + @Test + public void testInvalidStateUpdates() throws Exception { + CompletableFuture actionFuture = new CompletableFuture<>(); + actionFuture.complete(null); + testInvalidStateUpdatesHelper( + (d) -> d.setPrefetch(actionFuture), + BufferData.State.BLANK, + BufferData.State.READY); + + testInvalidStateUpdatesHelper( + (d) -> d.setCaching(actionFuture), + BufferData.State.PREFETCHING, + BufferData.State.READY); + } + + @Test + public void testSetReady() throws Exception { + byte[] bytes1 = new byte[5]; + initBytes(bytes1); + + ByteBuffer buffer = ByteBuffer.allocate(10); + buffer.put(bytes1); + buffer.limit(bytes1.length); + BufferData data = new BufferData(1, buffer); + assertNotEquals(BufferData.State.READY, data.getState()); + assertEquals(0, data.getChecksum()); + + data.setReady(BufferData.State.BLANK); + assertEquals(BufferData.State.READY, data.getState()); + assertNotEquals(0, data.getChecksum()); + + // Verify that buffer cannot be modified once in READY state. + ExceptionAsserts.assertThrows( + ReadOnlyBufferException.class, + null, + () -> data.getBuffer().put(bytes1)); + + // Verify that buffer cannot be set to READY state more than once. + ExceptionAsserts.assertThrows( + IllegalStateException.class, + "Checksum cannot be changed once set", + () -> data.setReady(BufferData.State.BLANK)); + + // Verify that we detect post READY buffer modification. + buffer.array()[2] = (byte) 42; + ExceptionAsserts.assertThrows( + IllegalStateException.class, + "checksum changed after setReady()", + () -> data.setDone()); + } + + @Test + public void testChecksum() { + byte[] bytes1 = new byte[5]; + byte[] bytes2 = new byte[10]; + + initBytes(bytes1); + initBytes(bytes2); + + ByteBuffer buffer1 = ByteBuffer.wrap(bytes1); + ByteBuffer buffer2 = ByteBuffer.wrap(bytes2); + buffer2.limit(bytes1.length); + + long checksum1 = BufferData.getChecksum(buffer1); + long checksum2 = BufferData.getChecksum(buffer2); + + assertEquals(checksum1, checksum2); + } + + private void initBytes(byte[] bytes) { + for (int i = 0; i < bytes.length; i++) { + bytes[i] = (byte) i; + } + } + + @FunctionalInterface + public interface StateChanger { + + void run(BufferData data) throws Exception; + } + + private void testInvalidStateUpdatesHelper( + StateChanger changeState, + BufferData.State... validFromState) throws Exception { + + ByteBuffer buffer = ByteBuffer.allocate(1); + BufferData data = new BufferData(1, buffer); + data.updateState(validFromState[0], BufferData.State.BLANK); + List states = this.getStatesExcept(validFromState); + BufferData.State prevState = validFromState[0]; + String expectedMessage = + String.format("Expected buffer state to be '%s", validFromState[0]); + for (BufferData.State s : states) { + data.updateState(s, prevState); + + ExceptionAsserts.assertThrows( + IllegalStateException.class, + expectedMessage, + () -> changeState.run(data)); + + assertEquals(s, data.getState()); + prevState = s; + } + } + + static final List ALL_STATES = Arrays.asList( + BufferData.State.UNKNOWN, + BufferData.State.BLANK, + BufferData.State.PREFETCHING, + BufferData.State.CACHING, + BufferData.State.READY + ); + + private List getStatesExcept(BufferData.State... states) { + + List result = new ArrayList<>(); + for (BufferData.State s : ALL_STATES) { + boolean found = false; + for (BufferData.State ss : states) { + if (s == ss) { + found = true; + } + } + + if (!found) { + result.add(s); + } + } + + return result; + } +} diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/impl/prefetch/TestBufferPool.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/impl/prefetch/TestBufferPool.java new file mode 100644 index 0000000000000..b8375fe66dcb1 --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/impl/prefetch/TestBufferPool.java @@ -0,0 +1,158 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hadoop.fs.impl.prefetch; + +import org.junit.Test; + +import org.apache.hadoop.test.AbstractHadoopTestBase; + +import static org.apache.hadoop.test.LambdaTestUtils.intercept; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertSame; + +public class TestBufferPool extends AbstractHadoopTestBase { + + private static final int POOL_SIZE = 2; + + private static final int BUFFER_SIZE = 10; + + private final PrefetchingStatistics statistics = + EmptyPrefetchingStatistics.getInstance(); + + @Test + public void testArgChecks() throws Exception { + // Should not throw. + BufferPool pool = new BufferPool(POOL_SIZE, BUFFER_SIZE, statistics); + + // Verify it throws correctly. + + intercept(IllegalArgumentException.class, + "'size' must be a positive integer", + () -> new BufferPool(0, 10, statistics)); + + intercept(IllegalArgumentException.class, + "'size' must be a positive integer", + () -> new BufferPool(-1, 10, statistics)); + + intercept(IllegalArgumentException.class, + "'bufferSize' must be a positive integer", + () -> new BufferPool(10, 0, statistics)); + + intercept(IllegalArgumentException.class, + "'bufferSize' must be a positive integer", + () -> new BufferPool(1, -10, statistics)); + + intercept(NullPointerException.class, + () -> new BufferPool(1, 10, null)); + + intercept(IllegalArgumentException.class, + "'blockNumber' must not be negative", + () -> pool.acquire(-1)); + + intercept(IllegalArgumentException.class, + "'blockNumber' must not be negative", + () -> pool.tryAcquire(-1)); + + intercept(NullPointerException.class, "data", + () -> pool.release((BufferData) null)); + + } + + @Test + public void testGetAndRelease() { + BufferPool pool = new BufferPool(POOL_SIZE, BUFFER_SIZE, statistics); + assertInitialState(pool, POOL_SIZE); + + int count = 0; + for (BufferData data : pool.getAll()) { + count++; + } + assertEquals(0, count); + + BufferData data1 = this.acquire(pool, 1); + BufferData data2 = this.acquire(pool, 2); + BufferData data3 = pool.tryAcquire(3); + assertNull(data3); + + count = 0; + for (BufferData data : pool.getAll()) { + count++; + } + assertEquals(2, count); + + assertEquals(2, pool.numCreated()); + assertEquals(0, pool.numAvailable()); + + data1.updateState(BufferData.State.READY, BufferData.State.BLANK); + pool.release(data1); + + assertEquals(2, pool.numCreated()); + assertEquals(1, pool.numAvailable()); + + data2.updateState(BufferData.State.READY, BufferData.State.BLANK); + pool.release(data2); + + assertEquals(2, pool.numCreated()); + assertEquals(2, pool.numAvailable()); + } + + @Test + public void testRelease() throws Exception { + testReleaseHelper(BufferData.State.BLANK, true); + testReleaseHelper(BufferData.State.PREFETCHING, true); + testReleaseHelper(BufferData.State.CACHING, true); + testReleaseHelper(BufferData.State.READY, false); + } + + private void testReleaseHelper(BufferData.State stateBeforeRelease, + boolean expectThrow) + throws Exception { + + BufferPool pool = new BufferPool(POOL_SIZE, BUFFER_SIZE, statistics); + assertInitialState(pool, POOL_SIZE); + + BufferData data = this.acquire(pool, 1); + data.updateState(stateBeforeRelease, BufferData.State.BLANK); + + if (expectThrow) { + + intercept(IllegalArgumentException.class, "Unable to release buffer", + () -> pool.release(data)); + + } else { + pool.release(data); + } + } + + private BufferData acquire(BufferPool pool, int blockNumber) { + BufferData data = pool.acquire(blockNumber); + assertNotNull(data); + assertSame(data, pool.acquire(blockNumber)); + assertEquals(blockNumber, data.getBlockNumber()); + return data; + } + + private void assertInitialState(BufferPool pool, int poolSize) { + assertEquals(poolSize, pool.numAvailable()); + assertEquals(0, pool.numCreated()); + } +} diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/impl/prefetch/TestExecutorServiceFuturePool.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/impl/prefetch/TestExecutorServiceFuturePool.java new file mode 100644 index 0000000000000..3b8bc75f14989 --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/impl/prefetch/TestExecutorServiceFuturePool.java @@ -0,0 +1,98 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hadoop.fs.impl.prefetch; + +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; + +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import org.apache.hadoop.test.AbstractHadoopTestBase; + +import static org.apache.hadoop.test.LambdaTestUtils.interceptFuture; +import static org.junit.Assert.assertTrue; + +public class TestExecutorServiceFuturePool extends AbstractHadoopTestBase { + + private ExecutorService executorService; + + @Before + public void setUp() { + executorService = Executors.newFixedThreadPool(3); + } + + @After + public void tearDown() { + if (executorService != null) { + executorService.shutdownNow(); + } + } + + @Test + public void testRunnableSucceeds() throws Exception { + ExecutorServiceFuturePool futurePool = + new ExecutorServiceFuturePool(executorService); + final AtomicBoolean atomicBoolean = new AtomicBoolean(false); + Future future = + futurePool.executeRunnable(() -> atomicBoolean.set(true)); + future.get(30, TimeUnit.SECONDS); + assertTrue("atomicBoolean set to true?", atomicBoolean.get()); + } + + @Test + public void testSupplierSucceeds() throws Exception { + ExecutorServiceFuturePool futurePool = + new ExecutorServiceFuturePool(executorService); + final AtomicBoolean atomicBoolean = new AtomicBoolean(false); + Future future = futurePool.executeFunction(() -> { + atomicBoolean.set(true); + return null; + }); + future.get(30, TimeUnit.SECONDS); + assertTrue("atomicBoolean set to true?", atomicBoolean.get()); + } + + @Test + public void testRunnableFails() throws Exception { + ExecutorServiceFuturePool futurePool = + new ExecutorServiceFuturePool(executorService); + Future future = futurePool.executeRunnable(() -> { + throw new IllegalStateException("deliberate"); + }); + interceptFuture(IllegalStateException.class, "deliberate", 30, + TimeUnit.SECONDS, future); + } + + @Test + public void testSupplierFails() throws Exception { + ExecutorServiceFuturePool futurePool = + new ExecutorServiceFuturePool(executorService); + Future future = futurePool.executeFunction(() -> { + throw new IllegalStateException("deliberate"); + }); + interceptFuture(IllegalStateException.class, "deliberate", 30, + TimeUnit.SECONDS, future); + } +} diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/impl/prefetch/TestFilePosition.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/impl/prefetch/TestFilePosition.java new file mode 100644 index 0000000000000..e86c4be97b961 --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/impl/prefetch/TestFilePosition.java @@ -0,0 +1,195 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hadoop.fs.impl.prefetch; + +import java.nio.ByteBuffer; + +import org.junit.Test; + +import org.apache.hadoop.test.AbstractHadoopTestBase; + +import static org.apache.hadoop.test.LambdaTestUtils.intercept; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +public class TestFilePosition extends AbstractHadoopTestBase { + + @Test + public void testArgChecks() throws Exception { + ByteBuffer buffer = ByteBuffer.allocate(10); + BufferData data = new BufferData(0, buffer); + + // Should not throw. + new FilePosition(0, 0); + new FilePosition(0, 5); + new FilePosition(10, 5); + new FilePosition(5, 10); + new FilePosition(10, 5).setData(data, 3, 4); + + // Verify it throws correctly. + + intercept(IllegalArgumentException.class, "'fileSize' must not be negative", + () -> new FilePosition(-1, 2)); + + intercept(IllegalArgumentException.class, + "'blockSize' must be a positive integer", + () -> new FilePosition(1, 0)); + + intercept(IllegalArgumentException.class, + "'blockSize' must be a positive integer", + () -> new FilePosition(1, -1)); + + FilePosition pos = new FilePosition(10, 3); + + // Verify that we cannot obtain buffer properties without setting buffer. + + intercept(IllegalStateException.class, "'buffer' must not be null", + () -> pos.buffer()); + + intercept(IllegalStateException.class, "'buffer' must not be null", + () -> pos.absolute()); + + intercept(IllegalStateException.class, "'buffer' must not be null", + () -> pos.isWithinCurrentBuffer(2)); + + intercept(IllegalStateException.class, "'buffer' must not be null", + () -> pos.blockNumber()); + + intercept(IllegalStateException.class, "'buffer' must not be null", + () -> pos.isLastBlock()); + + intercept(IllegalStateException.class, "'buffer' must not be null", + () -> pos.bufferFullyRead()); + + // Verify that we cannot set invalid buffer parameters. + + intercept(IllegalArgumentException.class, "'bufferData' must not be null", + () -> pos.setData(null, 4, 4)); + + intercept(IllegalArgumentException.class, + "'startOffset' must not be negative", () -> pos.setData(data, -4, 4)); + + intercept(IllegalArgumentException.class, + "'readOffset' must not be negative", () -> pos.setData(data, 4, -4)); + + intercept(IllegalArgumentException.class, + "'readOffset' must not be negative", () -> pos.setData(data, 4, -4)); + + intercept(IllegalArgumentException.class, + "'readOffset' (15) must be within the range [4, 13]", + () -> pos.setData(data, 4, 15)); + + intercept(IllegalArgumentException.class, + "'readOffset' (3) must be within the range [4, 13]", + () -> pos.setData(data, 4, 3)); + + } + + @Test + public void testValidity() { + int bufferSize = 8; + long fileSize = 100; + long bufferStartOffset = 7; + long readStartOffset = 9; + + ByteBuffer buffer = ByteBuffer.allocate(bufferSize); + BufferData data = new BufferData(0, buffer); + FilePosition pos = new FilePosition(fileSize, bufferSize); + + assertFalse(pos.isValid()); + pos.setData(data, bufferStartOffset, readStartOffset); + assertTrue(pos.isValid()); + + pos.invalidate(); + assertFalse(pos.isValid()); + } + + @Test + public void testOffsets() { + int bufferSize = 8; + long fileSize = 100; + long bufferStartOffset = 7; + long readStartOffset = 9; + + ByteBuffer buffer = ByteBuffer.allocate(bufferSize); + BufferData data = new BufferData(0, buffer); + FilePosition pos = new FilePosition(fileSize, bufferSize); + pos.setData(data, bufferStartOffset, readStartOffset); + assertTrue(pos.isValid()); + + assertEquals(readStartOffset, pos.absolute()); + assertEquals(readStartOffset - bufferStartOffset, pos.relative()); + assertTrue(pos.isWithinCurrentBuffer(8)); + assertFalse(pos.isWithinCurrentBuffer(6)); + assertFalse(pos.isWithinCurrentBuffer(1)); + + int expectedBlockNumber = (int) (bufferStartOffset / bufferSize); + assertEquals(expectedBlockNumber, pos.blockNumber()); + assertFalse(pos.isLastBlock()); + + pos.setData(data, fileSize - 3, fileSize - 2); + assertTrue(pos.isLastBlock()); + } + + @Test + public void testBufferStats() { + int bufferSize = 8; + long fileSize = 100; + long bufferStartOffset = 7; + long readStartOffset = 9; + + ByteBuffer buffer = ByteBuffer.allocate(bufferSize); + BufferData data = new BufferData(0, buffer); + FilePosition pos = new FilePosition(fileSize, bufferSize); + pos.setData(data, bufferStartOffset, readStartOffset); + assertTrue(pos.isValid()); + assertEquals(bufferStartOffset, pos.bufferStartOffset()); + + assertEquals(0, pos.numBytesRead()); + assertEquals(0, pos.numSingleByteReads()); + assertEquals(0, pos.numBufferReads()); + + pos.incrementBytesRead(1); + pos.incrementBytesRead(1); + pos.incrementBytesRead(1); + pos.incrementBytesRead(5); + pos.incrementBytesRead(51); + + assertEquals(59, pos.numBytesRead()); + assertEquals(3, pos.numSingleByteReads()); + assertEquals(2, pos.numBufferReads()); + + assertFalse(pos.bufferFullyRead()); + + pos.setData(data, bufferStartOffset, bufferStartOffset); + assertTrue(pos.isValid()); + + assertEquals(0, pos.numBytesRead()); + assertEquals(0, pos.numSingleByteReads()); + assertEquals(0, pos.numBufferReads()); + + for (int i = 0; i < bufferSize; i++) { + pos.buffer().get(); + pos.incrementBytesRead(1); + } + assertTrue(pos.bufferFullyRead()); + } +} diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/impl/prefetch/TestRetryer.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/impl/prefetch/TestRetryer.java new file mode 100644 index 0000000000000..50701c717a4b0 --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/impl/prefetch/TestRetryer.java @@ -0,0 +1,80 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hadoop.fs.impl.prefetch; + +import org.junit.Test; + +import org.apache.hadoop.test.AbstractHadoopTestBase; + +import static org.apache.hadoop.test.LambdaTestUtils.intercept; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +public class TestRetryer extends AbstractHadoopTestBase { + + @Test + public void testArgChecks() throws Exception { + // Should not throw. + new Retryer(10, 50, 500); + + // Verify it throws correctly. + + intercept(IllegalArgumentException.class, + "'perRetryDelay' must be a positive integer", + () -> new Retryer(-1, 50, 500)); + + intercept(IllegalArgumentException.class, + "'perRetryDelay' must be a positive integer", + () -> new Retryer(0, 50, 500)); + + intercept(IllegalArgumentException.class, + "'maxDelay' (5) must be greater than 'perRetryDelay' (10)", + () -> new Retryer(10, 5, 500)); + + intercept(IllegalArgumentException.class, + "'statusUpdateInterval' must be a positive integer", + () -> new Retryer(10, 50, -1)); + + intercept(IllegalArgumentException.class, + "'statusUpdateInterval' must be a positive integer", + () -> new Retryer(10, 50, 0)); + + } + + @Test + public void testRetry() { + int perRetryDelay = 1; + int statusUpdateInterval = 3; + int maxDelay = 10; + + Retryer retryer = + new Retryer(perRetryDelay, maxDelay, statusUpdateInterval); + for (int t = 1; t <= maxDelay; t++) { + assertTrue(retryer.continueRetry()); + if (t % statusUpdateInterval == 0) { + assertTrue(retryer.updateStatus()); + } else { + assertFalse(retryer.updateStatus()); + } + } + + assertFalse(retryer.continueRetry()); + } +} diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/impl/prefetch/TestValidate.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/impl/prefetch/TestValidate.java new file mode 100644 index 0000000000000..a42462b3355af --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/impl/prefetch/TestValidate.java @@ -0,0 +1,341 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hadoop.fs.impl.prefetch; + +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.Arrays; + +import org.junit.Test; + +import org.apache.hadoop.test.AbstractHadoopTestBase; + +import static org.apache.hadoop.fs.impl.prefetch.SampleDataForTests.EMPTY_BYTE_ARRAY; +import static org.apache.hadoop.fs.impl.prefetch.SampleDataForTests.EMPTY_INT_ARRAY; +import static org.apache.hadoop.fs.impl.prefetch.SampleDataForTests.EMPTY_LIST; +import static org.apache.hadoop.fs.impl.prefetch.SampleDataForTests.EMPTY_LONG_ARRAY; +import static org.apache.hadoop.fs.impl.prefetch.SampleDataForTests.EMPTY_SHORT_ARRAY; +import static org.apache.hadoop.fs.impl.prefetch.SampleDataForTests.NON_EMPTY_ARRAY; +import static org.apache.hadoop.fs.impl.prefetch.SampleDataForTests.NON_EMPTY_BYTE_ARRAY; +import static org.apache.hadoop.fs.impl.prefetch.SampleDataForTests.NON_EMPTY_INT_ARRAY; +import static org.apache.hadoop.fs.impl.prefetch.SampleDataForTests.NON_EMPTY_LONG_ARRAY; +import static org.apache.hadoop.fs.impl.prefetch.SampleDataForTests.NON_EMPTY_SHORT_ARRAY; +import static org.apache.hadoop.fs.impl.prefetch.SampleDataForTests.NULL_BYTE_ARRAY; +import static org.apache.hadoop.fs.impl.prefetch.SampleDataForTests.NULL_INT_ARRAY; +import static org.apache.hadoop.fs.impl.prefetch.SampleDataForTests.NULL_LIST; +import static org.apache.hadoop.fs.impl.prefetch.SampleDataForTests.NULL_LONG_ARRAY; +import static org.apache.hadoop.fs.impl.prefetch.SampleDataForTests.NULL_SHORT_ARRAY; +import static org.apache.hadoop.fs.impl.prefetch.SampleDataForTests.VALID_LIST; +import static org.apache.hadoop.fs.impl.prefetch.Validate.checkPositiveInteger; +import static org.apache.hadoop.test.LambdaTestUtils.intercept; + +public class TestValidate extends AbstractHadoopTestBase { + + @Test + public void testCheckNotNull() throws Exception { + String nonNullArg = "nonNullArg"; + String nullArg = null; + + // Should not throw. + Validate.checkNotNull(nonNullArg, "nonNullArg"); + + // Verify it throws. + + intercept(IllegalArgumentException.class, "'nullArg' must not be null", + () -> Validate.checkNotNull(nullArg, "nullArg")); + + } + + @Test + public void testCheckPositiveInteger() throws Exception { + int positiveArg = 1; + int zero = 0; + int negativeArg = -1; + + // Should not throw. + checkPositiveInteger(positiveArg, "positiveArg"); + + // Verify it throws. + + intercept(IllegalArgumentException.class, + "'negativeArg' must be a positive integer", + () -> checkPositiveInteger(negativeArg, "negativeArg")); + + intercept(IllegalArgumentException.class, + "'zero' must be a positive integer", + () -> checkPositiveInteger(zero, "zero")); + + } + + @Test + public void testCheckNotNegative() throws Exception { + int positiveArg = 1; + int zero = 0; + int negativeArg = -1; + + // Should not throw. + Validate.checkNotNegative(zero, "zeroArg"); + Validate.checkNotNegative(positiveArg, "positiveArg"); + + // Verify it throws. + + intercept(IllegalArgumentException.class, + "'negativeArg' must not be negative", + () -> Validate.checkNotNegative(negativeArg, "negativeArg")); + + } + + @Test + public void testCheckRequired() throws Exception { + // Should not throw. + Validate.checkRequired(true, "arg"); + + // Verify it throws. + + intercept(IllegalArgumentException.class, "'arg' is required", + () -> Validate.checkRequired(false, "arg")); + + } + + @Test + public void testCheckValid() throws Exception { + // Should not throw. + Validate.checkValid(true, "arg"); + + // Verify it throws. + ExceptionAsserts.assertThrows( + IllegalArgumentException.class, + "'arg' is invalid", + () -> Validate.checkValid(false, "arg")); + } + + @Test + public void testCheckValidWithValues() throws Exception { + String validValues = "foo, bar"; + + // Should not throw. + Validate.checkValid(true, "arg", validValues); + + // Verify it throws. + + intercept(IllegalArgumentException.class, + "'arg' is invalid. Valid values are: foo, bar", + () -> Validate.checkValid(false, "arg", validValues)); + + } + + @Test + public void testCheckNotNullAndNotEmpty() throws Exception { + // Should not throw. + Validate.checkNotNullAndNotEmpty(NON_EMPTY_ARRAY, "array"); + Validate.checkNotNullAndNotEmpty(NON_EMPTY_BYTE_ARRAY, "array"); + Validate.checkNotNullAndNotEmpty(NON_EMPTY_SHORT_ARRAY, "array"); + Validate.checkNotNullAndNotEmpty(NON_EMPTY_INT_ARRAY, "array"); + Validate.checkNotNullAndNotEmpty(NON_EMPTY_LONG_ARRAY, "array"); + + // Verify it throws. + + intercept(IllegalArgumentException.class, "'string' must not be empty", + () -> Validate.checkNotNullAndNotEmpty("", "string")); + + intercept(IllegalArgumentException.class, "'array' must not be null", () -> + Validate.checkNotNullAndNotEmpty(SampleDataForTests.NULL_ARRAY, + "array")); + + intercept(IllegalArgumentException.class, + "'array' must have at least one element", () -> + Validate.checkNotNullAndNotEmpty(SampleDataForTests.EMPTY_ARRAY, + "array")); + + ExceptionAsserts.assertThrows( + IllegalArgumentException.class, + "'array' must not be null", + () -> Validate.checkNotNullAndNotEmpty(NULL_BYTE_ARRAY, "array")); + + ExceptionAsserts.assertThrows( + IllegalArgumentException.class, + "'array' must have at least one element", + () -> Validate.checkNotNullAndNotEmpty(EMPTY_BYTE_ARRAY, "array")); + + ExceptionAsserts.assertThrows( + IllegalArgumentException.class, + "'array' must not be null", + () -> Validate.checkNotNullAndNotEmpty(NULL_SHORT_ARRAY, "array")); + + ExceptionAsserts.assertThrows( + IllegalArgumentException.class, + "'array' must have at least one element", + () -> Validate.checkNotNullAndNotEmpty(EMPTY_SHORT_ARRAY, "array")); + + ExceptionAsserts.assertThrows( + IllegalArgumentException.class, + "'array' must not be null", + () -> Validate.checkNotNullAndNotEmpty(NULL_INT_ARRAY, "array")); + + ExceptionAsserts.assertThrows( + IllegalArgumentException.class, + "'array' must have at least one element", + () -> Validate.checkNotNullAndNotEmpty(EMPTY_INT_ARRAY, "array")); + + ExceptionAsserts.assertThrows( + IllegalArgumentException.class, + "'array' must not be null", + () -> Validate.checkNotNullAndNotEmpty(NULL_LONG_ARRAY, "array")); + + ExceptionAsserts.assertThrows( + IllegalArgumentException.class, + "'array' must have at least one element", + () -> Validate.checkNotNullAndNotEmpty(EMPTY_LONG_ARRAY, "array")); + } + + @Test + public void testCheckListNotNullAndNotEmpty() throws Exception { + // Should not throw. + Validate.checkNotNullAndNotEmpty(VALID_LIST, "list"); + + // Verify it throws. + ExceptionAsserts.assertThrows( + IllegalArgumentException.class, + "'list' must not be null", + () -> Validate.checkNotNullAndNotEmpty(NULL_LIST, "list")); + + ExceptionAsserts.assertThrows( + IllegalArgumentException.class, + "'list' must have at least one element", + () -> Validate.checkNotNullAndNotEmpty(EMPTY_LIST, "list")); + } + + @Test + public void testCheckNotNullAndNumberOfElements() throws Exception { + // Should not throw. + Validate.checkNotNullAndNumberOfElements(Arrays.asList(1, 2, 3), 3, "arg"); + + // Verify it throws. + + intercept(IllegalArgumentException.class, "'arg' must not be null", + () -> Validate.checkNotNullAndNumberOfElements(null, 3, "arg")); + + // Verify it throws. + ExceptionAsserts.assertThrows( + IllegalArgumentException.class, + "Number of elements in 'arg' must be exactly 3, 2 given.", + () -> Validate.checkNotNullAndNumberOfElements(Arrays.asList(1, 2), 3, + "arg") + ); + } + + @Test + public void testCheckValuesEqual() throws Exception { + // Should not throw. + Validate.checkValuesEqual(1, "arg1", 1, "arg2"); + + // Verify it throws. + + intercept(IllegalArgumentException.class, + "'arg1' (1) must equal 'arg2' (2)", + () -> Validate.checkValuesEqual(1, "arg1", 2, "arg2")); + + } + + @Test + public void testCheckIntegerMultiple() throws Exception { + // Should not throw. + Validate.checkIntegerMultiple(10, "arg1", 5, "arg2"); + + // Verify it throws. + + intercept(IllegalArgumentException.class, + "'arg1' (10) must be an integer multiple of 'arg2' (3)", + () -> Validate.checkIntegerMultiple(10, "arg1", 3, "arg2")); + + } + + @Test + public void testCheckGreater() throws Exception { + // Should not throw. + Validate.checkGreater(10, "arg1", 5, "arg2"); + + // Verify it throws. + + intercept(IllegalArgumentException.class, + "'arg1' (5) must be greater than 'arg2' (10)", + () -> Validate.checkGreater(5, "arg1", 10, "arg2")); + + } + + @Test + public void testCheckGreaterOrEqual() throws Exception { + // Should not throw. + Validate.checkGreaterOrEqual(10, "arg1", 5, "arg2"); + + // Verify it throws. + + intercept(IllegalArgumentException.class, + "'arg1' (5) must be greater than or equal to 'arg2' (10)", + () -> Validate.checkGreaterOrEqual(5, "arg1", 10, "arg2")); + + } + + @Test + public void testCheckWithinRange() throws Exception { + // Should not throw. + Validate.checkWithinRange(10, "arg", 5, 15); + Validate.checkWithinRange(10.0, "arg", 5.0, 15.0); + + // Verify it throws. + + intercept(IllegalArgumentException.class, + "'arg' (5) must be within the range [10, 20]", + () -> Validate.checkWithinRange(5, "arg", 10, 20)); + + intercept(IllegalArgumentException.class, + "'arg' (5.0) must be within the range [10.0, 20.0]", + () -> Validate.checkWithinRange(5.0, "arg", 10.0, 20.0)); + + } + + @Test + public void testCheckPathExists() throws Exception { + Path tempFile = Files.createTempFile("foo", "bar"); + Path tempDir = tempFile.getParent(); + Path notFound = Paths.get(""); + + // Should not throw. + Validate.checkPathExists(tempFile, "tempFile"); + Validate.checkPathExists(tempDir, "tempDir"); + + // Verify it throws. + + intercept(IllegalArgumentException.class, "'nullArg' must not be null", + () -> Validate.checkPathExists(null, "nullArg")); + + intercept(IllegalArgumentException.class, + "Path notFound () does not exist", + () -> Validate.checkPathExists(notFound, "notFound")); + + intercept(IllegalArgumentException.class, "must point to a directory", + () -> Validate.checkPathExistsAsDir(tempFile, "tempFile")); + + intercept(IllegalArgumentException.class, "must point to a file", + () -> Validate.checkPathExistsAsFile(tempDir, "tempDir")); + + } +} diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/http/TestHttpServer.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/http/TestHttpServer.java index b1255d19d9086..b71610588c05b 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/http/TestHttpServer.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/http/TestHttpServer.java @@ -29,7 +29,6 @@ import org.apache.hadoop.security.ShellBasedUnixGroupsMapping; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.security.authorize.AccessControlList; -import org.apache.hadoop.test.Whitebox; import org.assertj.core.api.Assertions; import org.eclipse.jetty.server.ServerConnector; @@ -663,8 +662,7 @@ private HttpServer2 checkBindAddress(String host, int port, boolean findPort) HttpServer2 server = createServer(host, port); try { // not bound, ephemeral should return requested port (0 for ephemeral) - List listeners = (List) Whitebox.getInternalState(server, - "listeners"); + List listeners = server.getListeners(); ServerConnector listener = (ServerConnector)listeners.get(0); assertEquals(port, listener.getPort()); @@ -740,12 +738,18 @@ public void testBacklogSize() throws Exception Configuration conf = new Configuration(); conf.setInt(HttpServer2.HTTP_SOCKET_BACKLOG_SIZE_KEY, backlogSize); HttpServer2 srv = createServer("test", conf); - List listeners = (List) Whitebox.getInternalState(srv, - "listeners"); + List listeners = srv.getListeners(); ServerConnector listener = (ServerConnector)listeners.get(0); assertEquals(backlogSize, listener.getAcceptQueueSize()); } + @Test + public void testBacklogSize2() throws Exception + { + Configuration conf = new Configuration(); + assertEquals(500, conf.get(HttpServer2.HTTP_SOCKET_BACKLOG_SIZE_KEY)); + } + @Test public void testIdleTimeout() throws Exception { final int idleTimeout = 1000; diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/compress/TestCodecPool.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/compress/TestCodecPool.java index ec99598e79cee..4b18ee6047ba4 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/compress/TestCodecPool.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/compress/TestCodecPool.java @@ -19,6 +19,10 @@ import static org.junit.Assert.assertEquals; +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.OutputStream; +import java.util.Random; import java.util.concurrent.Callable; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; @@ -26,6 +30,9 @@ import java.util.concurrent.TimeUnit; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.io.compress.zlib.BuiltInGzipCompressor; +import org.apache.hadoop.io.compress.zlib.BuiltInGzipDecompressor; +import org.apache.hadoop.test.LambdaTestUtils; import org.junit.Before; import org.junit.Test; @@ -189,4 +196,56 @@ public void testDecompressorNotReturnSameInstance() { CodecPool.returnDecompressor(decompressor); } } + + @Test(timeout = 10000) + public void testDoNotPoolCompressorNotUseableAfterReturn() throws Exception { + + final GzipCodec gzipCodec = new GzipCodec(); + gzipCodec.setConf(new Configuration()); + + // BuiltInGzipCompressor is an explicit example of a Compressor with the @DoNotPool annotation + final Compressor compressor = new BuiltInGzipCompressor(new Configuration()); + CodecPool.returnCompressor(compressor); + + final CompressionOutputStream outputStream = + gzipCodec.createOutputStream(new ByteArrayOutputStream(), compressor); + LambdaTestUtils.intercept( + AlreadyClosedException.class, + "compress called on closed compressor", + "Compressor from Codec with @DoNotPool should not be " + + "useable after returning to CodecPool", + () -> outputStream.write(1)); + } + + @Test(timeout = 10000) + public void testDoNotPoolDecompressorNotUseableAfterReturn() throws Exception { + + final GzipCodec gzipCodec = new GzipCodec(); + gzipCodec.setConf(new Configuration()); + + final Random random = new Random(); + final byte[] bytes = new byte[1024]; + random.nextBytes(bytes); + + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + try (OutputStream outputStream = gzipCodec.createOutputStream(baos)) { + outputStream.write(bytes); + } + + final byte[] gzipBytes = baos.toByteArray(); + final ByteArrayInputStream bais = new ByteArrayInputStream(gzipBytes); + + // BuiltInGzipDecompressor is an explicit example of a Decompressor + // with the @DoNotPool annotation + final Decompressor decompressor = new BuiltInGzipDecompressor(); + CodecPool.returnDecompressor(decompressor); + + final CompressionInputStream inputStream = gzipCodec.createInputStream(bais, decompressor); + LambdaTestUtils.intercept( + AlreadyClosedException.class, + "decompress called on closed decompressor", + "Decompressor from Codec with @DoNotPool should not be " + + "useable after returning to CodecPool", + () -> inputStream.read()); + } } diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/compress/bzip2/BZip2TextFileWriter.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/compress/bzip2/BZip2TextFileWriter.java new file mode 100644 index 0000000000000..5ca99fd02dfbf --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/compress/bzip2/BZip2TextFileWriter.java @@ -0,0 +1,114 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.hadoop.io.compress.bzip2; + +import java.io.Closeable; +import java.io.IOException; +import java.io.OutputStream; +import java.nio.charset.StandardCharsets; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.io.compress.BZip2Codec; + +import static org.apache.hadoop.io.compress.bzip2.CBZip2OutputStream.MIN_BLOCKSIZE; +import static org.apache.hadoop.util.Preconditions.checkArgument; + +/** + * A writer that simplifies creating BZip2 compressed text data for testing + * purposes. + */ +public final class BZip2TextFileWriter implements Closeable { + + // Use minimum block size to reduce amount of data to require to be written + // to CBZip2OutputStream before a new block is created. + private static final int BLOCK_SIZE_100K = MIN_BLOCKSIZE; + + /** + * The amount of bytes of run-length encoded data that needs to be written + * to this writer in order for the next byte written starts a new BZip2 block. + */ + public static final int BLOCK_SIZE = + // The + 1 is needed because of how CBZip2OutputStream checks whether the + // last offset written is less than allowable block size. Because the last + // offset is one less of the amount of bytes written to the block, we need + // to write an extra byte to trigger writing a new block. + CBZip2OutputStream.getAllowableBlockSize(BLOCK_SIZE_100K) + 1; + + private final CBZip2OutputStream out; + + public BZip2TextFileWriter(Path path, Configuration conf) throws IOException { + this(path.getFileSystem(conf).create(path)); + } + + public BZip2TextFileWriter(OutputStream rawOut) throws IOException { + try { + BZip2Codec.writeHeader(rawOut); + out = new CBZip2OutputStream(rawOut, BLOCK_SIZE_100K); + } catch (Throwable e) { + rawOut.close(); + throw e; + } + } + + public void writeManyRecords(int totalSize, int numRecords, byte[] delimiter) + throws IOException { + checkArgument(numRecords > 0); + checkArgument(delimiter.length > 0); + + int minRecordSize = totalSize / numRecords; + checkArgument(minRecordSize >= delimiter.length); + + int lastRecordExtraSize = totalSize % numRecords; + + for (int i = 0; i < numRecords - 1; i++) { + writeRecord(minRecordSize, delimiter); + } + writeRecord(minRecordSize + lastRecordExtraSize, delimiter); + } + + public void writeRecord(int totalSize, byte[] delimiter) throws IOException { + checkArgument(delimiter.length > 0); + checkArgument(totalSize >= delimiter.length); + + int contentSize = totalSize - delimiter.length; + for (int i = 0; i < contentSize; i++) { + // Alternate between characters so that internals of CBZip2OutputStream + // cannot condensed the written bytes using run-length encoding. This + // allows the caller to use #BLOCK_SIZE in order to know whether the next + // write will end just before the end of the current block, or exceed it, + // and by how much. + out.write(i % 2 == 0 ? 'a' : 'b'); + } + write(delimiter); + } + + public void write(String bytes) throws IOException { + write(bytes.getBytes(StandardCharsets.UTF_8)); + } + + public void write(byte[] bytes) throws IOException { + out.write(bytes); + } + + @Override + public void close() throws IOException { + out.close(); + } +} diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/compress/bzip2/BZip2Utils.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/compress/bzip2/BZip2Utils.java new file mode 100644 index 0000000000000..d597ed0e7e0b8 --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/compress/bzip2/BZip2Utils.java @@ -0,0 +1,67 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.io.compress.bzip2; + +import java.io.IOException; +import java.io.InputStream; +import java.util.ArrayList; +import java.util.List; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; + +import static org.apache.hadoop.io.compress.SplittableCompressionCodec.READ_MODE.BYBLOCK; + +public final class BZip2Utils { + + private BZip2Utils() { + } + + /** + * Returns the start offsets of blocks that follow the first block in the + * BZip2 compressed file at the given path. The first offset corresponds to + * the first byte containing the BZip2 block marker of the second block. The + * i-th offset corresponds to the block marker of the (i + 1)-th block. + */ + public static List getNextBlockMarkerOffsets( + Path path, Configuration conf) throws IOException { + FileSystem fs = path.getFileSystem(conf); + try (InputStream fileIn = fs.open(path)) { + return getNextBlockMarkerOffsets(fileIn); + } + } + + /** + * Returns the start offsets of blocks that follow the first block in the + * BZip2 compressed input stream. The first offset corresponds to + * the first byte containing the BZip2 block marker of the second block. The + * i-th offset corresponds to the block marker of the (i + 1)-th block. + */ + public static List getNextBlockMarkerOffsets(InputStream rawIn) + throws IOException { + try (CBZip2InputStream in = new CBZip2InputStream(rawIn, BYBLOCK)) { + ArrayList offsets = new ArrayList<>(); + while (in.skipToNextBlockMarker()) { + offsets.add(in.getProcessedByteCount()); + } + return offsets; + } + } +} diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/compress/bzip2/TestBZip2TextFileWriter.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/compress/bzip2/TestBZip2TextFileWriter.java new file mode 100644 index 0000000000000..7d92e07f01b6a --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/compress/bzip2/TestBZip2TextFileWriter.java @@ -0,0 +1,91 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.io.compress.bzip2; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.util.List; + +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import static org.apache.hadoop.io.compress.bzip2.BZip2TextFileWriter.BLOCK_SIZE; +import static org.junit.Assert.assertEquals; + +public final class TestBZip2TextFileWriter { + + private static final byte[] DELIMITER = new byte[] {'\0'}; + + private ByteArrayOutputStream rawOut; + private BZip2TextFileWriter writer; + + @Before + public void setUp() throws Exception { + rawOut = new ByteArrayOutputStream(); + writer = new BZip2TextFileWriter(rawOut); + } + + @After + public void tearDown() throws Exception { + rawOut = null; + writer.close(); + } + + @Test + public void writingSingleBlockSizeOfData() throws Exception { + writer.writeRecord(BLOCK_SIZE, DELIMITER); + writer.close(); + + List nextBlocks = getNextBlockMarkerOffsets(); + assertEquals(0, nextBlocks.size()); + } + + @Test + public void justExceedingBeyondBlockSize() throws Exception { + writer.writeRecord(BLOCK_SIZE + 1, DELIMITER); + writer.close(); + + List nextBlocks = getNextBlockMarkerOffsets(); + assertEquals(1, nextBlocks.size()); + } + + @Test + public void writingTwoBlockSizesOfData() throws Exception { + writer.writeRecord(2 * BLOCK_SIZE, DELIMITER); + writer.close(); + + List nextBlocks = getNextBlockMarkerOffsets(); + assertEquals(1, nextBlocks.size()); + } + + @Test + public void justExceedingBeyondTwoBlocks() throws Exception { + writer.writeRecord(2 * BLOCK_SIZE + 1, DELIMITER); + writer.close(); + + List nextBlocks = getNextBlockMarkerOffsets(); + assertEquals(2, nextBlocks.size()); + } + + private List getNextBlockMarkerOffsets() throws IOException { + ByteArrayInputStream in = new ByteArrayInputStream(rawOut.toByteArray()); + return BZip2Utils.getNextBlockMarkerOffsets(in); + } +} \ No newline at end of file diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/retry/TestRetryProxy.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/retry/TestRetryProxy.java index e1fc29f88126b..59b9b13fbff55 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/retry/TestRetryProxy.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/retry/TestRetryProxy.java @@ -291,7 +291,7 @@ public void testRetryOtherThanRemoteException() throws Throwable { UnreliableInterface unreliable = (UnreliableInterface) RetryProxy.create(UnreliableInterface.class, unreliableImpl, - retryOtherThanRemoteException(TRY_ONCE_THEN_FAIL, + retryOtherThanRemoteAndSaslException(TRY_ONCE_THEN_FAIL, exceptionToPolicyMap)); // should retry with local IOException. unreliable.failsOnceWithIOException(); diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestIPC.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestIPC.java index 95ff302103d89..ffa17224b03bf 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestIPC.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestIPC.java @@ -18,6 +18,7 @@ package org.apache.hadoop.ipc; +import static org.assertj.core.api.Assertions.assertThatExceptionOfType; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotNull; @@ -91,8 +92,8 @@ import org.apache.hadoop.security.token.SecretManager.InvalidToken; import org.apache.hadoop.test.GenericTestUtils; import org.apache.hadoop.test.LambdaTestUtils; -import org.apache.hadoop.test.Whitebox; import org.apache.hadoop.util.StringUtils; +import org.assertj.core.api.Condition; import org.junit.Assert; import org.junit.Assume; import org.junit.Before; @@ -815,6 +816,81 @@ public Void call() throws IOException { } } + /** + * The {@link ConnectionId#hashCode} has to be stable despite updates that occur as the the + * address evolves over time. The {@link ConnectionId} is used as a primary key in maps, so + * its hashCode can't change. + * + * @throws IOException if there is a client or server failure + */ + @Test + public void testStableHashCode() throws IOException { + Server server = new TestServer(5, false); + try { + server.start(); + + // Leave host unresolved to start. Use "localhost" as opposed + // to local IP from NetUtils.getConnectAddress(server) to force + // resolution later + InetSocketAddress unresolvedAddr = InetSocketAddress.createUnresolved( + "localhost", NetUtils.getConnectAddress(server).getPort()); + + // Setup: Create a ConnectionID using an unresolved address, and get it's hashCode to serve + // as a point of comparison. + int rpcTimeout = MIN_SLEEP_TIME * 2; + final ConnectionId remoteId = getConnectionId(unresolvedAddr, rpcTimeout, conf); + int expected = remoteId.hashCode(); + + // Start client + Client.setConnectTimeout(conf, 100); + Client client = new Client(LongWritable.class, conf); + try { + // Test: Call should re-resolve host and succeed + LongWritable param = new LongWritable(RANDOM.nextLong()); + client.call(RPC.RpcKind.RPC_BUILTIN, param, remoteId, + RPC.RPC_SERVICE_CLASS_DEFAULT, null); + int actual = remoteId.hashCode(); + + // Verify: The hashCode should match, although the InetAddress is different since it has + // now been resolved + assertThat(remoteId.getAddress()).isNotEqualTo(unresolvedAddr); + assertThat(remoteId.getAddress().getHostName()).isEqualTo(unresolvedAddr.getHostName()); + assertThat(remoteId.hashCode()).isEqualTo(expected); + + // Test: Call should succeed without having to re-resolve + InetSocketAddress expectedSocketAddress = remoteId.getAddress(); + param = new LongWritable(RANDOM.nextLong()); + client.call(RPC.RpcKind.RPC_BUILTIN, param, remoteId, + RPC.RPC_SERVICE_CLASS_DEFAULT, null); + + // Verify: The same instance of the InetSocketAddress has been used to make the second + // call + assertThat(remoteId.getAddress()).isSameAs(expectedSocketAddress); + + // Verify: The hashCode is protected against updates to the host name + String hostName = InetAddress.getLocalHost().getHostName(); + InetSocketAddress mismatchedHostName = NetUtils.createSocketAddr( + InetAddress.getLocalHost().getHostName(), + remoteId.getAddress().getPort()); + assertThatExceptionOfType(IllegalArgumentException.class) + .isThrownBy(() -> remoteId.setAddress(mismatchedHostName)) + .withMessageStartingWith("Hostname must match"); + + // Verify: The hashCode is protected against updates to the port + InetSocketAddress mismatchedPort = NetUtils.createSocketAddr( + remoteId.getAddress().getHostName(), + remoteId.getAddress().getPort() + 1); + assertThatExceptionOfType(IllegalArgumentException.class) + .isThrownBy(() -> remoteId.setAddress(mismatchedPort)) + .withMessageStartingWith("Port must match"); + } finally { + client.stop(); + } + } finally { + server.stop(); + } + } + @Test(timeout=60000) public void testIpcFlakyHostResolution() throws IOException { // start server @@ -861,7 +937,6 @@ public void testIpcWithReaderQueuing() throws Exception { // goal is to jam a handler with a connection, fill the callq with // connections, in turn jamming the readers - then flood the server and // ensure that the listener blocks when the reader connection queues fill - @SuppressWarnings("unchecked") private void checkBlocking(int readers, int readerQ, int callQ) throws Exception { int handlers = 1; // makes it easier @@ -881,9 +956,8 @@ private void checkBlocking(int readers, int readerQ, int callQ) throws Exception // start server final TestServerQueue server = new TestServerQueue(clients, readers, callQ, handlers, conf); - CallQueueManager spy = spy( - (CallQueueManager)Whitebox.getInternalState(server, "callQueue")); - Whitebox.setInternalState(server, "callQueue", spy); + CallQueueManager spy = spy(server.getCallQueue()); + server.setCallQueue(spy); final InetSocketAddress addr = NetUtils.getConnectAddress(server); server.start(); diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestRPC.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestRPC.java index 5caabd22a88c6..101750d72c86d 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestRPC.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestRPC.java @@ -18,6 +18,8 @@ package org.apache.hadoop.ipc; +import org.apache.hadoop.fs.CommonConfigurationKeysPublic; +import org.apache.hadoop.io.retry.RetryUtils; import org.apache.hadoop.ipc.metrics.RpcMetrics; import org.apache.hadoop.thirdparty.com.google.common.util.concurrent.ThreadFactoryBuilder; @@ -50,7 +52,6 @@ import org.apache.hadoop.test.GenericTestUtils; import org.apache.hadoop.test.MetricsAsserts; import org.apache.hadoop.test.MockitoUtil; -import org.apache.hadoop.test.Whitebox; import org.junit.Assert; import org.junit.Before; import org.junit.Test; @@ -289,6 +290,14 @@ public ProtocolProxy getProxy( rpcTimeout, connectionRetryPolicy, null, null); } + @Override + public ProtocolProxy getProxy(Class protocol, long clientVersion, + ConnectionId connId, Configuration conf, SocketFactory factory, + AlignmentContext alignmentContext) + throws IOException { + throw new UnsupportedOperationException("This proxy is not supported"); + } + @SuppressWarnings("unchecked") @Override public ProtocolProxy getProxy( @@ -299,7 +308,7 @@ public ProtocolProxy getProxy( throws IOException { T proxy = (T) Proxy.newProxyInstance(protocol.getClassLoader(), new Class[] { protocol }, new StoppedInvocationHandler()); - return new ProtocolProxy(protocol, proxy, false); + return new ProtocolProxy<>(protocol, proxy, false); } @Override @@ -390,6 +399,53 @@ public void testProxyAddress() throws Exception { } } + @Test + public void testConnectionWithSocketFactory() throws IOException, ServiceException { + TestRpcService firstProxy = null; + TestRpcService secondProxy = null; + + Configuration newConf = new Configuration(conf); + newConf.set(CommonConfigurationKeysPublic. + HADOOP_RPC_SOCKET_FACTORY_CLASS_DEFAULT_KEY, ""); + + RetryPolicy retryPolicy = RetryUtils.getDefaultRetryPolicy( + newConf, "Test.No.Such.Key", + true, + "Test.No.Such.Key", "10000,6", + null); + + // create a server with two handlers + Server server = setupTestServer(newConf, 2); + try { + // create the first client + firstProxy = getClient(addr, newConf); + // create the second client + secondProxy = getClient(addr, newConf); + + firstProxy.ping(null, newEmptyRequest()); + secondProxy.ping(null, newEmptyRequest()); + + Client client = ProtobufRpcEngine2.getClient(newConf); + assertEquals(1, client.getConnectionIds().size()); + + stop(null, firstProxy, secondProxy); + ProtobufRpcEngine2.clearClientCache(); + + // create the first client with index 1 + firstProxy = getMultipleClientWithIndex(addr, newConf, retryPolicy, 1); + // create the second client with index 2 + secondProxy = getMultipleClientWithIndex(addr, newConf, retryPolicy, 2); + firstProxy.ping(null, newEmptyRequest()); + secondProxy.ping(null, newEmptyRequest()); + + Client client2 = ProtobufRpcEngine2.getClient(newConf); + assertEquals(2, client2.getConnectionIds().size()); + } finally { + System.out.println("Down slow rpc testing"); + stop(server, firstProxy, secondProxy); + } + } + @Test public void testSlowRpc() throws IOException, ServiceException { Server server; @@ -1162,10 +1218,8 @@ public void testClientBackOff() throws Exception { .setQueueSizePerHandler(1).setNumHandlers(1).setVerbose(true); server = setupTestServer(builder); - @SuppressWarnings("unchecked") - CallQueueManager spy = spy((CallQueueManager) Whitebox - .getInternalState(server, "callQueue")); - Whitebox.setInternalState(server, "callQueue", spy); + CallQueueManager spy = spy(server.getCallQueue()); + server.setCallQueue(spy); Exception lastException = null; proxy = getClient(addr, conf); @@ -1217,7 +1271,7 @@ public void testClientBackOffByResponseTime() throws Exception { GenericTestUtils.setLogLevel(DecayRpcScheduler.LOG, Level.DEBUG); GenericTestUtils.setLogLevel(RPC.LOG, Level.DEBUG); - final List> res = new ArrayList>(); + final List> res = new ArrayList<>(); final ExecutorService executorService = Executors.newFixedThreadPool(numClients); conf.setInt(CommonConfigurationKeys.IPC_CLIENT_CONNECT_MAX_RETRIES_KEY, 0); @@ -1225,10 +1279,8 @@ public void testClientBackOffByResponseTime() throws Exception { final String ns = CommonConfigurationKeys.IPC_NAMESPACE + ".0"; Server server = setupDecayRpcSchedulerandTestServer(ns + "."); - @SuppressWarnings("unchecked") - CallQueueManager spy = spy((CallQueueManager) Whitebox - .getInternalState(server, "callQueue")); - Whitebox.setInternalState(server, "callQueue", spy); + CallQueueManager spy = spy(server.getCallQueue()); + server.setCallQueue(spy); Exception lastException = null; proxy = getClient(addr, conf); @@ -1567,11 +1619,8 @@ public RpcStatusProto getRpcStatusProto() { RPC.Builder builder = newServerBuilder(conf) .setQueueSizePerHandler(1).setNumHandlers(1).setVerbose(true); server = setupTestServer(builder); - Whitebox.setInternalState( - server, "rpcRequestClass", FakeRequestClass.class); - MutableCounterLong authMetric = - (MutableCounterLong)Whitebox.getInternalState( - server.getRpcMetrics(), "rpcAuthorizationSuccesses"); + server.setRpcRequestClass(FakeRequestClass.class); + MutableCounterLong authMetric = server.getRpcMetrics().getRpcAuthorizationSuccesses(); proxy = getClient(addr, conf); boolean isDisconnected = true; diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestRetryCache.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestRetryCache.java index 64607deb908fa..b789ada5271ff 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestRetryCache.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestRetryCache.java @@ -50,14 +50,14 @@ public void setup() { static class TestServer { AtomicInteger retryCount = new AtomicInteger(); AtomicInteger operationCount = new AtomicInteger(); - private RetryCache retryCache = new RetryCache("TestRetryCache", 1, - 100 * 1000 * 1000 * 1000L); + private final RetryCache retryCache = new RetryCache( + "TestRetryCache", 1, 100 * 1000 * 1000 * 1000L); /** * A server method implemented using {@link RetryCache}. * * @param input is returned back in echo, if {@code success} is true. - * @param failureOuput returned on failure, if {@code success} is false. + * @param failureOutput returned on failure, if {@code success} is false. * @param methodTime time taken by the operation. By passing smaller/larger * value one can simulate an operation that takes short/long time. * @param success whether this operation completes successfully or not @@ -67,7 +67,7 @@ static class TestServer { int echo(int input, int failureOutput, long methodTime, boolean success) throws InterruptedException { CacheEntryWithPayload entry = RetryCache.waitForCompletion(retryCache, - null); + null, Server.getClientId(), Server.getCallId()); if (entry != null && entry.isSuccess()) { System.out.println("retryCount incremented " + retryCount.get()); retryCount.incrementAndGet(); @@ -173,16 +173,13 @@ public void testOperations(final int input, final int numberOfThreads, final int failureOutput = input + 1; ExecutorService executorService = Executors .newFixedThreadPool(numberOfThreads); - List> list = new ArrayList>(); + List> list = new ArrayList<>(); for (int i = 0; i < numberOfThreads; i++) { - Callable worker = new Callable() { - @Override - public Integer call() throws Exception { - Server.getCurCall().set(call); - Assert.assertEquals(Server.getCurCall().get(), call); - int randomPause = pause == 0 ? pause : r.nextInt(pause); - return testServer.echo(input, failureOutput, randomPause, success); - } + Callable worker = () -> { + Server.getCurCall().set(call); + Assert.assertEquals(Server.getCurCall().get(), call); + int randomPause = pause == 0 ? pause : r.nextInt(pause); + return testServer.echo(input, failureOutput, randomPause, success); }; Future submit = executorService.submit(worker); list.add(submit); diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestRpcBase.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestRpcBase.java index e9019e3d24eba..5b5c8bbaa9b73 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestRpcBase.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestRpcBase.java @@ -18,6 +18,8 @@ package org.apache.hadoop.ipc; +import org.apache.commons.lang3.builder.EqualsBuilder; +import org.apache.commons.lang3.builder.HashCodeBuilder; import org.apache.hadoop.thirdparty.protobuf.BlockingService; import org.apache.hadoop.thirdparty.protobuf.RpcController; import org.apache.hadoop.thirdparty.protobuf.ServiceException; @@ -26,6 +28,7 @@ import java.util.concurrent.locks.Lock; import java.util.concurrent.locks.ReentrantLock; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.ipc.Client.ConnectionId; import org.apache.hadoop.ipc.protobuf.TestProtos; import org.apache.hadoop.ipc.protobuf.TestRpcServiceProtos; import org.apache.hadoop.net.NetUtils; @@ -154,11 +157,54 @@ protected static TestRpcService getClient(InetSocketAddress serverAddr, } } - protected static void stop(Server server, TestRpcService proxy) { - if (proxy != null) { - try { - RPC.stopProxy(proxy); - } catch (Exception ignored) {} + /** + * Try to obtain a proxy of TestRpcService with an index. + * @param serverAddr input server address + * @param clientConf input client configuration + * @param retryPolicy input retryPolicy + * @param index input index + * @return one proxy of TestRpcService + */ + protected static TestRpcService getMultipleClientWithIndex(InetSocketAddress serverAddr, + Configuration clientConf, RetryPolicy retryPolicy, int index) + throws ServiceException, IOException { + MockConnectionId connectionId = new MockConnectionId(serverAddr, + TestRpcService.class, UserGroupInformation.getCurrentUser(), + RPC.getRpcTimeout(clientConf), retryPolicy, clientConf, index); + return getClient(connectionId, clientConf); + } + + /** + * Obtain a TestRpcService Proxy by a connectionId. + * @param connId input connectionId + * @param clientConf input configuration + * @return a TestRpcService Proxy + * @throws ServiceException a ServiceException + */ + protected static TestRpcService getClient(ConnectionId connId, + Configuration clientConf) throws ServiceException { + try { + return RPC.getProtocolProxy( + TestRpcService.class, + 0, + connId, + clientConf, + NetUtils.getDefaultSocketFactory(clientConf), + null).getProxy(); + } catch (IOException e) { + throw new ServiceException(e); + } + } + + protected static void stop(Server server, TestRpcService... proxies) { + if (proxies != null) { + for (TestRpcService proxy : proxies) { + if (proxy != null) { + try { + RPC.stopProxy(proxy); + } catch (Exception ignored) {} + } + } } if (server != null) { @@ -189,6 +235,40 @@ protected static int countThreads(String search) { return count; } + public static class MockConnectionId extends ConnectionId { + private static final int PRIME = 16777619; + private final int index; + + public MockConnectionId(InetSocketAddress address, Class protocol, + UserGroupInformation ticket, int rpcTimeout, RetryPolicy connectionRetryPolicy, + Configuration conf, int index) { + super(address, protocol, ticket, rpcTimeout, connectionRetryPolicy, conf); + this.index = index; + } + + @Override + public int hashCode() { + return new HashCodeBuilder() + .append(PRIME * super.hashCode()) + .append(this.index) + .toHashCode(); + } + + @Override + public boolean equals(Object obj) { + if (!super.equals(obj)) { + return false; + } + if (obj instanceof MockConnectionId) { + MockConnectionId other = (MockConnectionId)obj; + return new EqualsBuilder() + .append(this.index, other.index) + .isEquals(); + } + return false; + } + } + public static class TestTokenIdentifier extends TokenIdentifier { private Text tokenid; private Text realUser; diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/metrics2/impl/TestGraphiteMetrics.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/metrics2/impl/TestGraphiteMetrics.java deleted file mode 100644 index 743080acd7a5e..0000000000000 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/metrics2/impl/TestGraphiteMetrics.java +++ /dev/null @@ -1,215 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.metrics2.impl; - -import org.apache.hadoop.metrics2.AbstractMetric; -import org.apache.hadoop.metrics2.MetricsRecord; -import org.apache.hadoop.metrics2.MetricsTag; -import org.apache.hadoop.metrics2.sink.GraphiteSink; -import org.apache.hadoop.test.Whitebox; -import org.junit.Test; -import org.mockito.ArgumentCaptor; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.HashSet; -import java.util.List; -import java.util.Set; -import java.util.Collections; - -import static org.junit.Assert.assertEquals; -import static org.mockito.ArgumentMatchers.anyString; -import static org.mockito.ArgumentMatchers.eq; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.when; -import static org.mockito.Mockito.doThrow; -import static org.mockito.Mockito.reset; - - -public class TestGraphiteMetrics { - private AbstractMetric makeMetric(String name, Number value) { - AbstractMetric metric = mock(AbstractMetric.class); - when(metric.name()).thenReturn(name); - when(metric.value()).thenReturn(value); - return metric; - } - - private GraphiteSink.Graphite makeGraphite() { - GraphiteSink.Graphite mockGraphite = mock(GraphiteSink.Graphite.class); - when(mockGraphite.isConnected()).thenReturn(true); - return mockGraphite; - } - - @Test - public void testPutMetrics() { - GraphiteSink sink = new GraphiteSink(); - List tags = new ArrayList(); - tags.add(new MetricsTag(MsInfo.Context, "all")); - tags.add(new MetricsTag(MsInfo.Hostname, "host")); - Set metrics = new HashSet(); - metrics.add(makeMetric("foo1", 1.25)); - metrics.add(makeMetric("foo2", 2.25)); - MetricsRecord record = new MetricsRecordImpl(MsInfo.Context, (long) 10000, tags, metrics); - - ArgumentCaptor argument = ArgumentCaptor.forClass(String.class); - final GraphiteSink.Graphite mockGraphite = makeGraphite(); - Whitebox.setInternalState(sink, "graphite", mockGraphite); - sink.putMetrics(record); - - try { - verify(mockGraphite).write(argument.capture()); - } catch (IOException e) { - e.printStackTrace(); - } - - String result = argument.getValue(); - - assertEquals(true, - result.equals("null.all.Context.Context=all.Hostname=host.foo1 1.25 10\n" + - "null.all.Context.Context=all.Hostname=host.foo2 2.25 10\n") || - result.equals("null.all.Context.Context=all.Hostname=host.foo2 2.25 10\n" + - "null.all.Context.Context=all.Hostname=host.foo1 1.25 10\n")); - } - - @Test - public void testPutMetrics2() { - GraphiteSink sink = new GraphiteSink(); - List tags = new ArrayList(); - tags.add(new MetricsTag(MsInfo.Context, "all")); - tags.add(new MetricsTag(MsInfo.Hostname, null)); - Set metrics = new HashSet(); - metrics.add(makeMetric("foo1", 1)); - metrics.add(makeMetric("foo2", 2)); - MetricsRecord record = new MetricsRecordImpl(MsInfo.Context, (long) 10000, tags, metrics); - - - ArgumentCaptor argument = ArgumentCaptor.forClass(String.class); - final GraphiteSink.Graphite mockGraphite = makeGraphite(); - Whitebox.setInternalState(sink, "graphite", mockGraphite); - sink.putMetrics(record); - - try { - verify(mockGraphite).write(argument.capture()); - } catch (IOException e) { - e.printStackTrace(); - } - - String result = argument.getValue(); - - assertEquals(true, - result.equals("null.all.Context.Context=all.foo1 1 10\n" + - "null.all.Context.Context=all.foo2 2 10\n") || - result.equals("null.all.Context.Context=all.foo2 2 10\n" + - "null.all.Context.Context=all.foo1 1 10\n")); - } - - /** - * Assert that timestamps are converted correctly, ticket HADOOP-11182 - */ - @Test - public void testPutMetrics3() { - - // setup GraphiteSink - GraphiteSink sink = new GraphiteSink(); - final GraphiteSink.Graphite mockGraphite = makeGraphite(); - Whitebox.setInternalState(sink, "graphite", mockGraphite); - - // given two metrics records with timestamps 1000 milliseconds apart. - List tags = Collections.emptyList(); - Set metrics = new HashSet(); - metrics.add(makeMetric("foo1", 1)); - MetricsRecord record1 = new MetricsRecordImpl(MsInfo.Context, 1000000000000L, tags, metrics); - MetricsRecord record2 = new MetricsRecordImpl(MsInfo.Context, 1000000001000L, tags, metrics); - - sink.putMetrics(record1); - sink.putMetrics(record2); - - sink.flush(); - try { - sink.close(); - } catch(IOException e) { - e.printStackTrace(); - } - - // then the timestamps in the graphite stream should differ by one second. - try { - verify(mockGraphite).write(eq("null.default.Context.foo1 1 1000000000\n")); - verify(mockGraphite).write(eq("null.default.Context.foo1 1 1000000001\n")); - } catch (IOException e) { - e.printStackTrace(); - } - } - - @Test - public void testFailureAndPutMetrics() throws IOException { - GraphiteSink sink = new GraphiteSink(); - List tags = new ArrayList(); - tags.add(new MetricsTag(MsInfo.Context, "all")); - tags.add(new MetricsTag(MsInfo.Hostname, "host")); - Set metrics = new HashSet(); - metrics.add(makeMetric("foo1", 1.25)); - metrics.add(makeMetric("foo2", 2.25)); - MetricsRecord record = new MetricsRecordImpl(MsInfo.Context, (long) 10000, tags, metrics); - - final GraphiteSink.Graphite mockGraphite = makeGraphite(); - Whitebox.setInternalState(sink, "graphite", mockGraphite); - - // throw exception when first try - doThrow(new IOException("IO exception")).when(mockGraphite).write(anyString()); - - sink.putMetrics(record); - verify(mockGraphite).write(anyString()); - verify(mockGraphite).close(); - - // reset mock and try again - reset(mockGraphite); - when(mockGraphite.isConnected()).thenReturn(false); - - ArgumentCaptor argument = ArgumentCaptor.forClass(String.class); - sink.putMetrics(record); - - verify(mockGraphite).write(argument.capture()); - String result = argument.getValue(); - - assertEquals(true, - result.equals("null.all.Context.Context=all.Hostname=host.foo1 1.25 10\n" + - "null.all.Context.Context=all.Hostname=host.foo2 2.25 10\n") || - result.equals("null.all.Context.Context=all.Hostname=host.foo2 2.25 10\n" + - "null.all.Context.Context=all.Hostname=host.foo1 1.25 10\n")); - } - - @Test - public void testClose(){ - GraphiteSink sink = new GraphiteSink(); - final GraphiteSink.Graphite mockGraphite = makeGraphite(); - Whitebox.setInternalState(sink, "graphite", mockGraphite); - try { - sink.close(); - } catch (IOException ioe) { - ioe.printStackTrace(); - } - - try { - verify(mockGraphite).close(); - } catch (IOException ioe) { - ioe.printStackTrace(); - } - } -} diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/metrics2/lib/TestMutableMetrics.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/metrics2/lib/TestMutableMetrics.java index 5d20abdd8bf10..10c8057c69e04 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/metrics2/lib/TestMutableMetrics.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/metrics2/lib/TestMutableMetrics.java @@ -29,6 +29,8 @@ import static org.mockito.Mockito.verify; import static org.junit.Assert.*; +import java.util.ArrayList; +import java.util.List; import java.util.Map; import java.util.Map.Entry; import java.util.Random; @@ -36,6 +38,7 @@ import org.apache.hadoop.metrics2.MetricsRecordBuilder; import org.apache.hadoop.metrics2.util.Quantile; +import org.apache.hadoop.thirdparty.com.google.common.math.Stats; import org.junit.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -47,7 +50,7 @@ public class TestMutableMetrics { private static final Logger LOG = LoggerFactory.getLogger(TestMutableMetrics.class); - private final double EPSILON = 1e-42; + private static final double EPSILON = 1e-42; /** * Test the snapshot method @@ -306,19 +309,56 @@ public void testDuplicateMetrics() { /** * Tests that when using {@link MutableStat#add(long, long)}, even with a high - * sample count, the mean does not lose accuracy. + * sample count, the mean does not lose accuracy. This also validates that + * the std dev is correct, assuming samples of equal value. */ - @Test public void testMutableStatWithBulkAdd() { + @Test + public void testMutableStatWithBulkAdd() { + List samples = new ArrayList<>(); + for (int i = 0; i < 1000; i++) { + samples.add(1000L); + } + for (int i = 0; i < 1000; i++) { + samples.add(2000L); + } + Stats stats = Stats.of(samples); + + for (int bulkSize : new int[] {1, 10, 100, 1000}) { + MetricsRecordBuilder rb = mockMetricsRecordBuilder(); + MetricsRegistry registry = new MetricsRegistry("test"); + MutableStat stat = registry.newStat("Test", "Test", "Ops", "Val", true); + + for (int i = 0; i < samples.size(); i += bulkSize) { + stat.add(bulkSize, samples + .subList(i, i + bulkSize) + .stream() + .mapToLong(Long::longValue) + .sum() + ); + } + registry.snapshot(rb, false); + + assertCounter("TestNumOps", 2000L, rb); + assertGauge("TestAvgVal", stats.mean(), rb); + assertGauge("TestStdevVal", stats.sampleStandardDeviation(), rb); + } + } + + @Test + public void testLargeMutableStatAdd() { MetricsRecordBuilder rb = mockMetricsRecordBuilder(); MetricsRegistry registry = new MetricsRegistry("test"); - MutableStat stat = registry.newStat("Test", "Test", "Ops", "Val", false); + MutableStat stat = registry.newStat("Test", "Test", "Ops", "Val", true); - stat.add(1000, 1000); - stat.add(1000, 2000); + long sample = 1000000000000009L; + for (int i = 0; i < 100; i++) { + stat.add(1, sample); + } registry.snapshot(rb, false); - assertCounter("TestNumOps", 2000L, rb); - assertGauge("TestAvgVal", 1.5, rb); + assertCounter("TestNumOps", 100L, rb); + assertGauge("TestAvgVal", (double) sample, rb); + assertGauge("TestStdevVal", 0.0, rb); } /** diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/metrics2/sink/TestGraphiteMetrics.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/metrics2/sink/TestGraphiteMetrics.java new file mode 100644 index 0000000000000..9ea81c6e4c62e --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/metrics2/sink/TestGraphiteMetrics.java @@ -0,0 +1,219 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.metrics2.sink; + +import org.apache.hadoop.metrics2.AbstractMetric; +import org.apache.hadoop.metrics2.MetricsRecord; +import org.apache.hadoop.metrics2.MetricsTag; +import org.apache.hadoop.metrics2.impl.MetricsRecordImpl; +import org.apache.hadoop.metrics2.impl.MsInfo; +import org.junit.Test; +import org.mockito.ArgumentCaptor; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Set; +import java.util.Collections; + +import static org.junit.Assert.assertEquals; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; +import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.reset; + + +public class TestGraphiteMetrics { + private AbstractMetric makeMetric(String name, Number value) { + AbstractMetric metric = mock(AbstractMetric.class); + when(metric.name()).thenReturn(name); + when(metric.value()).thenReturn(value); + return metric; + } + + private GraphiteSink.Graphite makeGraphite() { + GraphiteSink.Graphite mockGraphite = mock(GraphiteSink.Graphite.class); + when(mockGraphite.isConnected()).thenReturn(true); + return mockGraphite; + } + + @Test + public void testPutMetrics() { + GraphiteSink sink = new GraphiteSink(); + List tags = new ArrayList<>(); + tags.add(new MetricsTag(MsInfo.Context, "all")); + tags.add(new MetricsTag(MsInfo.Hostname, "host")); + Set metrics = new HashSet<>(); + metrics.add(makeMetric("foo1", 1.25)); + metrics.add(makeMetric("foo2", 2.25)); + MetricsRecord record = + new MetricsRecordImpl(MsInfo.Context, 10000, tags, metrics); + + ArgumentCaptor argument = ArgumentCaptor.forClass(String.class); + final GraphiteSink.Graphite mockGraphite = makeGraphite(); + sink.setGraphite(mockGraphite); + sink.putMetrics(record); + + try { + verify(mockGraphite).write(argument.capture()); + } catch (IOException e) { + e.printStackTrace(); + } + + String result = argument.getValue(); + + assertEquals(true, + result.equals("null.all.Context.Context=all.Hostname=host.foo1 1.25 10\n" + + "null.all.Context.Context=all.Hostname=host.foo2 2.25 10\n") || + result.equals("null.all.Context.Context=all.Hostname=host.foo2 2.25 10\n" + + "null.all.Context.Context=all.Hostname=host.foo1 1.25 10\n")); + } + + @Test + public void testPutMetrics2() throws IllegalAccessException { + GraphiteSink sink = new GraphiteSink(); + List tags = new ArrayList<>(); + tags.add(new MetricsTag(MsInfo.Context, "all")); + tags.add(new MetricsTag(MsInfo.Hostname, null)); + Set metrics = new HashSet<>(); + metrics.add(makeMetric("foo1", 1)); + metrics.add(makeMetric("foo2", 2)); + MetricsRecord record = + new MetricsRecordImpl(MsInfo.Context, 10000, tags, metrics); + + ArgumentCaptor argument = ArgumentCaptor.forClass(String.class); + final GraphiteSink.Graphite mockGraphite = makeGraphite(); + sink.setGraphite(mockGraphite); + sink.putMetrics(record); + + try { + verify(mockGraphite).write(argument.capture()); + } catch (IOException e) { + e.printStackTrace(); + } + + String result = argument.getValue(); + + assertEquals(true, + result.equals("null.all.Context.Context=all.foo1 1 10\n" + + "null.all.Context.Context=all.foo2 2 10\n") || + result.equals("null.all.Context.Context=all.foo2 2 10\n" + + "null.all.Context.Context=all.foo1 1 10\n")); + } + + /** + * Assert that timestamps are converted correctly, ticket HADOOP-11182. + */ + @Test + public void testPutMetrics3() throws IllegalAccessException { + + // setup GraphiteSink + GraphiteSink sink = new GraphiteSink(); + final GraphiteSink.Graphite mockGraphite = makeGraphite(); + sink.setGraphite(mockGraphite); + + // given two metrics records with timestamps 1000 milliseconds apart. + List tags = Collections.emptyList(); + Set metrics = new HashSet<>(); + metrics.add(makeMetric("foo1", 1)); + MetricsRecord record1 = + new MetricsRecordImpl(MsInfo.Context, 1000000000000L, tags, metrics); + MetricsRecord record2 = + new MetricsRecordImpl(MsInfo.Context, 1000000001000L, tags, metrics); + + sink.putMetrics(record1); + sink.putMetrics(record2); + + sink.flush(); + try { + sink.close(); + } catch(IOException e) { + e.printStackTrace(); + } + + // then the timestamps in the graphite stream should differ by one second. + try { + verify(mockGraphite).write(eq("null.default.Context.foo1 1 1000000000\n")); + verify(mockGraphite).write(eq("null.default.Context.foo1 1 1000000001\n")); + } catch (IOException e) { + e.printStackTrace(); + } + } + + @Test + public void testFailureAndPutMetrics() throws IOException, IllegalAccessException { + GraphiteSink sink = new GraphiteSink(); + List tags = new ArrayList<>(); + tags.add(new MetricsTag(MsInfo.Context, "all")); + tags.add(new MetricsTag(MsInfo.Hostname, "host")); + Set metrics = new HashSet<>(); + metrics.add(makeMetric("foo1", 1.25)); + metrics.add(makeMetric("foo2", 2.25)); + MetricsRecord record = + new MetricsRecordImpl(MsInfo.Context, 10000, tags, metrics); + + final GraphiteSink.Graphite mockGraphite = makeGraphite(); + sink.setGraphite(mockGraphite); + + // throw exception when first try + doThrow(new IOException("IO exception")).when(mockGraphite).write(anyString()); + + sink.putMetrics(record); + verify(mockGraphite).write(anyString()); + verify(mockGraphite).close(); + + // reset mock and try again + reset(mockGraphite); + when(mockGraphite.isConnected()).thenReturn(false); + + ArgumentCaptor argument = ArgumentCaptor.forClass(String.class); + sink.putMetrics(record); + + verify(mockGraphite).write(argument.capture()); + String result = argument.getValue(); + + assertEquals(true, + result.equals("null.all.Context.Context=all.Hostname=host.foo1 1.25 10\n" + + "null.all.Context.Context=all.Hostname=host.foo2 2.25 10\n") || + result.equals("null.all.Context.Context=all.Hostname=host.foo2 2.25 10\n" + + "null.all.Context.Context=all.Hostname=host.foo1 1.25 10\n")); + } + + @Test + public void testClose() throws IllegalAccessException { + GraphiteSink sink = new GraphiteSink(); + final GraphiteSink.Graphite mockGraphite = makeGraphite(); + sink.setGraphite(mockGraphite); + try { + sink.close(); + } catch (IOException ioe) { + ioe.printStackTrace(); + } + + try { + verify(mockGraphite).close(); + } catch (IOException ioe) { + ioe.printStackTrace(); + } + } +} diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/metrics2/impl/TestStatsDMetrics.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/metrics2/sink/TestStatsDMetrics.java similarity index 91% rename from hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/metrics2/impl/TestStatsDMetrics.java rename to hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/metrics2/sink/TestStatsDMetrics.java index 4cf4894ff8352..99a75787ad841 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/metrics2/impl/TestStatsDMetrics.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/metrics2/sink/TestStatsDMetrics.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package org.apache.hadoop.metrics2.impl; +package org.apache.hadoop.metrics2.sink; import static org.junit.Assert.assertTrue; import static org.mockito.Mockito.mock; @@ -35,9 +35,9 @@ import org.apache.hadoop.metrics2.MetricType; import org.apache.hadoop.metrics2.MetricsRecord; import org.apache.hadoop.metrics2.MetricsTag; -import org.apache.hadoop.metrics2.sink.StatsDSink; +import org.apache.hadoop.metrics2.impl.MetricsRecordImpl; +import org.apache.hadoop.metrics2.impl.MsInfo; import org.apache.hadoop.metrics2.sink.StatsDSink.StatsD; -import org.apache.hadoop.test.Whitebox; import org.junit.Test; public class TestStatsDMetrics { @@ -52,7 +52,7 @@ private AbstractMetric makeMetric(String name, Number value, } @Test(timeout=3000) - public void testPutMetrics() throws IOException, InterruptedException { + public void testPutMetrics() throws IOException, IllegalAccessException { final StatsDSink sink = new StatsDSink(); List tags = new ArrayList(); tags.add(new MetricsTag(MsInfo.Hostname, "host")); @@ -69,7 +69,7 @@ public void testPutMetrics() throws IOException, InterruptedException { final StatsDSink.StatsD mockStatsD = new StatsD(sock.getLocalAddress().getHostName(), sock.getLocalPort()); - Whitebox.setInternalState(sink, "statsd", mockStatsD); + sink.setStatsd(mockStatsD); final DatagramPacket p = new DatagramPacket(new byte[8192], 8192); sink.putMetrics(record); sock.receive(p); @@ -87,7 +87,7 @@ public void testPutMetrics() throws IOException, InterruptedException { } @Test(timeout=3000) - public void testPutMetrics2() throws IOException { + public void testPutMetrics2() throws IOException, IllegalAccessException { StatsDSink sink = new StatsDSink(); List tags = new ArrayList(); tags.add(new MetricsTag(MsInfo.Hostname, null)); @@ -104,7 +104,7 @@ public void testPutMetrics2() throws IOException { final StatsDSink.StatsD mockStatsD = new StatsD(sock.getLocalAddress().getHostName(), sock.getLocalPort()); - Whitebox.setInternalState(sink, "statsd", mockStatsD); + sink.setStatsd(mockStatsD); final DatagramPacket p = new DatagramPacket(new byte[8192], 8192); sink.putMetrics(record); sock.receive(p); diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/metrics2/sink/ganglia/TestGangliaSink.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/metrics2/sink/ganglia/TestGangliaSink.java index 30e8961ef60e7..59ba18803f6bc 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/metrics2/sink/ganglia/TestGangliaSink.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/metrics2/sink/ganglia/TestGangliaSink.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -20,6 +20,7 @@ import org.apache.commons.configuration2.SubsetConfiguration; import org.apache.hadoop.metrics2.impl.ConfigBuilder; + import org.junit.Test; import java.net.DatagramSocket; @@ -30,52 +31,61 @@ import static org.junit.Assert.assertTrue; public class TestGangliaSink { - @Test - public void testShouldCreateDatagramSocketByDefault() throws Exception { - SubsetConfiguration conf = new ConfigBuilder() - .subset("test.sink.ganglia"); + @Test + public void testShouldCreateDatagramSocketByDefault() throws Exception { + SubsetConfiguration conf = new ConfigBuilder().subset("test.sink.ganglia"); + + GangliaSink30 gangliaSink = new GangliaSink30(); + gangliaSink.init(conf); + DatagramSocket socket = gangliaSink.getDatagramSocket(); + assertFalse("Did not create DatagramSocket", + socket == null || socket instanceof MulticastSocket); + } - GangliaSink30 gangliaSink = new GangliaSink30(); - gangliaSink.init(conf); - DatagramSocket socket = gangliaSink.getDatagramSocket(); - assertFalse("Did not create DatagramSocket", socket == null || socket instanceof MulticastSocket); - } + @Test + public void testShouldCreateDatagramSocketIfMulticastIsDisabled() throws Exception { + SubsetConfiguration conf = + new ConfigBuilder().add("test.sink.ganglia.multicast", false).subset("test.sink.ganglia"); + GangliaSink30 gangliaSink = new GangliaSink30(); + gangliaSink.init(conf); + DatagramSocket socket = gangliaSink.getDatagramSocket(); + assertFalse("Did not create DatagramSocket", + socket == null || socket instanceof MulticastSocket); + } - @Test - public void testShouldCreateDatagramSocketIfMulticastIsDisabled() throws Exception { - SubsetConfiguration conf = new ConfigBuilder() - .add("test.sink.ganglia.multicast", false) - .subset("test.sink.ganglia"); - GangliaSink30 gangliaSink = new GangliaSink30(); - gangliaSink.init(conf); - DatagramSocket socket = gangliaSink.getDatagramSocket(); - assertFalse("Did not create DatagramSocket", socket == null || socket instanceof MulticastSocket); - } + @Test + public void testShouldCreateMulticastSocket() throws Exception { + SubsetConfiguration conf = + new ConfigBuilder().add("test.sink.ganglia.multicast", true).subset("test.sink.ganglia"); + GangliaSink30 gangliaSink = new GangliaSink30(); + gangliaSink.init(conf); + DatagramSocket socket = gangliaSink.getDatagramSocket(); + assertTrue("Did not create MulticastSocket", + socket != null && socket instanceof MulticastSocket); + int ttl = ((MulticastSocket) socket).getTimeToLive(); + assertEquals("Did not set default TTL", 1, ttl); + } - @Test - public void testShouldCreateMulticastSocket() throws Exception { - SubsetConfiguration conf = new ConfigBuilder() - .add("test.sink.ganglia.multicast", true) - .subset("test.sink.ganglia"); - GangliaSink30 gangliaSink = new GangliaSink30(); - gangliaSink.init(conf); - DatagramSocket socket = gangliaSink.getDatagramSocket(); - assertTrue("Did not create MulticastSocket", socket != null && socket instanceof MulticastSocket); - int ttl = ((MulticastSocket) socket).getTimeToLive(); - assertEquals("Did not set default TTL", 1, ttl); - } + @Test + public void testShouldSetMulticastSocketTtl() throws Exception { + SubsetConfiguration conf = new ConfigBuilder().add("test.sink.ganglia.multicast", true) + .add("test.sink.ganglia.multicast.ttl", 3).subset("test.sink.ganglia"); + GangliaSink30 gangliaSink = new GangliaSink30(); + gangliaSink.init(conf); + DatagramSocket socket = gangliaSink.getDatagramSocket(); + assertTrue("Did not create MulticastSocket", + socket != null && socket instanceof MulticastSocket); + int ttl = ((MulticastSocket) socket).getTimeToLive(); + assertEquals("Did not set TTL", 3, ttl); + } - @Test - public void testShouldSetMulticastSocketTtl() throws Exception { - SubsetConfiguration conf = new ConfigBuilder() - .add("test.sink.ganglia.multicast", true) - .add("test.sink.ganglia.multicast.ttl", 3) - .subset("test.sink.ganglia"); - GangliaSink30 gangliaSink = new GangliaSink30(); - gangliaSink.init(conf); - DatagramSocket socket = gangliaSink.getDatagramSocket(); - assertTrue("Did not create MulticastSocket", socket != null && socket instanceof MulticastSocket); - int ttl = ((MulticastSocket) socket).getTimeToLive(); - assertEquals("Did not set TTL", 3, ttl); - } + @Test + public void testMultipleMetricsServers() { + SubsetConfiguration conf = + new ConfigBuilder().add("test.sink.ganglia.servers", "server1,server2") + .subset("test.sink.ganglia"); + GangliaSink30 gangliaSink = new GangliaSink30(); + gangliaSink.init(conf); + assertEquals(2, gangliaSink.getMetricsServers().size()); + } } diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/net/TestDNSDomainNameResolver.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/net/TestDNSDomainNameResolver.java new file mode 100644 index 0000000000000..4729cee118818 --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/net/TestDNSDomainNameResolver.java @@ -0,0 +1,54 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.net; + +import org.junit.Test; + +import java.net.InetAddress; +import java.net.UnknownHostException; +import java.util.Objects; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotEquals; +import static org.junit.Assume.assumeFalse; + + +public class TestDNSDomainNameResolver { + + static DNSDomainNameResolver DNR = new DNSDomainNameResolver(); + + @Test + public void testGetHostNameByIP() throws UnknownHostException { + InetAddress localhost = InetAddress.getLocalHost(); + assumeFalse("IP lookup support required", + Objects.equals(localhost.getCanonicalHostName(), localhost.getHostAddress())); + + // Precondition: host name and canonical host name for unresolved returns an IP address. + InetAddress unresolved = InetAddress.getByAddress(localhost.getHostAddress(), + localhost.getAddress()); + assertEquals(localhost.getHostAddress(), unresolved.getHostName()); + + // Test: Get the canonical name despite InetAddress caching + String canonicalHostName = DNR.getHostnameByIP(unresolved); + + // Verify: The canonical host name doesn't match the host address but does match the localhost. + assertNotEquals(localhost.getHostAddress(), canonicalHostName); + assertEquals(localhost.getCanonicalHostName(), canonicalHostName); + } + +} diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestLdapGroupsMapping.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestLdapGroupsMapping.java index aba3997187747..82e80fd9fa504 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestLdapGroupsMapping.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestLdapGroupsMapping.java @@ -18,6 +18,7 @@ package org.apache.hadoop.security; import static org.apache.hadoop.security.LdapGroupsMapping.CONNECTION_TIMEOUT; +import static org.apache.hadoop.security.LdapGroupsMapping.GROUP_SEARCH_FILTER_PATTERN; import static org.apache.hadoop.security.LdapGroupsMapping.LDAP_NUM_ATTEMPTS_KEY; import static org.apache.hadoop.security.LdapGroupsMapping.READ_TIMEOUT; import static org.apache.hadoop.test.GenericTestUtils.assertExceptionContains; @@ -27,6 +28,8 @@ import static org.junit.Assert.fail; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.mock; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; @@ -44,6 +47,8 @@ import javax.naming.CommunicationException; import javax.naming.NamingException; +import javax.naming.directory.Attribute; +import javax.naming.directory.Attributes; import javax.naming.directory.SearchControls; import org.apache.hadoop.conf.Configuration; @@ -120,6 +125,49 @@ public void testGetGroupsWithDefaultBaseDN() throws Exception { doTestGetGroupsWithBaseDN(conf, baseDN.trim(), baseDN.trim()); } + @Test + public void testGetGroupsWithDynamicGroupFilter() throws Exception { + // Set basic mock stuff. + Configuration conf = getBaseConf(TEST_LDAP_URL); + String baseDN = "dc=xxx,dc=com"; + conf.set(LdapGroupsMapping.BASE_DN_KEY, baseDN); + Attributes attributes = getAttributes(); + + // Set the groupFilter conf to take the csv. + conf.set(GROUP_SEARCH_FILTER_PATTERN, "userDN,userName"); + + // Set the value for userName attribute that is to be used as part of the + // group filter at argument 1. + final String userName = "some_user"; + Attribute userNameAttr = mock(Attribute.class); + when(userNameAttr.get()).thenReturn(userName); + when(attributes.get(eq("userName"))).thenReturn(userNameAttr); + + // Set the dynamic group search filter. + final String groupSearchFilter = + "(|(memberUid={0})(uname={1}))" + "(objectClass=group)"; + conf.set(LdapGroupsMapping.GROUP_SEARCH_FILTER_KEY, groupSearchFilter); + + final LdapGroupsMapping groupsMapping = getGroupsMapping(); + groupsMapping.setConf(conf); + + // The group search filter should be resolved and should be passed as the + // below. + String groupFilter = "(|(memberUid={0})(uname={1}))(objectClass=group)"; + String[] resolvedFilterArgs = + new String[] {"CN=some_user,DC=test,DC=com", "some_user"}; + + // Return groups only if the resolved filter is passed. + when(getContext() + .search(anyString(), eq(groupFilter), eq(resolvedFilterArgs), + any(SearchControls.class))) + .thenReturn(getUserNames(), getGroupNames()); + + // Check the group filter got resolved and get the desired values. + List groups = groupsMapping.getGroups(userName); + Assert.assertEquals(Arrays.asList(getTestGroups()), groups); + } + /** * Helper method to do the LDAP getGroups operation using given user base DN * and group base DN. diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestLdapGroupsMappingWithOneQuery.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestLdapGroupsMappingWithOneQuery.java index 7ae802e26d36d..8686d5c6e3b46 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestLdapGroupsMappingWithOneQuery.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestLdapGroupsMappingWithOneQuery.java @@ -18,19 +18,22 @@ package org.apache.hadoop.security; +import java.util.ArrayList; import java.util.Arrays; import java.util.List; +import java.util.Set; import javax.naming.NamingEnumeration; import javax.naming.NamingException; import javax.naming.directory.Attribute; +import javax.naming.directory.DirContext; import javax.naming.directory.SearchControls; import javax.naming.directory.SearchResult; import org.apache.hadoop.conf.Configuration; import org.junit.Assert; -import org.junit.Before; import org.junit.Test; +import org.mockito.stubbing.Stubber; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyString; @@ -49,48 +52,121 @@ public class TestLdapGroupsMappingWithOneQuery extends TestLdapGroupsMappingBase { - @Before - public void setupMocks() throws NamingException { + public void setupMocks(List listOfDNs) throws NamingException { Attribute groupDN = mock(Attribute.class); NamingEnumeration groupNames = getGroupNames(); doReturn(groupNames).when(groupDN).getAll(); - String groupName1 = "CN=abc,DC=foo,DC=bar,DC=com"; - String groupName2 = "CN=xyz,DC=foo,DC=bar,DC=com"; - String groupName3 = "CN=sss,CN=foo,DC=bar,DC=com"; - doReturn(groupName1).doReturn(groupName2).doReturn(groupName3). - when(groupNames).next(); - when(groupNames.hasMore()).thenReturn(true).thenReturn(true). - thenReturn(true).thenReturn(false); + buildListOfGroupDNs(listOfDNs).when(groupNames).next(); + when(groupNames.hasMore()). + thenReturn(true).thenReturn(true). + thenReturn(true).thenReturn(false); when(getAttributes().get(eq("memberOf"))).thenReturn(groupDN); } + /** + * Build and return a list of individually added group DNs such + * that calls to .next() will result in a single value each time. + * + * @param listOfDNs + * @return the stubber to use for the .when().next() call + */ + private Stubber buildListOfGroupDNs(List listOfDNs) { + Stubber stubber = null; + for (String s : listOfDNs) { + if (stubber != null) { + stubber.doReturn(s); + } else { + stubber = doReturn(s); + } + } + return stubber; + } + @Test public void testGetGroups() throws NamingException { // given a user whose ldap query returns a user object with three "memberOf" // properties, return an array of strings representing its groups. String[] testGroups = new String[] {"abc", "xyz", "sss"}; doTestGetGroups(Arrays.asList(testGroups)); + + // test fallback triggered by NamingException + doTestGetGroupsWithFallback(); } private void doTestGetGroups(List expectedGroups) throws NamingException { + List groupDns = new ArrayList<>(); + groupDns.add("CN=abc,DC=foo,DC=bar,DC=com"); + groupDns.add("CN=xyz,DC=foo,DC=bar,DC=com"); + groupDns.add("CN=sss,DC=foo,DC=bar,DC=com"); + + setupMocks(groupDns); String ldapUrl = "ldap://test"; Configuration conf = getBaseConf(ldapUrl); // enable single-query lookup conf.set(LdapGroupsMapping.MEMBEROF_ATTR_KEY, "memberOf"); - LdapGroupsMapping groupsMapping = getGroupsMapping(); + TestLdapGroupsMapping groupsMapping = new TestLdapGroupsMapping(); groupsMapping.setConf(conf); // Username is arbitrary, since the spy is mocked to respond the same, // regardless of input List groups = groupsMapping.getGroups("some_user"); Assert.assertEquals(expectedGroups, groups); + Assert.assertFalse("Second LDAP query should NOT have been called.", + groupsMapping.isSecondaryQueryCalled()); // We should have only made one query because single-query lookup is enabled verify(getContext(), times(1)).search(anyString(), anyString(), any(Object[].class), any(SearchControls.class)); } -} \ No newline at end of file + + private void doTestGetGroupsWithFallback() + throws NamingException { + List groupDns = new ArrayList<>(); + groupDns.add("CN=abc,DC=foo,DC=bar,DC=com"); + groupDns.add("CN=xyz,DC=foo,DC=bar,DC=com"); + groupDns.add("ipaUniqueID=e4a9a634-bb24-11ec-aec1-06ede52b5fe1," + + "CN=sudo,DC=foo,DC=bar,DC=com"); + setupMocks(groupDns); + String ldapUrl = "ldap://test"; + Configuration conf = getBaseConf(ldapUrl); + // enable single-query lookup + conf.set(LdapGroupsMapping.MEMBEROF_ATTR_KEY, "memberOf"); + conf.set(LdapGroupsMapping.LDAP_NUM_ATTEMPTS_KEY, "1"); + + TestLdapGroupsMapping groupsMapping = new TestLdapGroupsMapping(); + groupsMapping.setConf(conf); + // Username is arbitrary, since the spy is mocked to respond the same, + // regardless of input + List groups = groupsMapping.getGroups("some_user"); + + // expected to be empty due to invalid memberOf + Assert.assertEquals(0, groups.size()); + + // expect secondary query to be called: getGroups() + Assert.assertTrue("Second LDAP query should have been called.", + groupsMapping.isSecondaryQueryCalled()); + + // We should have fallen back to the second query because first threw + // NamingException expected count is 3 since testGetGroups calls + // doTestGetGroups and doTestGetGroupsWithFallback in succession and + // the count is across both test scenarios. + verify(getContext(), times(3)).search(anyString(), anyString(), + any(Object[].class), any(SearchControls.class)); + } + + private static final class TestLdapGroupsMapping extends LdapGroupsMapping { + private boolean secondaryQueryCalled = false; + public boolean isSecondaryQueryCalled() { + return secondaryQueryCalled; + } + Set lookupGroup(SearchResult result, DirContext c, + int goUpHierarchy) throws NamingException { + secondaryQueryCalled = true; + return super.lookupGroup(result, c, goUpHierarchy); + } + } +} diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/token/delegation/TestZKDelegationTokenSecretManager.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/token/delegation/TestZKDelegationTokenSecretManager.java index 63d01ccab9e47..e92a25ea0ed8f 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/token/delegation/TestZKDelegationTokenSecretManager.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/token/delegation/TestZKDelegationTokenSecretManager.java @@ -28,6 +28,8 @@ import org.apache.curator.framework.CuratorFramework; import org.apache.curator.framework.CuratorFrameworkFactory; import org.apache.curator.framework.api.ACLProvider; +import org.apache.curator.framework.api.CreateBuilder; +import org.apache.curator.framework.api.ProtectACLCreateModeStatPathAndBytesable; import org.apache.curator.retry.ExponentialBackoffRetry; import org.apache.curator.test.TestingServer; import org.apache.hadoop.conf.Configuration; @@ -38,9 +40,12 @@ import org.apache.hadoop.security.token.delegation.web.DelegationTokenIdentifier; import org.apache.hadoop.security.token.delegation.web.DelegationTokenManager; import org.apache.hadoop.test.GenericTestUtils; +import org.apache.hadoop.test.LambdaTestUtils; +import org.apache.zookeeper.KeeperException; import org.apache.zookeeper.ZooDefs; import org.apache.zookeeper.data.ACL; import org.apache.zookeeper.data.Id; +import org.apache.zookeeper.data.Stat; import org.apache.zookeeper.server.auth.DigestAuthenticationProvider; import org.junit.After; import org.junit.Assert; @@ -507,4 +512,65 @@ public Boolean get() { } }, 1000, 5000); } + + @Test + public void testCreatingParentContainersIfNeeded() throws Exception { + + String connectString = zkServer.getConnectString(); + RetryPolicy retryPolicy = new ExponentialBackoffRetry(1000, 3); + Configuration conf = getSecretConf(connectString); + CuratorFramework curatorFramework = + CuratorFrameworkFactory.builder() + .connectString(connectString) + .retryPolicy(retryPolicy) + .build(); + curatorFramework.start(); + ZKDelegationTokenSecretManager.setCurator(curatorFramework); + DelegationTokenManager tm1 = new DelegationTokenManager(conf, new Text("foo")); + + // When the init method is called, + // the ZKDelegationTokenSecretManager#startThread method will be called, + // and the creatingParentContainersIfNeeded will be called to create the nameSpace. + tm1.init(); + + String workingPath = "/" + conf.get(ZKDelegationTokenSecretManager.ZK_DTSM_ZNODE_WORKING_PATH, + ZKDelegationTokenSecretManager.ZK_DTSM_ZNODE_WORKING_PATH_DEAFULT) + "/ZKDTSMRoot"; + + // Check if the created NameSpace exists. + Stat stat = curatorFramework.checkExists().forPath(workingPath); + Assert.assertNotNull(stat); + + tm1.destroy(); + curatorFramework.close(); + } + + @Test + public void testCreateNameSpaceRepeatedly() throws Exception { + + String connectString = zkServer.getConnectString(); + RetryPolicy retryPolicy = new ExponentialBackoffRetry(1000, 3); + Configuration conf = getSecretConf(connectString); + CuratorFramework curatorFramework = + CuratorFrameworkFactory.builder(). + connectString(connectString). + retryPolicy(retryPolicy). + build(); + curatorFramework.start(); + + String workingPath = "/" + conf.get(ZKDelegationTokenSecretManager.ZK_DTSM_ZNODE_WORKING_PATH, + ZKDelegationTokenSecretManager.ZK_DTSM_ZNODE_WORKING_PATH_DEAFULT) + "/ZKDTSMRoot-Test"; + CreateBuilder createBuilder = curatorFramework.create(); + ProtectACLCreateModeStatPathAndBytesable createModeStat = + createBuilder.creatingParentContainersIfNeeded(); + createModeStat.forPath(workingPath); + + // Check if the created NameSpace exists. + Stat stat = curatorFramework.checkExists().forPath(workingPath); + Assert.assertNotNull(stat); + + // Repeated creation will throw NodeExists exception + LambdaTestUtils.intercept(KeeperException.class, + "KeeperErrorCode = NodeExists for "+workingPath, + () -> createModeStat.forPath(workingPath)); + } } diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestExitUtil.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestExitUtil.java new file mode 100644 index 0000000000000..58a1997e9bc59 --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestExitUtil.java @@ -0,0 +1,127 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.util; + +import static org.apache.hadoop.test.LambdaTestUtils.intercept; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertSame; +import static org.junit.Assert.assertTrue; + +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import org.apache.hadoop.util.ExitUtil.ExitException; +import org.apache.hadoop.util.ExitUtil.HaltException; +import org.apache.hadoop.test.AbstractHadoopTestBase; + +public class TestExitUtil extends AbstractHadoopTestBase { + + @Before + public void before() { + ExitUtil.disableSystemExit(); + ExitUtil.disableSystemHalt(); + ExitUtil.resetFirstExitException(); + ExitUtil.resetFirstHaltException(); + } + + @After + public void after() { + ExitUtil.resetFirstExitException(); + ExitUtil.resetFirstHaltException(); + } + + @Test + public void testGetSetExitExceptions() throws Throwable { + // prepare states and exceptions + ExitException ee1 = new ExitException(1, "TestExitUtil forged 1st ExitException"); + ExitException ee2 = new ExitException(2, "TestExitUtil forged 2nd ExitException"); + // check proper initial settings + assertFalse("ExitUtil.terminateCalled initial value should be false", + ExitUtil.terminateCalled()); + assertNull("ExitUtil.getFirstExitException initial value should be null", + ExitUtil.getFirstExitException()); + + // simulate/check 1st call + ExitException ee = intercept(ExitException.class, ()->ExitUtil.terminate(ee1)); + assertSame("ExitUtil.terminate should have rethrown its ExitException argument but it " + + "had thrown something else", ee1, ee); + assertTrue("ExitUtil.terminateCalled should be true after 1st ExitUtil.terminate call", + ExitUtil.terminateCalled()); + assertSame("ExitUtil.terminate should store its 1st call's ExitException", + ee1, ExitUtil.getFirstExitException()); + + // simulate/check 2nd call not overwritting 1st one + ee = intercept(ExitException.class, ()->ExitUtil.terminate(ee2)); + assertSame("ExitUtil.terminate should have rethrown its HaltException argument but it " + + "had thrown something else", ee2, ee); + assertTrue("ExitUtil.terminateCalled should still be true after 2nd ExitUtil.terminate call", + ExitUtil.terminateCalled()); + // 2nd call rethrown the 2nd ExitException yet only the 1st only should have been stored + assertSame("ExitUtil.terminate when called twice should only remember 1st call's " + + "ExitException", ee1, ExitUtil.getFirstExitException()); + + // simulate cleanup, also tries to make sure state is ok for all junit still has to do + ExitUtil.resetFirstExitException(); + assertFalse("ExitUtil.terminateCalled should be false after " + + "ExitUtil.resetFirstExitException call", ExitUtil.terminateCalled()); + assertNull("ExitUtil.getFirstExitException should be null after " + + "ExitUtil.resetFirstExitException call", ExitUtil.getFirstExitException()); + } + + @Test + public void testGetSetHaltExceptions() throws Throwable { + // prepare states and exceptions + ExitUtil.disableSystemHalt(); + ExitUtil.resetFirstHaltException(); + HaltException he1 = new HaltException(1, "TestExitUtil forged 1st HaltException"); + HaltException he2 = new HaltException(2, "TestExitUtil forged 2nd HaltException"); + + // check proper initial settings + assertFalse("ExitUtil.haltCalled initial value should be false", + ExitUtil.haltCalled()); + assertNull("ExitUtil.getFirstHaltException initial value should be null", + ExitUtil.getFirstHaltException()); + + // simulate/check 1st call + HaltException he = intercept(HaltException.class, ()->ExitUtil.halt(he1)); + assertSame("ExitUtil.halt should have rethrown its HaltException argument but it had " + +"thrown something else", he1, he); + assertTrue("ExitUtil.haltCalled should be true after 1st ExitUtil.halt call", + ExitUtil.haltCalled()); + assertSame("ExitUtil.halt should store its 1st call's HaltException", + he1, ExitUtil.getFirstHaltException()); + + // simulate/check 2nd call not overwritting 1st one + he = intercept(HaltException.class, ()->ExitUtil.halt(he2)); + assertSame("ExitUtil.halt should have rethrown its HaltException argument but it had " + +"thrown something else", he2, he); + assertTrue("ExitUtil.haltCalled should still be true after 2nd ExitUtil.halt call", + ExitUtil.haltCalled()); + assertSame("ExitUtil.halt when called twice should only remember 1st call's HaltException", + he1, ExitUtil.getFirstHaltException()); + + // simulate cleanup, also tries to make sure state is ok for all junit still has to do + ExitUtil.resetFirstHaltException(); + assertFalse("ExitUtil.haltCalled should be false after " + + "ExitUtil.resetFirstHaltException call", ExitUtil.haltCalled()); + assertNull("ExitUtil.getFirstHaltException should be null after " + + "ExitUtil.resetFirstHaltException call", ExitUtil.getFirstHaltException()); + } +} diff --git a/hadoop-common-project/hadoop-common/src/test/scripts/hadoop-functions_test_helper.bash b/hadoop-common-project/hadoop-common/src/test/scripts/hadoop-functions_test_helper.bash index fa34bdfc4b5dd..9ba5b7e1bbbfc 100755 --- a/hadoop-common-project/hadoop-common/src/test/scripts/hadoop-functions_test_helper.bash +++ b/hadoop-common-project/hadoop-common/src/test/scripts/hadoop-functions_test_helper.bash @@ -16,6 +16,7 @@ setup() { + export LD_LIBRARY_PATH="" RELTMP="${BATS_TEST_DIRNAME}/../../../target/test-dir/bats.$$.${RANDOM}" mkdir -p ${RELTMP} TMP=$(cd -P -- "${RELTMP}" >/dev/null && pwd -P) diff --git a/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/portmap/RpcProgramPortmap.java b/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/portmap/RpcProgramPortmap.java index 7b33a644fbe76..a585dbc6b20b0 100644 --- a/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/portmap/RpcProgramPortmap.java +++ b/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/portmap/RpcProgramPortmap.java @@ -18,6 +18,7 @@ package org.apache.hadoop.portmap; import java.util.concurrent.ConcurrentHashMap; +import java.util.Map; import io.netty.buffer.ByteBuf; import io.netty.buffer.Unpooled; @@ -54,7 +55,7 @@ final class RpcProgramPortmap extends IdleStateHandler { private static final Logger LOG = LoggerFactory.getLogger(RpcProgramPortmap.class); - private final ConcurrentHashMap map = new ConcurrentHashMap(); + private final ConcurrentHashMap map = new ConcurrentHashMap<>(); /** ChannelGroup that remembers all active channels for gracefully shutdown. */ private final ChannelGroup allChannels; @@ -208,4 +209,8 @@ public void exceptionCaught(ChannelHandlerContext ctx, Throwable t) { LOG.warn("Encountered ", t); ctx.channel().close(); } + + public Map getMap() { + return map; + } } diff --git a/hadoop-common-project/hadoop-nfs/src/test/java/org/apache/hadoop/portmap/TestPortmap.java b/hadoop-common-project/hadoop-nfs/src/test/java/org/apache/hadoop/portmap/TestPortmap.java index 8ebf9d03c6c30..84fa71a269d71 100644 --- a/hadoop-common-project/hadoop-nfs/src/test/java/org/apache/hadoop/portmap/TestPortmap.java +++ b/hadoop-common-project/hadoop-nfs/src/test/java/org/apache/hadoop/portmap/TestPortmap.java @@ -31,7 +31,6 @@ import org.apache.hadoop.oncrpc.XDR; import org.apache.hadoop.oncrpc.security.CredentialsNone; import org.apache.hadoop.oncrpc.security.VerifierNone; -import org.apache.hadoop.test.Whitebox; import org.junit.AfterClass; import org.junit.BeforeClass; import org.junit.Test; @@ -76,7 +75,7 @@ public void testIdle() throws InterruptedException, IOException { } @Test(timeout = 10000) - public void testRegistration() throws IOException, InterruptedException { + public void testRegistration() throws IOException, InterruptedException, IllegalAccessException { XDR req = new XDR(); RpcCall.getInstance(++xid, RpcProgramPortmap.PROGRAM, RpcProgramPortmap.VERSION, @@ -100,9 +99,7 @@ public void testRegistration() throws IOException, InterruptedException { // Give the server a chance to process the request Thread.sleep(100); boolean found = false; - @SuppressWarnings("unchecked") - Map map = (Map) Whitebox - .getInternalState(pm.getHandler(), "map"); + Map map = pm.getHandler().getMap(); for (PortmapMapping m : map.values()) { if (m.getPort() == sent.getPort() diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/ClientGSIContext.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/ClientGSIContext.java index 4de969642d574..bcbb4b96c2aeb 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/ClientGSIContext.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/ClientGSIContext.java @@ -26,6 +26,7 @@ import java.io.IOException; import java.util.concurrent.atomic.LongAccumulator; +import org.apache.hadoop.thirdparty.protobuf.ByteString; /** * Global State Id context for the client. @@ -37,8 +38,17 @@ @InterfaceStability.Evolving public class ClientGSIContext implements AlignmentContext { - private final LongAccumulator lastSeenStateId = - new LongAccumulator(Math::max, Long.MIN_VALUE); + private final LongAccumulator lastSeenStateId; + private ByteString routerFederatedState; + + public ClientGSIContext() { + this(new LongAccumulator(Math::max, Long.MIN_VALUE)); + } + + public ClientGSIContext(LongAccumulator lastSeenStateId) { + this.lastSeenStateId = lastSeenStateId; + routerFederatedState = null; + } @Override public long getLastSeenStateId() { @@ -65,16 +75,25 @@ public void updateResponseState(RpcResponseHeaderProto.Builder header) { * in responses. */ @Override - public void receiveResponseState(RpcResponseHeaderProto header) { - lastSeenStateId.accumulate(header.getStateId()); + public synchronized void receiveResponseState(RpcResponseHeaderProto header) { + if (header.hasRouterFederatedState()) { + routerFederatedState = header.getRouterFederatedState(); + } else { + lastSeenStateId.accumulate(header.getStateId()); + } } /** * Client side implementation for providing state alignment info in requests. */ @Override - public void updateRequestState(RpcRequestHeaderProto.Builder header) { - header.setStateId(lastSeenStateId.longValue()); + public synchronized void updateRequestState(RpcRequestHeaderProto.Builder header) { + if (lastSeenStateId.get() != Long.MIN_VALUE) { + header.setStateId(lastSeenStateId.get()); + } + if (routerFederatedState != null) { + header.setRouterFederatedState(routerFederatedState); + } } /** diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java index 2682549cba126..acfca6799f4f8 100755 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java @@ -41,6 +41,7 @@ import java.util.ArrayList; import java.util.EnumSet; import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Random; @@ -274,7 +275,7 @@ Configuration getConfiguration() { * that are currently being written by this client. * Note that a file can only be written by a single client. */ - private final Map filesBeingWritten = new HashMap<>(); + private final Map filesBeingWritten = new HashMap<>(); /** * Same as this(NameNode.getNNAddress(conf), conf); @@ -501,9 +502,9 @@ public LeaseRenewer getLeaseRenewer() { } /** Get a lease and start automatic renewal */ - private void beginFileLease(final long inodeId, final DFSOutputStream out) { + private void beginFileLease(final String key, final DFSOutputStream out) { synchronized (filesBeingWritten) { - putFileBeingWritten(inodeId, out); + putFileBeingWritten(key, out); LeaseRenewer renewer = getLeaseRenewer(); boolean result = renewer.put(this); if (!result) { @@ -517,9 +518,9 @@ private void beginFileLease(final long inodeId, final DFSOutputStream out) { } /** Stop renewal of lease for the file. */ - void endFileLease(final long inodeId) { + void endFileLease(final String renewLeaseKey) { synchronized (filesBeingWritten) { - removeFileBeingWritten(inodeId); + removeFileBeingWritten(renewLeaseKey); // remove client from renewer if no files are open if (filesBeingWritten.isEmpty()) { getLeaseRenewer().closeClient(this); @@ -531,10 +532,10 @@ void endFileLease(final long inodeId) { * enforced to consistently update its local dfsclients array and * client's filesBeingWritten map. */ - public void putFileBeingWritten(final long inodeId, + public void putFileBeingWritten(final String key, final DFSOutputStream out) { synchronized(filesBeingWritten) { - filesBeingWritten.put(inodeId, out); + filesBeingWritten.put(key, out); // update the last lease renewal time only when there was no // writes. once there is one write stream open, the lease renewer // thread keeps it updated well with in anyone's expiration time. @@ -545,9 +546,9 @@ public void putFileBeingWritten(final long inodeId, } /** Remove a file. Only called from LeaseRenewer. */ - public void removeFileBeingWritten(final long inodeId) { + public void removeFileBeingWritten(final String key) { synchronized(filesBeingWritten) { - filesBeingWritten.remove(inodeId); + filesBeingWritten.remove(key); if (filesBeingWritten.isEmpty()) { lastLeaseRenewal = 0; } @@ -579,6 +580,34 @@ void updateLastLeaseRenewal() { } } + @VisibleForTesting + public int getNumOfFilesBeingWritten() { + synchronized (filesBeingWritten) { + return filesBeingWritten.size(); + } + } + + /** + * Get all namespaces of DFSOutputStreams. + */ + private List getNamespaces() { + HashSet namespaces = new HashSet<>(); + synchronized (filesBeingWritten) { + for (DFSOutputStream outputStream : filesBeingWritten.values()) { + String namespace = outputStream.getNamespace(); + if (namespace == null || namespace.isEmpty()) { + return null; + } else { + namespaces.add(namespace); + } + } + if (namespaces.isEmpty()) { + return null; + } + } + return new ArrayList<>(namespaces); + } + /** * Renew leases. * @return true if lease was renewed. May return false if this @@ -587,7 +616,7 @@ void updateLastLeaseRenewal() { public boolean renewLease() throws IOException { if (clientRunning && !isFilesBeingWrittenEmpty()) { try { - namenode.renewLease(clientName); + namenode.renewLease(clientName, getNamespaces()); updateLastLeaseRenewal(); return true; } catch (IOException e) { @@ -618,14 +647,14 @@ void closeConnectionToNamenode() { /** Close/abort all files being written. */ public void closeAllFilesBeingWritten(final boolean abort) { for(;;) { - final long inodeId; + final String key; final DFSOutputStream out; synchronized(filesBeingWritten) { if (filesBeingWritten.isEmpty()) { return; } - inodeId = filesBeingWritten.keySet().iterator().next(); - out = filesBeingWritten.remove(inodeId); + key = filesBeingWritten.keySet().iterator().next(); + out = filesBeingWritten.remove(key); } if (out != null) { try { @@ -636,7 +665,7 @@ public void closeAllFilesBeingWritten(final boolean abort) { } } catch(IOException ie) { LOG.error("Failed to " + (abort ? "abort" : "close") + " file: " - + out.getSrc() + " with inode: " + inodeId, ie); + + out.getSrc() + " with renewLeaseKey: " + key, ie); } } } @@ -1275,7 +1304,7 @@ public DFSOutputStream create(String src, FsPermission permission, src, masked, flag, createParent, replication, blockSize, progress, dfsClientConf.createChecksum(checksumOpt), getFavoredNodesStr(favoredNodes), ecPolicyName, storagePolicy); - beginFileLease(result.getFileId(), result); + beginFileLease(result.getUniqKey(), result); return result; } @@ -1330,7 +1359,7 @@ public DFSOutputStream primitiveCreate(String src, FsPermission absPermission, flag, createParent, replication, blockSize, progress, checksum, null, null, null); } - beginFileLease(result.getFileId(), result); + beginFileLease(result.getUniqKey(), result); return result; } @@ -1475,7 +1504,7 @@ private DFSOutputStream append(String src, int buffersize, checkOpen(); final DFSOutputStream result = callAppend(src, flag, progress, favoredNodes); - beginFileLease(result.getFileId(), result); + beginFileLease(result.getUniqKey(), result); return result; } @@ -2396,8 +2425,8 @@ long rollEdits() throws IOException { } @VisibleForTesting - ExtendedBlock getPreviousBlock(long fileId) { - return filesBeingWritten.get(fileId).getBlock(); + ExtendedBlock getPreviousBlock(String key) { + return filesBeingWritten.get(key).getBlock(); } /** diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java index 0deaa41cf4175..6ddd56cf72703 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java @@ -113,6 +113,8 @@ public class DFSOutputStream extends FSOutputSummer protected final String src; protected final long fileId; + private final String namespace; + private final String uniqKey; protected final long blockSize; protected final int bytesPerChecksum; @@ -195,6 +197,15 @@ private DFSOutputStream(DFSClient dfsClient, String src, this.dfsClient = dfsClient; this.src = src; this.fileId = stat.getFileId(); + this.namespace = stat.getNamespace(); + if (this.namespace == null) { + String defaultKey = dfsClient.getConfiguration().get( + HdfsClientConfigKeys.DFS_OUTPUT_STREAM_UNIQ_DEFAULT_KEY, + HdfsClientConfigKeys.DFS_OUTPUT_STREAM_UNIQ_DEFAULT_KEY_DEFAULT); + this.uniqKey = defaultKey + "_" + this.fileId; + } else { + this.uniqKey = this.namespace + "_" + this.fileId; + } this.blockSize = stat.getBlockSize(); this.blockReplication = stat.getReplication(); this.fileEncryptionInfo = stat.getFileEncryptionInfo(); @@ -818,7 +829,7 @@ boolean isClosed() { void setClosed() { closed = true; - dfsClient.endFileLease(fileId); + dfsClient.endFileLease(getUniqKey()); getStreamer().release(); } @@ -921,7 +932,7 @@ protected synchronized void closeImpl() throws IOException { protected void recoverLease(boolean recoverLeaseOnCloseException) { if (recoverLeaseOnCloseException) { try { - dfsClient.endFileLease(fileId); + dfsClient.endFileLease(getUniqKey()); dfsClient.recoverLease(src); leaseRecovered = true; } catch (Exception e) { @@ -1084,6 +1095,16 @@ public long getFileId() { return fileId; } + @VisibleForTesting + public String getNamespace() { + return namespace; + } + + @VisibleForTesting + public String getUniqKey() { + return this.uniqKey; + } + /** * Return the source of stream. */ diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSStripedOutputStream.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSStripedOutputStream.java index 0f60027269d8f..1233c033ee0e6 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSStripedOutputStream.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSStripedOutputStream.java @@ -1055,7 +1055,7 @@ void abort() throws IOException { } } - dfsClient.endFileLease(fileId); + dfsClient.endFileLease(getUniqKey()); final IOException ioe = b.build(); if (ioe != null) { throw ioe; diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/FileChecksumHelper.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/FileChecksumHelper.java index c1804a2c22f44..21bc885358076 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/FileChecksumHelper.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/FileChecksumHelper.java @@ -303,7 +303,8 @@ FileChecksum makeCompositeCrcResult() throws IOException { byte[] blockChecksumBytes = blockChecksumBuf.getData(); long sumBlockLengths = 0; - for (int i = 0; i < locatedBlocks.size() - 1; ++i) { + int i = 0; + for (; i < locatedBlocks.size() - 1; ++i) { LocatedBlock block = locatedBlocks.get(i); // For everything except the last LocatedBlock, we expect getBlockSize() // to accurately reflect the number of file bytes digested in the block @@ -316,19 +317,8 @@ FileChecksum makeCompositeCrcResult() throws IOException { "Added blockCrc 0x{} for block index {} of size {}", Integer.toString(blockCrc, 16), i, block.getBlockSize()); } - - // NB: In some cases the located blocks have their block size adjusted - // explicitly based on the requested length, but not all cases; - // these numbers may or may not reflect actual sizes on disk. - long reportedLastBlockSize = - blockLocations.getLastLocatedBlock().getBlockSize(); - long consumedLastBlockLength = reportedLastBlockSize; - if (length - sumBlockLengths < reportedLastBlockSize) { - LOG.warn( - "Last block length {} is less than reportedLastBlockSize {}", - length - sumBlockLengths, reportedLastBlockSize); - consumedLastBlockLength = length - sumBlockLengths; - } + LocatedBlock nextBlock = locatedBlocks.get(i); + long consumedLastBlockLength = Math.min(length - sumBlockLengths, nextBlock.getBlockSize()); // NB: blockChecksumBytes.length may be much longer than actual bytes // written into the DataOutput. int lastBlockCrc = CrcUtil.readInt( diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/NameNodeProxiesClient.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/NameNodeProxiesClient.java index aa9577330cfae..4acec82824238 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/NameNodeProxiesClient.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/NameNodeProxiesClient.java @@ -349,6 +349,9 @@ public static ClientProtocol createProxyWithAlignmentContext( boolean withRetries, AtomicBoolean fallbackToSimpleAuth, AlignmentContext alignmentContext) throws IOException { + if (alignmentContext == null) { + alignmentContext = new ClientGSIContext(); + } RPC.setProtocolEngine(conf, ClientNamenodeProtocolPB.class, ProtobufRpcEngine2.class); diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/HdfsClientConfigKeys.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/HdfsClientConfigKeys.java index 8e9a5b62490d0..e3e01fde3a51c 100755 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/HdfsClientConfigKeys.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/HdfsClientConfigKeys.java @@ -281,6 +281,10 @@ public interface HdfsClientConfigKeys { "dfs.client.fsck.read.timeout"; int DFS_CLIENT_FSCK_READ_TIMEOUT_DEFAULT = 60 * 1000; + String DFS_OUTPUT_STREAM_UNIQ_DEFAULT_KEY = + "dfs.client.output.stream.uniq.default.key"; + String DFS_OUTPUT_STREAM_UNIQ_DEFAULT_KEY_DEFAULT = "DEFAULT"; + /** * These are deprecated config keys to client code. */ diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java index e9ae803a541b1..4f2da496a1a3d 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java @@ -759,11 +759,19 @@ SnapshotStatus[] getSnapshotListing(String snapshotRoot) * the last call to renewLease(), the NameNode assumes the * client has died. * + * @param namespaces The full Namespace list that the renewLease rpc + * should be forwarded by RBF. + * Tips: NN side, this value should be null. + * RBF side, if this value is null, this rpc will + * be forwarded to all available namespaces, + * else this rpc will be forwarded to + * the special namespaces. + * * @throws org.apache.hadoop.security.AccessControlException permission denied * @throws IOException If an I/O error occurred */ @Idempotent - void renewLease(String clientName) throws IOException; + void renewLease(String clientName, List namespaces) throws IOException; /** * Start lease recovery. diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsFileStatus.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsFileStatus.java index 264e3f4050fd7..efc3b90b5a970 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsFileStatus.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsFileStatus.java @@ -490,6 +490,10 @@ default FileStatus makeQualified(URI defaultUri, Path parent) { */ int compareTo(FileStatus stat); + void setNamespace(String namespace); + + String getNamespace(); + /** * Set redundant flags for compatibility with existing applications. */ diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsLocatedFileStatus.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsLocatedFileStatus.java index bf4e0d2f9f16e..a3d4867cff45d 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsLocatedFileStatus.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsLocatedFileStatus.java @@ -54,6 +54,8 @@ public class HdfsLocatedFileStatus // BlockLocations[] is the user-facing type private transient LocatedBlocks hdfsloc; + private String namespace = null; + /** * Constructor. * @param length the number of bytes the file has @@ -217,4 +219,14 @@ public LocatedFileStatus makeQualifiedLocated(URI defaultUri, Path path) { return this; } + @Override + public String getNamespace() { + return namespace; + } + + @Override + public void setNamespace(String namespace) { + this.namespace = namespace; + } + } diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsNamedFileStatus.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsNamedFileStatus.java index 9434423d721b9..4c90e17e4a5d5 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsNamedFileStatus.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsNamedFileStatus.java @@ -44,6 +44,8 @@ public class HdfsNamedFileStatus extends FileStatus implements HdfsFileStatus { private final int childrenNum; private final byte storagePolicy; + private String namespace = null; + /** * Constructor. * @param length the number of bytes the file has @@ -177,4 +179,13 @@ public int hashCode() { return super.hashCode(); } + @Override + public String getNamespace() { + return namespace; + } + + @Override + public void setNamespace(String namespace) { + this.namespace = namespace; + } } diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolTranslatorPB.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolTranslatorPB.java index 7b8ca42c50f7b..541a4361896dc 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolTranslatorPB.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolTranslatorPB.java @@ -744,11 +744,15 @@ public BatchedDirectoryListing getBatchedListing( @Override - public void renewLease(String clientName) throws IOException { - RenewLeaseRequestProto req = RenewLeaseRequestProto.newBuilder() - .setClientName(clientName).build(); + public void renewLease(String clientName, List namespaces) + throws IOException { + RenewLeaseRequestProto.Builder builder = RenewLeaseRequestProto + .newBuilder().setClientName(clientName); + if (namespaces != null && !namespaces.isEmpty()) { + builder.addAllNamespaces(namespaces); + } try { - rpcProxy.renewLease(null, req); + rpcProxy.renewLease(null, builder.build()); } catch (ServiceException e) { throw ProtobufHelper.getRemoteException(e); } diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelperClient.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelperClient.java index 6097d2f495dac..496a5cf46146d 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelperClient.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelperClient.java @@ -1764,7 +1764,7 @@ public static HdfsFileStatus convert(HdfsFileStatusProto fs) { EnumSet flags = fs.hasFlags() ? convertFlags(fs.getFlags()) : convertFlags(fs.getPermission()); - return new HdfsFileStatus.Builder() + HdfsFileStatus hdfsFileStatus = new HdfsFileStatus.Builder() .length(fs.getLength()) .isdir(fs.getFileType().equals(FileType.IS_DIR)) .replication(fs.getBlockReplication()) @@ -1794,6 +1794,10 @@ public static HdfsFileStatus convert(HdfsFileStatusProto fs) { ? convertErasureCodingPolicy(fs.getEcPolicy()) : null) .build(); + if (fs.hasNamespace()) { + hdfsFileStatus.setNamespace(fs.getNamespace()); + } + return hdfsFileStatus; } private static EnumSet convertFlags(int flags) { @@ -2399,6 +2403,9 @@ public static HdfsFileStatusProto convert(HdfsFileStatus fs) { flags |= fs.isSnapshotEnabled() ? HdfsFileStatusProto.Flags .SNAPSHOT_ENABLED_VALUE : 0; builder.setFlags(flags); + if (fs.getNamespace() != null && !fs.getNamespace().isEmpty()) { + builder.setNamespace(fs.getNamespace()); + } return builder.build(); } diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/ClientNamenodeProtocol.proto b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/ClientNamenodeProtocol.proto index 1e8d0b0a2667d..60792b5b6c94c 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/ClientNamenodeProtocol.proto +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/ClientNamenodeProtocol.proto @@ -332,6 +332,7 @@ message GetSnapshotDiffReportListingResponseProto { } message RenewLeaseRequestProto { required string clientName = 1; + repeated string namespaces = 2; } message RenewLeaseResponseProto { //void response diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/erasurecoding.proto b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/erasurecoding.proto index d92dd4cb84c97..fd3618fe73175 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/erasurecoding.proto +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/erasurecoding.proto @@ -108,6 +108,7 @@ message BlockECReconstructionInfoProto { required StorageTypesProto targetStorageTypes = 5; required bytes liveBlockIndices = 6; required ErasureCodingPolicyProto ecPolicy = 7; + optional bytes excludeReconstructedIndices = 8; } /** diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/hdfs.proto b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/hdfs.proto index 163d3a49d3014..a4d36180c2c7a 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/hdfs.proto +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/hdfs.proto @@ -481,6 +481,7 @@ message HdfsFileStatusProto { // Set of flags optional uint32 flags = 18 [default = 0]; + optional string namespace = 19; } /** diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/pom.xml b/hadoop-hdfs-project/hadoop-hdfs-native-client/pom.xml index 82db2d14facae..3f25354e293b9 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-native-client/pom.xml +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/pom.xml @@ -147,7 +147,23 @@ https://maven.apache.org/xsd/maven-4.0.0.xsd"> - + + + + + + + + + + + + + + + + + \nint main(int argc, char **argv) { return !EVP_aes_256_ctr; }" HAS_NEW_ENOUGH_OPENSSL) set(CMAKE_REQUIRED_INCLUDES ${OLD_CMAKE_REQUIRED_INCLUDES}) if(NOT HAS_NEW_ENOUGH_OPENSSL) message("The OpenSSL library installed at ${OPENSSL_LIBRARY} is too old. You need a version at least new enough to have EVP_aes_256_ctr.") diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs-tests/test_libhdfs_mini_stress.c b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs-tests/test_libhdfs_mini_stress.c index 1641470733f2c..7318aaafbae46 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs-tests/test_libhdfs_mini_stress.c +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs-tests/test_libhdfs_mini_stress.c @@ -18,24 +18,27 @@ #include "common/util_c.h" #include "expect.h" -#include "hdfs/hdfs.h" #include "hdfspp/hdfs_ext.h" #include "native_mini_dfs.h" #include "os/thread.h" #include "x-platform/c-api/syscall.h" +#include "hdfs/hdfs.h" #include #include -#include #include #include #include #include -#include #include -#include #include +#ifndef WIN32 +#include +#include +#include +#endif + #define TO_STR_HELPER(X) #X #define TO_STR(X) TO_STR_HELPER(X) @@ -197,7 +200,7 @@ static int fileEventCallback1(const char * event, const char * cluster, const ch if (randomErrRatioStr) randomErrRatio = (int64_t)atoi(randomErrRatioStr); if (randomErrRatio == 0) return DEBUG_SIMULATE_ERROR; else if (randomErrRatio < 0) return LIBHDFSPP_EVENT_OK; - return random() % randomErrRatio == 0 ? DEBUG_SIMULATE_ERROR : LIBHDFSPP_EVENT_OK; + return rand() % randomErrRatio == 0 ? DEBUG_SIMULATE_ERROR : LIBHDFSPP_EVENT_OK; } static int fileEventCallback2(const char * event, const char * cluster, const char * file, int64_t value, int64_t cookie) @@ -235,7 +238,7 @@ static int doTestHdfsMiniStress(struct tlhThreadInfo *ti, int randomErr) EXPECT_ZERO(hdfsCloseFile(ti->hdfs, file)); file = hdfsOpenFile(ti->hdfs, ti->fileNm, O_RDONLY, 0, 0, 0); EXPECT_NONNULL(file); - seekPos = (((double)random()) / RAND_MAX) * (fileInfo->mSize - expected); + seekPos = (((double)rand()) / RAND_MAX) * (fileInfo->mSize - expected); seekPos = (seekPos / expected) * expected; ret = hdfsSeek(ti->hdfs, file, seekPos); if (ret < 0) { diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs/os/windows/unistd.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs/os/windows/unistd.h index b82ce48968dd5..900264240e8b5 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs/os/windows/unistd.h +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs/os/windows/unistd.h @@ -22,7 +22,7 @@ /* On Windows, unistd.h does not exist, so manually define what we need. */ #include /* Declares getpid(). */ -#include +#include /* Re-route sleep to Sleep, converting units from seconds to milliseconds. */ #define sleep(seconds) Sleep((seconds) * 1000) diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/CMakeLists.txt b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/CMakeLists.txt index 078e392d4da47..a42ee2e211494 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/CMakeLists.txt +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/CMakeLists.txt @@ -28,7 +28,7 @@ project (libhdfspp) cmake_minimum_required(VERSION 2.8) -find_package (Boost 1.72.0 REQUIRED) +find_package (Boost 1.72.0 REQUIRED COMPONENTS date_time) enable_testing() set(CMAKE_CXX_STANDARD 17) @@ -274,7 +274,8 @@ endif() set(LIBHDFSPP_VERSION "0.1.0") set(LIBHDFSPP_ALL_OBJECTS $ $ $ $ $ $ $ $ $) -if (HADOOP_BUILD) +# HDFS-16464: We don't support building Hadoop DLL for Windows yet. +if (HADOOP_BUILD AND NOT MSVC) hadoop_add_dual_library(hdfspp ${EMPTY_FILE_CC} ${LIBHDFSPP_ALL_OBJECTS}) hadoop_target_link_dual_libraries(hdfspp ${LIB_DL} @@ -282,23 +283,23 @@ if (HADOOP_BUILD) ${OPENSSL_LIBRARIES} ${SASL_LIBRARIES} ${CMAKE_THREAD_LIBS_INIT} - ) + ${Boost_LIBRARIES}) set_target_properties(hdfspp PROPERTIES SOVERSION ${LIBHDFSPP_VERSION}) hadoop_dual_output_directory(hdfspp ${OUT_DIR}) -else (HADOOP_BUILD) +else (HADOOP_BUILD AND NOT MSVC) add_library(hdfspp_static STATIC ${EMPTY_FILE_CC} ${LIBHDFSPP_ALL_OBJECTS}) - target_link_libraries(hdfspp_static + target_link_libraries(hdfspp_static PUBLIC ${LIB_DL} ${PROTOBUF_LIBRARY} ${OPENSSL_LIBRARIES} ${SASL_LIBRARIES} ${CMAKE_THREAD_LIBS_INIT} - ) + ${Boost_LIBRARIES}) if(BUILD_SHARED_HDFSPP) add_library(hdfspp SHARED ${EMPTY_FILE_CC} ${LIBHDFSPP_ALL_OBJECTS}) set_target_properties(hdfspp PROPERTIES SOVERSION ${LIBHDFSPP_VERSION}) endif(BUILD_SHARED_HDFSPP) -endif (HADOOP_BUILD) +endif (HADOOP_BUILD AND NOT MSVC) # Set up make install targets # Can be installed to a particular location via "make DESTDIR=... install" diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/bindings/c/hdfs.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/bindings/c/hdfs.cc index 80f9316160216..9386453c929ce 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/bindings/c/hdfs.cc +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/bindings/c/hdfs.cc @@ -16,6 +16,8 @@ * limitations under the License. */ +#include "fs/filehandle.h" + #include "hdfspp/hdfspp.h" #include "hdfspp/hdfs_ext.h" @@ -23,7 +25,6 @@ #include "common/configuration_loader.h" #include "common/logging.h" #include "fs/filesystem.h" -#include "fs/filehandle.h" #include "x-platform/utils.h" #include "x-platform/syscall.h" diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/fsinfo.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/fsinfo.cc index f8f5923832711..ec7efc8f2708d 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/fsinfo.cc +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/fsinfo.cc @@ -21,6 +21,7 @@ #include #include #include +#include namespace hdfs { diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/logging.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/logging.h index 8935287fe0108..9d75a48791da0 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/logging.h +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/logging.h @@ -19,10 +19,10 @@ #ifndef LIB_COMMON_LOGGING_H_ #define LIB_COMMON_LOGGING_H_ -#include - #include "hdfspp/log.h" +#include + #include #include #include diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/namenode_info.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/namenode_info.cc index 92054fce07e31..4b375711f7419 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/namenode_info.cc +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/namenode_info.cc @@ -26,6 +26,7 @@ #include #include +#include namespace hdfs { diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/statinfo.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/statinfo.cc index 2fb744fbddea8..072a3f36584f0 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/statinfo.cc +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/statinfo.cc @@ -17,10 +17,12 @@ */ #include -#include + #include #include +#include "x-platform/stat.h" + namespace hdfs { StatInfo::StatInfo() diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/util.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/util.cc index c0e10183297ac..45798bdd8e940 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/util.cc +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/util.cc @@ -19,6 +19,7 @@ #include "common/util.h" #include "common/util_c.h" +#include #include #include diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/connection/datanodeconnection.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/connection/datanodeconnection.h index a0cb8375a8680..b5fe7b975c7fc 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/connection/datanodeconnection.h +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/connection/datanodeconnection.h @@ -18,9 +18,10 @@ #ifndef LIBHDFSPP_LIB_CONNECTION_DATANODECONNECTION_H_ #define LIBHDFSPP_LIB_CONNECTION_DATANODECONNECTION_H_ +#include "ClientNamenodeProtocol.pb.h" + #include "hdfspp/ioservice.h" #include "common/async_stream.h" -#include "ClientNamenodeProtocol.pb.h" #include "common/libhdfs_events_impl.h" #include "common/logging.h" #include "common/util.h" diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/filehandle.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/filehandle.cc index 7cfd6df3b9861..d5118089027b0 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/filehandle.cc +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/filehandle.cc @@ -16,11 +16,12 @@ * limitations under the License. */ +#include "reader/block_reader.h" #include "filehandle.h" + #include "common/continuation/continuation.h" #include "common/logging.h" #include "connection/datanodeconnection.h" -#include "reader/block_reader.h" #include "hdfspp/events.h" #include "x-platform/types.h" diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/filehandle.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/filehandle.h index 0e4eed7af4ec9..cfff9b1b3f037 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/filehandle.h +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/filehandle.h @@ -18,13 +18,14 @@ #ifndef LIBHDFSPP_LIB_FS_FILEHANDLE_H_ #define LIBHDFSPP_LIB_FS_FILEHANDLE_H_ +#include "reader/readergroup.h" + #include "hdfspp/ioservice.h" #include "common/async_stream.h" #include "common/cancel_tracker.h" #include "common/libhdfs_events_impl.h" #include "common/new_delete.h" #include "reader/fileinfo.h" -#include "reader/readergroup.h" #include "bad_datanode_tracker.h" #include "ClientNamenodeProtocol.pb.h" diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/filesystem.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/filesystem.cc index e92a9ee48d6b9..e2b6cea30b2f9 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/filesystem.cc +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/filesystem.cc @@ -16,21 +16,24 @@ * limitations under the License. */ -#include "filesystem.h" - #include "filehandle.h" +#include "filesystem.h" #include "common/namenode_info.h" #include #include #include #include -#include +#include #include #include "x-platform/syscall.h" +#ifndef WIN32 +#include +#endif + #define FMT_THIS_ADDR "this=" << (void*)this namespace hdfs { diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/namenode_operations.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/namenode_operations.cc index 96744e5d03d2f..4c9e398286c98 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/namenode_operations.cc +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/namenode_operations.cc @@ -26,9 +26,12 @@ #include #include #include -#include #include +#ifndef WIN32 +#include +#endif + #define FMT_THIS_ADDR "this=" << (void*)this using boost::asio::ip::tcp; diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/namenode_operations.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/namenode_operations.h index 445aa08653dcf..bcccd52a56d3d 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/namenode_operations.h +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/namenode_operations.h @@ -18,17 +18,19 @@ #ifndef LIBHDFSPP_LIB_FS_NAMENODEOPERATIONS_H_ #define LIBHDFSPP_LIB_FS_NAMENODEOPERATIONS_H_ +#include "ClientNamenodeProtocol.pb.h" + #include "rpc/rpc_engine.h" #include "hdfspp/statinfo.h" #include "hdfspp/fsinfo.h" #include "hdfspp/content_summary.h" #include "common/namenode_info.h" -#include "ClientNamenodeProtocol.pb.h" -#include "ClientNamenodeProtocol.hrpc.inl" #include #include +#include "ClientNamenodeProtocol.hrpc.inl" + namespace hdfs { /** diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/reader/block_reader.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/reader/block_reader.cc index acecfce52374e..8db645e48d049 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/reader/block_reader.cc +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/reader/block_reader.cc @@ -15,8 +15,10 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + #include "reader/block_reader.h" #include "reader/datatransfer.h" + #include "common/continuation/continuation.h" #include "common/continuation/asio.h" #include "common/logging.h" diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/reader/block_reader.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/reader/block_reader.h index 167c57d3a1704..605d641b3b317 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/reader/block_reader.h +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/reader/block_reader.h @@ -18,11 +18,12 @@ #ifndef BLOCK_READER_H_ #define BLOCK_READER_H_ +#include "datatransfer.pb.h" + #include "hdfspp/status.h" #include "common/async_stream.h" #include "common/cancel_tracker.h" #include "common/new_delete.h" -#include "datatransfer.pb.h" #include "connection/datanodeconnection.h" #include diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/reader/datatransfer.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/reader/datatransfer.h index cfa94bea2baf2..025cb8d37080d 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/reader/datatransfer.h +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/reader/datatransfer.h @@ -18,6 +18,8 @@ #ifndef LIB_READER_DATA_TRANSFER_H_ #define LIB_READER_DATA_TRANSFER_H_ +#include "datatransfer.pb.h" + #include "common/sasl_authenticator.h" #include "common/async_stream.h" #include "connection/datanodeconnection.h" diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/reader/datatransfer_impl.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/reader/datatransfer_impl.h index d77685dd45a70..e3249ece51e13 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/reader/datatransfer_impl.h +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/reader/datatransfer_impl.h @@ -18,10 +18,13 @@ #ifndef LIB_READER_DATATRANFER_IMPL_H_ #define LIB_READER_DATATRANFER_IMPL_H_ -#include "datatransfer.pb.h" #include "common/continuation/continuation.h" #include "common/continuation/asio.h" #include "common/continuation/protobuf.h" +#include "common/sasl_authenticator.h" + +#include +#include #include #include diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/reader/readergroup.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/reader/readergroup.h index e6173f7fa70e2..47b6046d80fae 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/reader/readergroup.h +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/reader/readergroup.h @@ -15,14 +15,15 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + #ifndef READER_READER_GROUP_H_ #define READER_READER_GROUP_H_ #include "block_reader.h" #include -#include #include +#include namespace hdfs { diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/CMakeLists.txt b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/CMakeLists.txt index b50134eda9536..5503ef59fb1cc 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/CMakeLists.txt +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/CMakeLists.txt @@ -30,3 +30,4 @@ target_include_directories(rpc_obj PRIVATE ../../lib) add_dependencies(rpc_obj proto) add_library(rpc $) target_include_directories(rpc PRIVATE ../../lib) +target_link_libraries(rpc PRIVATE ${Boost_LIBRARIES}) diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/request.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/request.cc index 99762c89ee9e4..bf25e60c68cab 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/request.cc +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/request.cc @@ -16,16 +16,17 @@ * limitations under the License. */ +#include "RpcHeader.pb.h" +#include "ProtobufRpcEngine.pb.h" +#include "IpcConnectionContext.pb.h" + #include + #include "request.h" #include "rpc_engine.h" #include "sasl_protocol.h" #include "hdfspp/ioservice.h" -#include "RpcHeader.pb.h" -#include "ProtobufRpcEngine.pb.h" -#include "IpcConnectionContext.pb.h" - #include namespace hdfs { diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/rpc_connection.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/rpc_connection.h index f599d36ee5c4d..21c475c1e8bbf 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/rpc_connection.h +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/rpc_connection.h @@ -43,6 +43,7 @@ #include #include +#include namespace hdfs { diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/rpc_connection_impl.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/rpc_connection_impl.cc index 5d434ef370a9d..2a4e358e9bbf4 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/rpc_connection_impl.cc +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/rpc_connection_impl.cc @@ -15,15 +15,17 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -#include "rpc_engine.h" -#include "rpc_connection_impl.h" -#include "sasl_protocol.h" #include "RpcHeader.pb.h" #include "ProtobufRpcEngine.pb.h" #include "IpcConnectionContext.pb.h" +#include "rpc_engine.h" +#include "rpc_connection_impl.h" +#include "sasl_protocol.h" + #include +#include namespace hdfs { diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/rpc_connection_impl.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/rpc_connection_impl.h index 884bd64ac642c..4f90ab72f991e 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/rpc_connection_impl.h +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/rpc_connection_impl.h @@ -35,6 +35,7 @@ #include #include +#include namespace hdfs { diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/rpc_engine.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/rpc_engine.cc index e3274cb88aacf..0bea5bf192cfa 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/rpc_engine.cc +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/rpc_engine.cc @@ -27,6 +27,7 @@ #include #include +#include #include #include diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/sasl_protocol.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/sasl_protocol.cc index bc9adbff313d7..c45ab84d2137b 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/sasl_protocol.cc +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/sasl_protocol.cc @@ -16,6 +16,7 @@ * limitations under the License. */ +#include "sasl_protocol.h" #include "rpc_engine.h" #include "rpc_connection.h" #include "common/logging.h" @@ -23,7 +24,6 @@ #include "x-platform/syscall.h" #include "sasl_engine.h" -#include "sasl_protocol.h" #if defined USE_SASL #if defined USE_CYRUS_SASL diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/sasl_protocol.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/sasl_protocol.h index a46ae08074d24..2c7fd09b96b5c 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/sasl_protocol.h +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/sasl_protocol.h @@ -19,12 +19,12 @@ #ifndef LIB_RPC_SASLPROTOCOL_H #define LIB_RPC_SASLPROTOCOL_H +#include "RpcHeader.pb.h" + #include #include #include -#include - #include "hdfspp/status.h" #include "common/auth_info.h" #include "common/libhdfs_events_impl.h" diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/x-platform/stat.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/x-platform/stat.h new file mode 100644 index 0000000000000..c078d44a51831 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/x-platform/stat.h @@ -0,0 +1,42 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef NATIVE_LIBHDFSPP_LIB_CROSS_PLATFORM_STAT +#define NATIVE_LIBHDFSPP_LIB_CROSS_PLATFORM_STAT + +#include + +#if defined(_WIN32) + +// Windows defines the macros for user RWX (_S_IREAD, _S_IWRITE and _S_IEXEC), +// but not for group and others. We implement the permissions for group and +// others through appropriate bit shifting. + +#define S_IRUSR _S_IREAD +#define S_IWUSR _S_IWRITE +#define S_IXUSR _S_IEXEC +#define S_IRGRP (S_IRUSR >> 3) +#define S_IWGRP (S_IWUSR >> 3) +#define S_IXGRP (S_IXUSR >> 3) +#define S_IROTH (S_IRGRP >> 3) +#define S_IWOTH (S_IWGRP >> 3) +#define S_IXOTH (S_IXGRP >> 3) + +#endif + +#endif diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/CMakeLists.txt b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/CMakeLists.txt index 1a88b5c81e521..7eb432f31ac0b 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/CMakeLists.txt +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/CMakeLists.txt @@ -16,16 +16,17 @@ # limitations under the License. # -find_package(Boost REQUIRED COMPONENTS date_time) - # Delegate some functionality to libhdfs, until libhdfspp is complete. set (LIBHDFS_SRC_DIR ../../libhdfs) set (LIBHDFS_TESTS_DIR ../../libhdfs-tests) set (LIBHDFSPP_SRC_DIR ..) set (LIBHDFSPP_LIB_DIR ${LIBHDFSPP_SRC_DIR}/lib) set (LIBHDFSPP_BINDING_C ${LIBHDFSPP_LIB_DIR}/bindings/c) -set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-missing-field-initializers") -set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wno-missing-field-initializers") + +if (NOT MSVC) + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-missing-field-initializers") + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wno-missing-field-initializers") +endif (NOT MSVC) include_directories( ${GENERATED_JAVAH} @@ -150,7 +151,7 @@ include_directories ( ${CMAKE_CURRENT_SOURCE_DIR}/../../libhdfs-tests/ ) -add_library(hdfspp_test_shim_static STATIC $ hdfs_shim.c libhdfs_wrapper.c libhdfspp_wrapper.cc ${LIBHDFSPP_BINDING_C}/hdfs.cc) +add_library(hdfspp_test_shim_static STATIC $ hdfs_shim.c libhdfs_wrapper.c libhdfspp_wrapper.cc) add_dependencies(hdfspp_test_shim_static proto) add_library(hdfspp_test_static STATIC $ ${LIBHDFSPP_BINDING_C}/hdfs.cc) @@ -169,10 +170,12 @@ build_libhdfs_test(hdfspp_mini_dfs_smoke hdfspp_test_shim_static ${CMAKE_CURRENT link_libhdfs_test (hdfspp_mini_dfs_smoke hdfspp_test_shim_static fs reader rpc proto common connection gmock_main ${PROTOBUF_LIBRARIES} ${OPENSSL_LIBRARIES} native_mini_dfs ${JAVA_JVM_LIBRARY} ${SASL_LIBRARIES}) add_libhdfs_test (hdfspp_mini_dfs_smoke hdfspp_test_shim_static) -build_libhdfs_test(libhdfs_mini_stress_valgrind hdfspp_test_static expect.c test_libhdfs_mini_stress.c ${OS_DIR}/thread.c) -link_libhdfs_test(libhdfs_mini_stress_valgrind hdfspp_test_static fs reader rpc proto common connection ${PROTOBUF_LIBRARIES} ${OPENSSL_LIBRARIES} native_mini_dfs ${JAVA_JVM_LIBRARY} ${SASL_LIBRARIES}) -add_memcheck_test(libhdfs_mini_stress_valgrind_hdfspp_test_static libhdfs_mini_stress_valgrind_hdfspp_test_static) -set_target_properties(libhdfs_mini_stress_valgrind_hdfspp_test_static PROPERTIES COMPILE_DEFINITIONS "VALGRIND") +if (NOT MSVC) + build_libhdfs_test(libhdfs_mini_stress_valgrind hdfspp_test_static expect.c test_libhdfs_mini_stress.c ${OS_DIR}/thread.c) + link_libhdfs_test(libhdfs_mini_stress_valgrind hdfspp_test_static fs reader rpc proto common connection ${PROTOBUF_LIBRARIES} ${OPENSSL_LIBRARIES} native_mini_dfs ${JAVA_JVM_LIBRARY} ${SASL_LIBRARIES}) + add_memcheck_test(libhdfs_mini_stress_valgrind_hdfspp_test_static libhdfs_mini_stress_valgrind_hdfspp_test_static) + set_target_properties(libhdfs_mini_stress_valgrind_hdfspp_test_static PROPERTIES COMPILE_DEFINITIONS "VALGRIND") +endif (NOT MSVC) build_libhdfs_test(libhdfs_mini_stress hdfspp_test_shim_static expect.c test_libhdfs_mini_stress.c ${OS_DIR}/thread.c) link_libhdfs_test(libhdfs_mini_stress hdfspp_test_shim_static fs reader rpc proto common connection ${PROTOBUF_LIBRARIES} ${OPENSSL_LIBRARIES} native_mini_dfs ${JAVA_JVM_LIBRARY} ${SASL_LIBRARIES}) diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/bad_datanode_test.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/bad_datanode_test.cc index 973212647e764..225dd5e20ef4d 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/bad_datanode_test.cc +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/bad_datanode_test.cc @@ -16,12 +16,13 @@ * limitations under the License. */ +#include "reader/block_reader.h" +#include "fs/filehandle.h" + #include "common/libhdfs_events_impl.h" #include "common/util.h" #include "fs/filesystem.h" -#include "fs/filehandle.h" #include "fs/bad_datanode_tracker.h" -#include "reader/block_reader.h" #include #include diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/hdfs_shim.c b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/hdfs_shim.c index 2d265b8f03c0c..ad8ad712c9d4f 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/hdfs_shim.c +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/hdfs_shim.c @@ -24,6 +24,10 @@ #include #include +#ifdef WIN32 +#define __PRETTY_FUNCTION__ __FUNCSIG__ +#endif + /* Shim structs and functions that delegate to libhdfspp and libhdfs. */ struct hdfs_internal { libhdfs_hdfsFS libhdfsRep; diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/logging_test.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/logging_test.cc index d487bf5f0f006..b14c22b02b2e2 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/logging_test.cc +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/logging_test.cc @@ -16,8 +16,8 @@ * limitations under the License. */ -#include -#include +#include "bindings/c/hdfs.cc" +#include "common/logging.h" #include #include diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/mock_connection.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/mock_connection.cc index 37fabf568d275..c99087011bf33 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/mock_connection.cc +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/mock_connection.cc @@ -18,6 +18,8 @@ #include "mock_connection.h" +#include + namespace hdfs { MockConnectionBase::MockConnectionBase(boost::asio::io_service *io_service) diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/mock_connection.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/mock_connection.h index 7a7b5f076ed39..eb62a2b35c7fc 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/mock_connection.h +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/mock_connection.h @@ -15,6 +15,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + #ifndef LIBHDFSPP_TEST_MOCK_CONNECTION_H_ #define LIBHDFSPP_TEST_MOCK_CONNECTION_H_ diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/remote_block_reader_test.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/remote_block_reader_test.cc index ccec5812f61d1..0c22e89f4c388 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/remote_block_reader_test.cc +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/remote_block_reader_test.cc @@ -16,13 +16,13 @@ * limitations under the License. */ +#include "reader/block_reader.h" +#include "reader/datatransfer.h" + #include "mock_connection.h" -#include "datatransfer.pb.h" #include "common/util.h" #include "common/cancel_tracker.h" -#include "reader/block_reader.h" -#include "reader/datatransfer.h" #include "reader/fileinfo.h" #include @@ -124,11 +124,11 @@ static inline string ToDelimitedString(const pb::MessageLite *msg) { return res; } -static inline std::pair Produce(const std::string &s) { - return make_pair(error_code(), s); +static inline std::pair Produce(const std::string &s) { + return make_pair(boost::system::error_code(), s); } -static inline std::pair ProducePacket( +static inline std::pair ProducePacket( const std::string &data, const std::string &checksum, int offset_in_block, int seqno, bool last_packet) { PacketHeaderProto proto; @@ -148,7 +148,7 @@ static inline std::pair ProducePacket( proto.AppendToString(&payload); payload += checksum; payload += data; - return std::make_pair(error_code(), std::move(payload)); + return std::make_pair(boost::system::error_code(), std::move(payload)); } TEST(RemoteBlockReaderTest, TestReadSingleTrunk) { diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/rpc_engine_test.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/rpc_engine_test.cc index caf4842b29899..ba09e8fc90781 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/rpc_engine_test.cc +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/rpc_engine_test.cc @@ -16,11 +16,11 @@ * limitations under the License. */ -#include "hdfspp/ioservice.h" +#include "RpcHeader.pb.h" +#include "hdfspp/ioservice.h" #include "mock_connection.h" #include "test.pb.h" -#include "RpcHeader.pb.h" #include "rpc/rpc_connection_impl.h" #include "common/namenode_info.h" @@ -41,6 +41,8 @@ using ::hadoop::common::EmptyResponseProto; using ::hadoop::common::EchoRequestProto; using ::hadoop::common::EchoResponseProto; +using boost::system::error_code; + using ::testing::Return; using ::std::make_pair; diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/third_party/uriparser2/uriparser2/uriparser2.c b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/third_party/uriparser2/uriparser2/uriparser2.c index ab6209c864a31..3036acfd353c1 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/third_party/uriparser2/uriparser2/uriparser2.c +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/third_party/uriparser2/uriparser2/uriparser2.c @@ -71,9 +71,11 @@ static const char *copy_path(const UriPathSegmentA *ps, char **buffer) { static int parse_int(const char *first, const char *after_last) { const int size = after_last - first; if (size) { - char buffer[size + 1]; + char* buffer = (char*) malloc(size + 1); memcpyz(buffer, first, size); - return atoi(buffer); + const int value = atoi(buffer); + free(buffer); + return value; } return 0; } diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/fairness/AbstractRouterRpcFairnessPolicyController.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/fairness/AbstractRouterRpcFairnessPolicyController.java index fe498c66b7ee8..db917be71285c 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/fairness/AbstractRouterRpcFairnessPolicyController.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/fairness/AbstractRouterRpcFairnessPolicyController.java @@ -29,6 +29,9 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import static org.apache.hadoop.hdfs.server.federation.router.RBFConfigKeys.DFS_ROUTER_FAIRNESS_ACQUIRE_TIMEOUT; +import static org.apache.hadoop.hdfs.server.federation.router.RBFConfigKeys.DFS_ROUTER_FAIRNESS_ACQUIRE_TIMEOUT_DEFAULT; + /** * Base fairness policy that implements @RouterRpcFairnessPolicyController. * Internally a map of nameservice to Semaphore is used to control permits. @@ -42,15 +45,26 @@ public class AbstractRouterRpcFairnessPolicyController /** Hash table to hold semaphore for each configured name service. */ private Map permits; + private long acquireTimeoutMs = DFS_ROUTER_FAIRNESS_ACQUIRE_TIMEOUT_DEFAULT; + public void init(Configuration conf) { this.permits = new HashMap<>(); + long timeoutMs = conf.getTimeDuration(DFS_ROUTER_FAIRNESS_ACQUIRE_TIMEOUT, + DFS_ROUTER_FAIRNESS_ACQUIRE_TIMEOUT_DEFAULT, TimeUnit.MILLISECONDS); + if (timeoutMs >= 0) { + acquireTimeoutMs = timeoutMs; + } else { + LOG.warn("Invalid value {} configured for {} should be greater than or equal to 0. " + + "Using default value of : {}ms instead.", timeoutMs, + DFS_ROUTER_FAIRNESS_ACQUIRE_TIMEOUT, DFS_ROUTER_FAIRNESS_ACQUIRE_TIMEOUT_DEFAULT); + } } @Override public boolean acquirePermit(String nsId) { try { LOG.debug("Taking lock for nameservice {}", nsId); - return this.permits.get(nsId).tryAcquire(1, TimeUnit.SECONDS); + return this.permits.get(nsId).tryAcquire(acquireTimeoutMs, TimeUnit.MILLISECONDS); } catch (InterruptedException e) { LOG.debug("Cannot get a permit for nameservice {}", nsId); } @@ -82,15 +96,13 @@ protected int getAvailablePermits(String nsId) { @Override public String getAvailableHandlerOnPerNs() { JSONObject json = new JSONObject(); - for (Map.Entry entry : permits.entrySet()) { + permits.forEach((k, v) -> { try { - String nsId = entry.getKey(); - int availableHandler = entry.getValue().availablePermits(); - json.put(nsId, availableHandler); + json.put(k, v.availablePermits()); } catch (JSONException e) { - LOG.warn("Cannot put {} into JSONObject", entry.getKey(), e); + LOG.warn("Cannot put {} into JSONObject", k, e); } - } + }); return json.toString(); } } diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/fairness/StaticRouterRpcFairnessPolicyController.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/fairness/StaticRouterRpcFairnessPolicyController.java index aa0777fc03d69..35045bdca8e27 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/fairness/StaticRouterRpcFairnessPolicyController.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/fairness/StaticRouterRpcFairnessPolicyController.java @@ -50,13 +50,10 @@ public StaticRouterRpcFairnessPolicyController(Configuration conf) { init(conf); } - public void init(Configuration conf) - throws IllegalArgumentException { + public void init(Configuration conf) throws IllegalArgumentException { super.init(conf); // Total handlers configured to process all incoming Rpc. - int handlerCount = conf.getInt( - DFS_ROUTER_HANDLER_COUNT_KEY, - DFS_ROUTER_HANDLER_COUNT_DEFAULT); + int handlerCount = conf.getInt(DFS_ROUTER_HANDLER_COUNT_KEY, DFS_ROUTER_HANDLER_COUNT_DEFAULT); LOG.info("Handlers available for fairness assignment {} ", handlerCount); @@ -71,8 +68,7 @@ public void init(Configuration conf) allConfiguredNS.add(CONCURRENT_NS); validateHandlersCount(conf, handlerCount, allConfiguredNS); for (String nsId : allConfiguredNS) { - int dedicatedHandlers = - conf.getInt(DFS_ROUTER_FAIR_HANDLER_COUNT_KEY_PREFIX + nsId, 0); + int dedicatedHandlers = conf.getInt(DFS_ROUTER_FAIR_HANDLER_COUNT_KEY_PREFIX + nsId, 0); LOG.info("Dedicated handlers {} for ns {} ", dedicatedHandlers, nsId); if (dedicatedHandlers > 0) { handlerCount -= dedicatedHandlers; @@ -86,7 +82,7 @@ public void init(Configuration conf) // Assign remaining handlers equally to remaining name services and // general pool if applicable. if (!unassignedNS.isEmpty()) { - LOG.info("Unassigned ns {}", unassignedNS.toString()); + LOG.info("Unassigned ns {}", unassignedNS); int handlersPerNS = handlerCount / unassignedNS.size(); LOG.info("Handlers available per ns {}", handlersPerNS); for (String nsId : unassignedNS) { @@ -101,24 +97,20 @@ public void init(Configuration conf) int existingPermits = getAvailablePermits(CONCURRENT_NS); if (leftOverHandlers > 0) { LOG.info("Assigned extra {} handlers to commons pool", leftOverHandlers); - insertNameServiceWithPermits(CONCURRENT_NS, - existingPermits + leftOverHandlers); + insertNameServiceWithPermits(CONCURRENT_NS, existingPermits + leftOverHandlers); } - LOG.info("Final permit allocation for concurrent ns: {}", - getAvailablePermits(CONCURRENT_NS)); + LOG.info("Final permit allocation for concurrent ns: {}", getAvailablePermits(CONCURRENT_NS)); } private static void logAssignment(String nsId, int count) { - LOG.info("Assigned {} handlers to nsId {} ", - count, nsId); + LOG.info("Assigned {} handlers to nsId {} ", count, nsId); } - private void validateHandlersCount(Configuration conf, int handlerCount, - Set allConfiguredNS) { + private void validateHandlersCount(Configuration conf, + int handlerCount, Set allConfiguredNS) { int totalDedicatedHandlers = 0; for (String nsId : allConfiguredNS) { - int dedicatedHandlers = - conf.getInt(DFS_ROUTER_FAIR_HANDLER_COUNT_KEY_PREFIX + nsId, 0); + int dedicatedHandlers = conf.getInt(DFS_ROUTER_FAIR_HANDLER_COUNT_KEY_PREFIX + nsId, 0); if (dedicatedHandlers > 0) { // Total handlers should not be less than sum of dedicated handlers. totalDedicatedHandlers += dedicatedHandlers; @@ -128,8 +120,7 @@ private void validateHandlersCount(Configuration conf, int handlerCount, } } if (totalDedicatedHandlers > handlerCount) { - String msg = String.format(ERROR_MSG, handlerCount, - totalDedicatedHandlers); + String msg = String.format(ERROR_MSG, handlerCount, totalDedicatedHandlers); LOG.error(msg); throw new IllegalArgumentException(msg); } diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/FederationRPCMBean.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/FederationRPCMBean.java index 979e7504a872b..65c6c34eb2ff6 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/FederationRPCMBean.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/FederationRPCMBean.java @@ -30,6 +30,10 @@ public interface FederationRPCMBean { long getProxyOps(); + long getActiveProxyOps(); + + long getObserverProxyOps(); + double getProxyAvg(); long getProcessingOps(); diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/FederationRPCMetrics.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/FederationRPCMetrics.java index 823bc7b8af21c..5d5f9fb8aa12a 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/FederationRPCMetrics.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/FederationRPCMetrics.java @@ -21,6 +21,7 @@ import static org.apache.hadoop.metrics2.impl.MsInfo.SessionId; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hdfs.server.federation.resolver.FederationNamenodeServiceState; import org.apache.hadoop.hdfs.server.federation.router.RouterRpcServer; import org.apache.hadoop.metrics2.MetricsSystem; import org.apache.hadoop.metrics2.annotation.Metric; @@ -49,7 +50,10 @@ public class FederationRPCMetrics implements FederationRPCMBean { private MutableRate proxy; @Metric("Number of operations the Router proxied to a Namenode") private MutableCounterLong proxyOp; - + @Metric("Number of operations the Router proxied to a Active Namenode") + private MutableCounterLong activeProxyOp; + @Metric("Number of operations the Router proxied to a Observer Namenode") + private MutableCounterLong observerProxyOp; @Metric("Number of operations to hit a standby NN") private MutableCounterLong proxyOpFailureStandby; @Metric("Number of operations to fail to reach NN") @@ -256,9 +260,15 @@ public String getAsyncCallerPool() { * Add the time to proxy an operation from the moment the Router sends it to * the Namenode until it replied. * @param time Proxy time of an operation in nanoseconds. + * @param state NameNode state. Maybe null */ - public void addProxyTime(long time) { + public void addProxyTime(long time, FederationNamenodeServiceState state) { proxy.add(time); + if(FederationNamenodeServiceState.ACTIVE == state) { + activeProxyOp.incr(); + } else if (FederationNamenodeServiceState.OBSERVER == state) { + observerProxyOp.incr(); + } proxyOp.incr(); } @@ -272,6 +282,16 @@ public long getProxyOps() { return proxyOp.value(); } + @Override + public long getActiveProxyOps() { + return activeProxyOp.value(); + } + + @Override + public long getObserverProxyOps() { + return observerProxyOp.value(); + } + /** * Add the time to process a request in the Router from the time we receive * the call until we send it to the Namenode. diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/FederationRPCPerformanceMonitor.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/FederationRPCPerformanceMonitor.java index 159d08e26a161..b57fa070546e7 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/FederationRPCPerformanceMonitor.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/FederationRPCPerformanceMonitor.java @@ -29,6 +29,7 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hdfs.server.federation.router.FederationUtil; +import org.apache.hadoop.hdfs.server.federation.resolver.FederationNamenodeServiceState; import org.apache.hadoop.hdfs.server.federation.router.RouterRpcMonitor; import org.apache.hadoop.hdfs.server.federation.router.RouterRpcServer; import org.apache.hadoop.hdfs.server.federation.store.StateStoreService; @@ -147,12 +148,13 @@ public long proxyOp() { } @Override - public void proxyOpComplete(boolean success, String nsId) { + public void proxyOpComplete(boolean success, String nsId, + FederationNamenodeServiceState state) { if (success) { long proxyTime = getProxyTime(); if (proxyTime >= 0) { if (metrics != null) { - metrics.addProxyTime(proxyTime); + metrics.addProxyTime(proxyTime, state); } if (nameserviceRPCMetricsMap != null && nameserviceRPCMetricsMap.containsKey(nsId)) { diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/RBFMetrics.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/RBFMetrics.java index be88069b49166..e1068394f6f42 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/RBFMetrics.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/RBFMetrics.java @@ -50,6 +50,7 @@ import javax.management.ObjectName; import javax.management.StandardMBean; +import org.apache.commons.math3.stat.descriptive.moment.StandardDeviation; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hdfs.protocol.DatanodeInfo; import org.apache.hadoop.hdfs.protocol.HdfsConstants.DatanodeReportType; @@ -113,6 +114,8 @@ public class RBFMetrics implements RouterMBean, FederationMBean { /** Prevent holding the page from load too long. */ private final long timeOut; + /** Enable/Disable getNodeUsage. **/ + private boolean enableGetDNUsage; /** Router interface. */ private final Router router; @@ -175,6 +178,8 @@ public RBFMetrics(Router router) throws IOException { Configuration conf = router.getConfig(); this.timeOut = conf.getTimeDuration(RBFConfigKeys.DN_REPORT_TIME_OUT, RBFConfigKeys.DN_REPORT_TIME_OUT_MS_DEFAULT, TimeUnit.MILLISECONDS); + this.enableGetDNUsage = conf.getBoolean(RBFConfigKeys.DFS_ROUTER_ENABLE_GET_DN_USAGE_KEY, + RBFConfigKeys.DFS_ROUTER_ENABLE_GET_DN_USAGE_DEFAULT); this.topTokenRealOwners = conf.getInt( RBFConfigKeys.DFS_ROUTER_METRICS_TOP_NUM_TOKEN_OWNERS_KEY, RBFConfigKeys.DFS_ROUTER_METRICS_TOP_NUM_TOKEN_OWNERS_KEY_DEFAULT); @@ -184,6 +189,11 @@ public RBFMetrics(Router router) throws IOException { ms.register(RBFMetrics.class.getName(), "RBFActivity Metrics", this); } + @VisibleForTesting + public void setEnableGetDNUsage(boolean enableGetDNUsage) { + this.enableGetDNUsage = enableGetDNUsage; + } + /** * Unregister the JMX beans. */ @@ -537,35 +547,34 @@ public int getNumEnteringMaintenanceDataNodes() { @Override // NameNodeMXBean public String getNodeUsage() { - float median = 0; - float max = 0; - float min = 0; - float dev = 0; + double median = 0; + double max = 0; + double min = 0; + double dev = 0; final Map> info = new HashMap<>(); try { - RouterRpcServer rpcServer = this.router.getRpcServer(); - DatanodeInfo[] live = rpcServer.getDatanodeReport( - DatanodeReportType.LIVE, false, timeOut); + DatanodeInfo[] live = null; + if (this.enableGetDNUsage) { + RouterRpcServer rpcServer = this.router.getRpcServer(); + live = rpcServer.getDatanodeReport(DatanodeReportType.LIVE, false, timeOut); + } else { + LOG.debug("Getting node usage is disabled."); + } - if (live.length > 0) { - float totalDfsUsed = 0; - float[] usages = new float[live.length]; + if (live != null && live.length > 0) { + double[] usages = new double[live.length]; int i = 0; for (DatanodeInfo dn : live) { usages[i++] = dn.getDfsUsedPercent(); - totalDfsUsed += dn.getDfsUsedPercent(); } - totalDfsUsed /= live.length; Arrays.sort(usages); median = usages[usages.length / 2]; max = usages[usages.length - 1]; min = usages[0]; - for (i = 0; i < usages.length; i++) { - dev += (usages[i] - totalDfsUsed) * (usages[i] - totalDfsUsed); - } - dev = (float) Math.sqrt(dev / usages.length); + StandardDeviation deviation = new StandardDeviation(); + dev = deviation.evaluate(usages); } } catch (IOException e) { LOG.error("Cannot get the live nodes: {}", e.getMessage()); @@ -877,7 +886,7 @@ private List getActiveNamenodeRegistrations() // Fetch the most recent namenode registration String nsId = nsInfo.getNameserviceId(); List nns = - namenodeResolver.getNamenodesForNameserviceId(nsId); + namenodeResolver.getNamenodesForNameserviceId(nsId, false); if (nns != null) { FederationNamenodeContext nn = nns.get(0); if (nn instanceof MembershipState) { diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/ActiveNamenodeResolver.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/ActiveNamenodeResolver.java index f06df70b517cf..cae1f478604d6 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/ActiveNamenodeResolver.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/ActiveNamenodeResolver.java @@ -43,6 +43,17 @@ @InterfaceStability.Evolving public interface ActiveNamenodeResolver { + /** + * Report a failed, unavailable NN address for a nameservice or blockPool. + * + * @param ns Nameservice identifier. + * @param failedAddress The address the failed responded to the command. + * + * @throws IOException If the state store cannot be accessed. + */ + void updateUnavailableNamenode( + String ns, InetSocketAddress failedAddress) throws IOException; + /** * Report a successful, active NN address for a nameservice or blockPool. * @@ -56,20 +67,30 @@ void updateActiveNamenode( /** * Returns a prioritized list of the most recent cached registration entries - * for a single nameservice ID. - * Returns an empty list if none are found. Returns entries in preference of: + * for a single nameservice ID. Returns an empty list if none are found. + * In the case of not observerRead Returns entries in preference of : *

      *
    • The most recent ACTIVE NN + *
    • The most recent OBSERVER NN + *
    • The most recent STANDBY NN + *
    • The most recent UNAVAILABLE NN + *
    + * + * In the case of observerRead Returns entries in preference of : + *
      + *
    • The most recent OBSERVER NN + *
    • The most recent ACTIVE NN *
    • The most recent STANDBY NN *
    • The most recent UNAVAILABLE NN *
    * * @param nameserviceId Nameservice identifier. + * @param listObserversFirst Observer read case, observer NN will be ranked first * @return Prioritized list of namenode contexts. * @throws IOException If the state store cannot be accessed. */ - List - getNamenodesForNameserviceId(String nameserviceId) throws IOException; + List getNamenodesForNameserviceId( + String nameserviceId, boolean listObserversFirst) throws IOException; /** * Returns a prioritized list of the most recent cached registration entries @@ -77,6 +98,7 @@ void updateActiveNamenode( * Returns an empty list if none are found. Returns entries in preference of: *
      *
    • The most recent ACTIVE NN + *
    • The most recent OBSERVER NN *
    • The most recent STANDBY NN *
    • The most recent UNAVAILABLE NN *
    diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/MembershipNamenodeResolver.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/MembershipNamenodeResolver.java index 9f0f78067aedd..d65ebdb628ef7 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/MembershipNamenodeResolver.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/MembershipNamenodeResolver.java @@ -19,6 +19,7 @@ import static org.apache.hadoop.hdfs.server.federation.resolver.FederationNamenodeServiceState.ACTIVE; import static org.apache.hadoop.hdfs.server.federation.resolver.FederationNamenodeServiceState.EXPIRED; +import static org.apache.hadoop.hdfs.server.federation.resolver.FederationNamenodeServiceState.OBSERVER; import static org.apache.hadoop.hdfs.server.federation.resolver.FederationNamenodeServiceState.UNAVAILABLE; import java.io.IOException; @@ -32,6 +33,7 @@ import java.util.TreeSet; import java.util.concurrent.ConcurrentHashMap; +import org.apache.commons.lang3.tuple.Pair; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hdfs.server.federation.store.DisabledNameserviceStore; import org.apache.hadoop.hdfs.server.federation.store.MembershipStore; @@ -73,8 +75,11 @@ public class MembershipNamenodeResolver /** Parent router ID. */ private String routerId; - /** Cached lookup of NN for nameservice. Invalidated on cache refresh. */ - private Map> cacheNS; + /** Cached lookup of namenodes for nameservice. The keys are a pair of the nameservice + * name and a boolean indicating if observer namenodes should be listed first. + * If true, observer namenodes are listed first. If false, active namenodes are listed first. + * Invalidated on cache refresh. */ + private Map, List> cacheNS; /** Cached lookup of NN for block pool. Invalidated on cache refresh. */ private Map> cacheBP; @@ -136,11 +141,21 @@ public boolean loadCache(boolean force) { return true; } + @Override public void updateUnavailableNamenode(String nsId, + InetSocketAddress address) throws IOException { + updateNameNodeState(nsId, address, UNAVAILABLE); + } + @Override public void updateActiveNamenode( final String nsId, final InetSocketAddress address) throws IOException { + updateNameNodeState(nsId, address, ACTIVE); + } - // Called when we have an RPC miss and successful hit on an alternate NN. + + private void updateNameNodeState(final String nsId, + final InetSocketAddress address, FederationNamenodeServiceState state) + throws IOException { // Temporarily update our cache, it will be overwritten on the next update. try { MembershipState partial = MembershipState.newInstance(); @@ -160,10 +175,11 @@ public void updateActiveNamenode( MembershipState record = records.get(0); UpdateNamenodeRegistrationRequest updateRequest = UpdateNamenodeRegistrationRequest.newInstance( - record.getNameserviceId(), record.getNamenodeId(), ACTIVE); + record.getNameserviceId(), record.getNamenodeId(), state); membership.updateNamenodeRegistration(updateRequest); - cacheNS.remove(nsId); + cacheNS.remove(Pair.of(nsId, Boolean.TRUE)); + cacheNS.remove(Pair.of(nsId, Boolean.FALSE)); // Invalidating the full cacheBp since getting the blockpool id from // namespace id is quite costly. cacheBP.clear(); @@ -175,9 +191,9 @@ public void updateActiveNamenode( @Override public List getNamenodesForNameserviceId( - final String nsId) throws IOException { + final String nsId, boolean listObserversFirst) throws IOException { - List ret = cacheNS.get(nsId); + List ret = cacheNS.get(Pair.of(nsId, listObserversFirst)); if (ret != null) { return ret; } @@ -189,7 +205,8 @@ public List getNamenodesForNameserviceId( partial.setNameserviceId(nsId); GetNamenodeRegistrationsRequest request = GetNamenodeRegistrationsRequest.newInstance(partial); - result = getRecentRegistrationForQuery(request, true, false); + result = getRecentRegistrationForQuery(request, true, + false, listObserversFirst); } catch (StateStoreUnavailableException e) { LOG.error("Cannot get active NN for {}, State Store unavailable", nsId); return null; @@ -218,7 +235,7 @@ public List getNamenodesForNameserviceId( // Cache the response ret = Collections.unmodifiableList(result); - cacheNS.put(nsId, result); + cacheNS.put(Pair.of(nsId, listObserversFirst), result); return ret; } @@ -235,7 +252,7 @@ public List getNamenodesForBlockPoolId( GetNamenodeRegistrationsRequest.newInstance(partial); final List result = - getRecentRegistrationForQuery(request, true, false); + getRecentRegistrationForQuery(request, true, false, false); if (result == null || result.isEmpty()) { LOG.error("Cannot locate eligible NNs for {}", bpId); } else { @@ -346,22 +363,34 @@ public Set getDisabledNamespaces() throws IOException { } /** - * Picks the most relevant record registration that matches the query. Return - * registrations matching the query in this preference: 1) Most recently - * updated ACTIVE registration 2) Most recently updated STANDBY registration - * (if showStandby) 3) Most recently updated UNAVAILABLE registration (if - * showUnavailable). EXPIRED registrations are ignored. + * Picks the most relevant record registration that matches the query. + * If not observer read, + * return registrations matching the query in this preference: + * 1) Most recently updated ACTIVE registration + * 2) Most recently updated Observer registration + * 3) Most recently updated STANDBY registration (if showStandby) + * 4) Most recently updated UNAVAILABLE registration (if showUnavailable). + * + * If observer read, + * return registrations matching the query in this preference: + * 1) Observer registrations, shuffled to disperse queries. + * 2) Most recently updated ACTIVE registration + * 3) Most recently updated STANDBY registration (if showStandby) + * 4) Most recently updated UNAVAILABLE registration (if showUnavailable). + * + * EXPIRED registrations are ignored. * * @param request The select query for NN registrations. * @param addUnavailable include UNAVAILABLE registrations. * @param addExpired include EXPIRED registrations. + * @param observerRead Observer read case, observer NN will be ranked first * @return List of memberships or null if no registrations that * both match the query AND the selected states. * @throws IOException */ private List getRecentRegistrationForQuery( GetNamenodeRegistrationsRequest request, boolean addUnavailable, - boolean addExpired) throws IOException { + boolean addExpired, boolean observerRead) throws IOException { // Retrieve a list of all registrations that match this query. // This may include all NN records for a namespace/blockpool, including @@ -371,24 +400,34 @@ private List getRecentRegistrationForQuery( membershipStore.getNamenodeRegistrations(request); List memberships = response.getNamenodeMemberships(); - if (!addExpired || !addUnavailable) { - Iterator iterator = memberships.iterator(); - while (iterator.hasNext()) { - MembershipState membership = iterator.next(); - if (membership.getState() == EXPIRED && !addExpired) { - iterator.remove(); - } else if (membership.getState() == UNAVAILABLE && !addUnavailable) { - iterator.remove(); - } + List observerMemberships = new ArrayList<>(); + Iterator iterator = memberships.iterator(); + while (iterator.hasNext()) { + MembershipState membership = iterator.next(); + if (membership.getState() == EXPIRED && !addExpired) { + iterator.remove(); + } else if (membership.getState() == UNAVAILABLE && !addUnavailable) { + iterator.remove(); + } else if (membership.getState() == OBSERVER && observerRead) { + iterator.remove(); + observerMemberships.add(membership); } } - List priorityList = new ArrayList<>(); - priorityList.addAll(memberships); - Collections.sort(priorityList, new NamenodePriorityComparator()); + memberships.sort(new NamenodePriorityComparator()); + if(observerRead) { + List ret = new ArrayList<>( + memberships.size() + observerMemberships.size()); + if(observerMemberships.size() > 1) { + Collections.shuffle(observerMemberships); + } + ret.addAll(observerMemberships); + ret.addAll(memberships); + memberships = ret; + } - LOG.debug("Selected most recent NN {} for query", priorityList); - return priorityList; + LOG.debug("Selected most recent NN {} for query", memberships); + return memberships; } @Override diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/ConnectionContext.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/ConnectionContext.java index bd2d8c9d69719..df026bb66efd4 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/ConnectionContext.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/ConnectionContext.java @@ -20,6 +20,7 @@ import java.net.InetSocketAddress; import java.util.concurrent.TimeUnit; +import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hdfs.NameNodeProxiesClient.ProxyAndInfo; import org.apache.hadoop.ipc.RPC; import org.apache.hadoop.util.Time; @@ -53,9 +54,14 @@ public class ConnectionContext { private long lastActiveTs = 0; /** The connection's active status would expire after this window. */ private final static long ACTIVE_WINDOW_TIME = TimeUnit.SECONDS.toMillis(30); + /** The maximum number of requests that this connection can handle concurrently. **/ + private final int maxConcurrencyPerConn; - public ConnectionContext(ProxyAndInfo connection) { + public ConnectionContext(ProxyAndInfo connection, Configuration conf) { this.client = connection; + this.maxConcurrencyPerConn = conf.getInt( + RBFConfigKeys.DFS_ROUTER_MAX_CONCURRENCY_PER_CONNECTION_KEY, + RBFConfigKeys.DFS_ROUTER_MAX_CONCURRENCY_PER_CONNECTION_DEFAULT); } /** @@ -93,6 +99,23 @@ public synchronized boolean isClosed() { * @return True if the connection can be used. */ public synchronized boolean isUsable() { + return hasAvailableConcurrency() && !isClosed(); + } + + /** + * Return true if this connection context still has available concurrency, + * else return false. + */ + private synchronized boolean hasAvailableConcurrency() { + return this.numThreads < maxConcurrencyPerConn; + } + + /** + * Check if the connection is idle. It checks if the connection is not used + * by another thread. + * @return True if the connection is not used by another thread. + */ + public synchronized boolean isIdle() { return !isActive() && !isClosed(); } diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/ConnectionManager.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/ConnectionManager.java index 5fe797bf5ce2c..c6db9837c7cae 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/ConnectionManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/ConnectionManager.java @@ -73,6 +73,14 @@ public class ConnectionManager { /** Queue for creating new connections. */ private final BlockingQueue creatorQueue; + /** + * Global federated namespace context for router. + */ + private final RouterStateIdContext routerStateIdContext; + /** + * Map from connection pool ID to namespace. + */ + private final Map connectionPoolToNamespaceMap; /** Max size of queue for creating new connections. */ private final int creatorQueueMaxSize; @@ -85,15 +93,19 @@ public class ConnectionManager { /** If the connection manager is running. */ private boolean running = false; + public ConnectionManager(Configuration config) { + this(config, new RouterStateIdContext(config)); + } /** * Creates a proxy client connection pool manager. * * @param config Configuration for the connections. */ - public ConnectionManager(Configuration config) { + public ConnectionManager(Configuration config, RouterStateIdContext routerStateIdContext) { this.conf = config; - + this.routerStateIdContext = routerStateIdContext; + this.connectionPoolToNamespaceMap = new HashMap<>(); // Configure minimum, maximum and active connection pools this.maxSize = this.conf.getInt( RBFConfigKeys.DFS_ROUTER_NAMENODE_CONNECTION_POOL_SIZE, @@ -160,6 +172,10 @@ public void close() { pool.close(); } this.pools.clear(); + for (String nsID: connectionPoolToNamespaceMap.values()) { + routerStateIdContext.removeNamespaceStateId(nsID); + } + connectionPoolToNamespaceMap.clear(); } finally { writeLock.unlock(); } @@ -172,12 +188,12 @@ public void close() { * @param ugi User group information. * @param nnAddress Namenode address for the connection. * @param protocol Protocol for the connection. + * @param nsId Nameservice identity. * @return Proxy client to connect to nnId as UGI. * @throws IOException If the connection cannot be obtained. */ public ConnectionContext getConnection(UserGroupInformation ugi, - String nnAddress, Class protocol) throws IOException { - + String nnAddress, Class protocol, String nsId) throws IOException { // Check if the manager is shutdown if (!this.running) { LOG.error( @@ -205,9 +221,13 @@ public ConnectionContext getConnection(UserGroupInformation ugi, if (pool == null) { pool = new ConnectionPool( this.conf, nnAddress, ugi, this.minSize, this.maxSize, - this.minActiveRatio, protocol); + this.minActiveRatio, protocol, + new PoolAlignmentContext(this.routerStateIdContext, nsId)); this.pools.put(connectionId, pool); + this.connectionPoolToNamespaceMap.put(connectionId, nsId); } + long clientStateId = RouterStateIdContext.getClientStateIdFromCurrentCall(nsId); + pool.getPoolAlignmentContext().advanceClientStateId(clientStateId); } finally { writeLock.unlock(); } @@ -430,6 +450,11 @@ public void run() { try { for (ConnectionPoolId poolId : toRemove) { pools.remove(poolId); + String nsID = connectionPoolToNamespaceMap.get(poolId); + connectionPoolToNamespaceMap.remove(poolId); + if (!connectionPoolToNamespaceMap.values().contains(nsID)) { + routerStateIdContext.removeNamespaceStateId(nsID); + } } } finally { writeLock.unlock(); diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/ConnectionPool.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/ConnectionPool.java index 293a4b64d2031..ef3580b35d6f7 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/ConnectionPool.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/ConnectionPool.java @@ -32,6 +32,7 @@ import javax.net.SocketFactory; import org.apache.hadoop.classification.VisibleForTesting; +import org.apache.hadoop.ipc.AlignmentContext; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.conf.Configuration; @@ -77,7 +78,6 @@ public class ConnectionPool { private static final Logger LOG = LoggerFactory.getLogger(ConnectionPool.class); - /** Configuration settings for the connection pool. */ private final Configuration conf; @@ -94,6 +94,8 @@ public class ConnectionPool { private volatile List connections = new ArrayList<>(); /** Connection index for round-robin. */ private final AtomicInteger clientIndex = new AtomicInteger(0); + /** Underlying socket index. **/ + private final AtomicInteger socketIndex = new AtomicInteger(0); /** Min number of connections per user. */ private final int minSize; @@ -105,6 +107,11 @@ public class ConnectionPool { /** The last time a connection was active. */ private volatile long lastActiveTime = 0; + /** Enable using multiple physical socket or not. **/ + private final boolean enableMultiSocket; + /** StateID alignment context. */ + private final PoolAlignmentContext alignmentContext; + /** Map for the protocols and their protobuf implementations. */ private final static Map, ProtoImpl> PROTO_MAP = new HashMap<>(); static { @@ -134,7 +141,8 @@ private static class ProtoImpl { protected ConnectionPool(Configuration config, String address, UserGroupInformation user, int minPoolSize, int maxPoolSize, - float minActiveRatio, Class proto) throws IOException { + float minActiveRatio, Class proto, PoolAlignmentContext alignmentContext) + throws IOException { this.conf = config; @@ -149,9 +157,14 @@ protected ConnectionPool(Configuration config, String address, this.minSize = minPoolSize; this.maxSize = maxPoolSize; this.minActiveRatio = minActiveRatio; + this.enableMultiSocket = conf.getBoolean( + RBFConfigKeys.DFS_ROUTER_NAMENODE_ENABLE_MULTIPLE_SOCKET_KEY, + RBFConfigKeys.DFS_ROUTER_NAMENODE_ENABLE_MULTIPLE_SOCKET_DEFAULT); + + this.alignmentContext = alignmentContext; // Add minimum connections to the pool - for (int i=0; i tmpConnections = this.connections; - int size = tmpConnections.size(); - // Inc and mask off sign bit, lookup index should be non-negative int - int threadIndex = this.clientIndex.getAndIncrement() & 0x7FFFFFFF; - for (int i=0; i 0) { + // Get a connection from the pool following round-robin + // Inc and mask off sign bit, lookup index should be non-negative int + int threadIndex = this.clientIndex.getAndIncrement() & 0x7FFFFFFF; + conn = tmpConnections.get(threadIndex % size); + } return conn; } @@ -256,10 +276,9 @@ public synchronized List removeConnections(int num) { int targetCount = Math.min(num, this.connections.size() - this.minSize); // Remove and close targetCount of connections List tmpConnections = new ArrayList<>(); - for (int i = 0; i < this.connections.size(); i++) { - ConnectionContext conn = this.connections.get(i); + for (ConnectionContext conn : this.connections) { // Only pick idle connections to close - if (removed.size() < targetCount && conn.isUsable()) { + if (removed.size() < targetCount && conn.isIdle()) { removed.add(conn); } else { tmpConnections.add(conn); @@ -267,8 +286,8 @@ public synchronized List removeConnections(int num) { } this.connections = tmpConnections; } - LOG.debug("Expected to remove {} connection " + - "and actually removed {} connections", num, removed.size()); + LOG.debug("Expected to remove {} connection and actually removed {} connections", + num, removed.size()); return removed; } @@ -303,7 +322,6 @@ protected int getNumConnections() { */ protected int getNumActiveConnections() { int ret = 0; - List tmpConnections = this.connections; for (ConnectionContext conn : tmpConnections) { if (conn.isActive()) { @@ -320,10 +338,9 @@ protected int getNumActiveConnections() { */ protected int getNumIdleConnections() { int ret = 0; - List tmpConnections = this.connections; for (ConnectionContext conn : tmpConnections) { - if (conn.isUsable()) { + if (conn.isIdle()) { ret++; } } @@ -393,8 +410,9 @@ public String getJSON() { * @throws IOException If it cannot get a new connection. */ public ConnectionContext newConnection() throws IOException { - return newConnection( - this.conf, this.namenodeAddress, this.ugi, this.protocol); + return newConnection(this.conf, this.namenodeAddress, + this.ugi, this.protocol, this.enableMultiSocket, + this.socketIndex.incrementAndGet(), alignmentContext); } /** @@ -402,19 +420,22 @@ public ConnectionContext newConnection() throws IOException { * context for a single user/security context. To maximize throughput it is * recommended to use multiple connection per user+server, allowing multiple * writes and reads to be dispatched in parallel. - * @param + * @param Input type T. * * @param conf Configuration for the connection. * @param nnAddress Address of server supporting the ClientProtocol. * @param ugi User context. * @param proto Interface of the protocol. + * @param enableMultiSocket Enable multiple socket or not. + * @param alignmentContext Client alignment context. * @return proto for the target ClientProtocol that contains the user's * security context. * @throws IOException If it cannot be created. */ protected static ConnectionContext newConnection(Configuration conf, - String nnAddress, UserGroupInformation ugi, Class proto) - throws IOException { + String nnAddress, UserGroupInformation ugi, Class proto, + boolean enableMultiSocket, int socketIndex, + AlignmentContext alignmentContext) throws IOException { if (!PROTO_MAP.containsKey(proto)) { String msg = "Unsupported protocol for connection to NameNode: " + ((proto != null) ? proto.getName() : "null"); @@ -437,15 +458,24 @@ protected static ConnectionContext newConnection(Configuration conf, } InetSocketAddress socket = NetUtils.createSocketAddr(nnAddress); final long version = RPC.getProtocolVersion(classes.protoPb); - Object proxy = RPC.getProtocolProxy(classes.protoPb, version, socket, ugi, - conf, factory, RPC.getRpcTimeout(conf), defaultPolicy, null).getProxy(); + Object proxy; + if (enableMultiSocket) { + FederationConnectionId connectionId = new FederationConnectionId( + socket, classes.protoPb, ugi, RPC.getRpcTimeout(conf), + defaultPolicy, conf, socketIndex); + proxy = RPC.getProtocolProxy(classes.protoPb, version, connectionId, + conf, factory, alignmentContext).getProxy(); + } else { + proxy = RPC.getProtocolProxy(classes.protoPb, version, socket, ugi, + conf, factory, RPC.getRpcTimeout(conf), defaultPolicy, null, + alignmentContext).getProxy(); + } + T client = newProtoClient(proto, classes, proxy); Text dtService = SecurityUtil.buildTokenService(socket); - ProxyAndInfo clientProxy = - new ProxyAndInfo(client, dtService, socket); - ConnectionContext connection = new ConnectionContext(clientProxy); - return connection; + ProxyAndInfo clientProxy = new ProxyAndInfo(client, dtService, socket); + return new ConnectionContext(clientProxy, conf); } private static T newProtoClient(Class proto, ProtoImpl classes, @@ -453,7 +483,7 @@ private static T newProtoClient(Class proto, ProtoImpl classes, try { Constructor constructor = classes.protoClientPb.getConstructor(classes.protoPb); - Object o = constructor.newInstance(new Object[] {proxy}); + Object o = constructor.newInstance(proxy); if (proto.isAssignableFrom(o.getClass())) { @SuppressWarnings("unchecked") T client = (T) o; diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/FederationConnectionId.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/FederationConnectionId.java new file mode 100644 index 0000000000000..0be1f8b1be2a5 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/FederationConnectionId.java @@ -0,0 +1,60 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hdfs.server.federation.router; + +import org.apache.commons.lang3.builder.EqualsBuilder; +import org.apache.commons.lang3.builder.HashCodeBuilder; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.io.retry.RetryPolicy; +import org.apache.hadoop.ipc.Client; +import org.apache.hadoop.security.UserGroupInformation; + +import java.net.InetSocketAddress; + +public class FederationConnectionId extends Client.ConnectionId { + private final int index; + + public FederationConnectionId(InetSocketAddress address, Class protocol, + UserGroupInformation ticket, int rpcTimeout, + RetryPolicy connectionRetryPolicy, Configuration conf, int index) { + super(address, protocol, ticket, rpcTimeout, connectionRetryPolicy, conf); + this.index = index; + } + + @Override + public int hashCode() { + return new HashCodeBuilder() + .append(super.hashCode()) + .append(this.index) + .toHashCode(); + } + + @Override + public boolean equals(Object obj) { + if (!super.equals(obj)) { + return false; + } + if (obj instanceof FederationConnectionId) { + FederationConnectionId other = (FederationConnectionId)obj; + return new EqualsBuilder() + .append(this.index, other.index) + .isEquals(); + } + return false; + } +} diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/NamenodeHeartbeatService.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/NamenodeHeartbeatService.java index ad9d5e2c2a72c..b2f60d931499c 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/NamenodeHeartbeatService.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/NamenodeHeartbeatService.java @@ -17,6 +17,8 @@ */ package org.apache.hadoop.hdfs.server.federation.router; +import static org.apache.hadoop.hdfs.server.federation.router.RBFConfigKeys.DFS_ROUTER_HEALTH_MONITOR_TIMEOUT; +import static org.apache.hadoop.hdfs.server.federation.router.RBFConfigKeys.DFS_ROUTER_HEALTH_MONITOR_TIMEOUT_DEFAULT; import static org.apache.hadoop.hdfs.server.federation.router.RBFConfigKeys.DFS_ROUTER_HEARTBEAT_INTERVAL_MS; import static org.apache.hadoop.hdfs.server.federation.router.RBFConfigKeys.DFS_ROUTER_HEARTBEAT_INTERVAL_MS_DEFAULT; @@ -25,6 +27,7 @@ import java.net.InetSocketAddress; import java.net.URI; import java.util.Map; +import java.util.concurrent.TimeUnit; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.ha.HAServiceProtocol; @@ -85,6 +88,10 @@ public class NamenodeHeartbeatService extends PeriodicService { private NNHAServiceTarget localTarget; /** Cache HA protocol. */ private HAServiceProtocol localTargetHAProtocol; + /** Cache NN protocol. */ + private NamenodeProtocol namenodeProtocol; + /** Cache Client protocol. */ + private ClientProtocol clientProtocol; /** RPC address for the namenode. */ private String rpcAddress; /** Service RPC address for the namenode. */ @@ -100,6 +107,9 @@ public class NamenodeHeartbeatService extends PeriodicService { private String resolvedHost; private String originalNnId; + + private int healthMonitorTimeoutMs = (int) DFS_ROUTER_HEALTH_MONITOR_TIMEOUT_DEFAULT; + /** * Create a new Namenode status updater. * @param resolver Namenode resolver service to handle NN registration. @@ -211,6 +221,15 @@ protected void serviceInit(Configuration configuration) throws Exception { DFS_ROUTER_HEARTBEAT_INTERVAL_MS, DFS_ROUTER_HEARTBEAT_INTERVAL_MS_DEFAULT)); + long timeoutMs = conf.getTimeDuration(DFS_ROUTER_HEALTH_MONITOR_TIMEOUT, + DFS_ROUTER_HEALTH_MONITOR_TIMEOUT_DEFAULT, TimeUnit.MILLISECONDS); + if (timeoutMs < 0) { + LOG.warn("Invalid value {} configured for {} should be greater than or equal to 0. " + + "Using value of : 0ms instead.", timeoutMs, DFS_ROUTER_HEALTH_MONITOR_TIMEOUT); + this.healthMonitorTimeoutMs = 0; + } else { + this.healthMonitorTimeoutMs = (int) timeoutMs; + } super.serviceInit(configuration); } @@ -309,66 +328,26 @@ protected NamenodeStatusReport getNamenodeStatusReport() { LOG.debug("Probing NN at service address: {}", serviceAddress); URI serviceURI = new URI("hdfs://" + serviceAddress); - // Read the filesystem info from RPC (required) - NamenodeProtocol nn = NameNodeProxies - .createProxy(this.conf, serviceURI, NamenodeProtocol.class) - .getProxy(); - if (nn != null) { - NamespaceInfo info = nn.versionRequest(); - if (info != null) { - report.setNamespaceInfo(info); - } - } + // Read the filesystem info from RPC (required) + updateNameSpaceInfoParameters(serviceURI, report); if (!report.registrationValid()) { return report; } // Check for safemode from the client protocol. Currently optional, but // should be required at some point for QoS - try { - ClientProtocol client = NameNodeProxies - .createProxy(this.conf, serviceURI, ClientProtocol.class) - .getProxy(); - if (client != null) { - boolean isSafeMode = client.setSafeMode( - SafeModeAction.SAFEMODE_GET, false); - report.setSafeMode(isSafeMode); - } - } catch (Exception e) { - LOG.error("Cannot fetch safemode state for {}", getNamenodeDesc(), e); - } + updateSafeModeParameters(serviceURI, report); // Read the stats from JMX (optional) updateJMXParameters(webAddress, report); - if (localTarget != null) { - // Try to get the HA status - try { - // Determine if NN is active - // TODO: dynamic timeout - if (localTargetHAProtocol == null) { - localTargetHAProtocol = localTarget.getHealthMonitorProxy(conf, 30*1000); - LOG.debug("Get HA status with address {}", lifelineAddress); - } - HAServiceStatus status = localTargetHAProtocol.getServiceStatus(); - report.setHAServiceState(status.getState()); - } catch (Throwable e) { - if (e.getMessage().startsWith("HA for namenode is not enabled")) { - LOG.error("HA for {} is not enabled", getNamenodeDesc()); - localTarget = null; - } else { - // Failed to fetch HA status, ignoring failure - LOG.error("Cannot fetch HA status for {}: {}", - getNamenodeDesc(), e.getMessage(), e); - } - localTargetHAProtocol = null; - } - } - } catch(IOException e) { + // Try to get the HA status + updateHAStatusParameters(report); + } catch (IOException e) { LOG.error("Cannot communicate with {}: {}", getNamenodeDesc(), e.getMessage()); - } catch(Throwable e) { + } catch (Throwable e) { // Generic error that we don't know about LOG.error("Unexpected exception while communicating with {}: {}", getNamenodeDesc(), e.getMessage(), e); @@ -399,6 +378,59 @@ private static String getNnHeartBeatServiceName(String nsId, String nnId) { (nnId == null ? "" : " " + nnId); } + /** + * Get the namespace information for a Namenode via RPC and add them to the report. + * @param serviceURI Server address of the Namenode to monitor. + * @param report Namenode status report updating with namespace information data. + * @throws IOException This method will throw IOException up, because RBF need + * use Namespace Info to identify this NS. If there are some IOExceptions, + * RBF doesn't need to get other information from NameNode, + * so throw IOException up. + */ + private void updateNameSpaceInfoParameters(URI serviceURI, + NamenodeStatusReport report) throws IOException { + try { + if (this.namenodeProtocol == null) { + this.namenodeProtocol = NameNodeProxies.createProxy(this.conf, serviceURI, + NamenodeProtocol.class).getProxy(); + } + if (namenodeProtocol != null) { + NamespaceInfo info = namenodeProtocol.versionRequest(); + if (info != null) { + report.setNamespaceInfo(info); + } + } + } catch (IOException e) { + this.namenodeProtocol = null; + throw e; + } + } + + /** + * Get the safemode information for a Namenode via RPC and add them to the report. + * Safemode is only one status of NameNode and is useless for RBF identify one NameNode. + * So If there are some IOExceptions, RBF can just ignore it and try to collect + * other information form namenode continue. + * @param serviceURI Server address of the Namenode to monitor. + * @param report Namenode status report updating with safemode information data. + */ + private void updateSafeModeParameters(URI serviceURI, NamenodeStatusReport report) { + try { + if (this.clientProtocol == null) { + this.clientProtocol = NameNodeProxies + .createProxy(this.conf, serviceURI, ClientProtocol.class) + .getProxy(); + } + if (clientProtocol != null) { + boolean isSafeMode = clientProtocol.setSafeMode(SafeModeAction.SAFEMODE_GET, false); + report.setSafeMode(isSafeMode); + } + } catch (Exception e) { + LOG.error("Cannot fetch safemode state for {}", getNamenodeDesc(), e); + this.clientProtocol = null; + } + } + /** * Get the parameters for a Namenode from JMX and add them to the report. * @param address Web interface of the Namenode to monitor. @@ -415,6 +447,34 @@ private void updateJMXParameters( } } + /** + * Get the HA status for a Namenode via RPC and add them to the report. + * @param report Namenode status report updating with HA status information data. + */ + private void updateHAStatusParameters(NamenodeStatusReport report) { + if (localTarget != null) { + try { + // Determine if NN is active + if (localTargetHAProtocol == null) { + localTargetHAProtocol = localTarget.getHealthMonitorProxy( + conf, this.healthMonitorTimeoutMs); + LOG.debug("Get HA status with address {}", lifelineAddress); + } + HAServiceStatus status = localTargetHAProtocol.getServiceStatus(); + report.setHAServiceState(status.getState()); + } catch (Throwable e) { + if (e.getMessage().startsWith("HA for namenode is not enabled")) { + LOG.error("HA for {} is not enabled", getNamenodeDesc()); + localTarget = null; + } else { + // Failed to fetch HA status, ignoring failure + LOG.error("Cannot fetch HA status for {}", getNamenodeDesc(), e); + } + localTargetHAProtocol = null; + } + } + } + /** * Fetches NamenodeInfo metrics from namenode. * @param address Web interface of the Namenode to monitor. diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/NoLocationException.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/NoLocationException.java new file mode 100644 index 0000000000000..1a010ac546cc5 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/NoLocationException.java @@ -0,0 +1,33 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hdfs.server.federation.router; + +import java.io.IOException; + +/** + * This exception is thrown when can not get any mount point for the input path. + * RBF cannot forward any requests for the path. + */ +public class NoLocationException extends IOException { + + private static final long serialVersionUID = 1L; + + public NoLocationException(String path, Class t) { + super("Cannot find locations for " + path + " in " + t.getSimpleName()); + } +} diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/PeriodicService.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/PeriodicService.java index b690b8685c0a4..0883ba3a3dbb9 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/PeriodicService.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/PeriodicService.java @@ -167,21 +167,18 @@ protected synchronized void startPeriodic() { stopPeriodic(); // Create the runnable service - Runnable updateRunnable = new Runnable() { - @Override - public void run() { - LOG.debug("Running {} update task", serviceName); - try { - if (!isRunning) { - return; - } - periodicInvoke(); - runCount++; - lastRun = Time.now(); - } catch (Exception ex) { - errorCount++; - LOG.warn(serviceName + " service threw an exception", ex); + Runnable updateRunnable = () -> { + LOG.debug("Running {} update task", serviceName); + try { + if (!isRunning) { + return; } + periodicInvoke(); + runCount++; + lastRun = Time.now(); + } catch (Exception ex) { + errorCount++; + LOG.warn("{} service threw an exception", serviceName, ex); } }; diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/PoolAlignmentContext.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/PoolAlignmentContext.java new file mode 100644 index 0000000000000..571f41c4d542c --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/PoolAlignmentContext.java @@ -0,0 +1,103 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hdfs.server.federation.router; + +import java.io.IOException; +import java.util.concurrent.atomic.LongAccumulator; +import org.apache.hadoop.ipc.AlignmentContext; +import org.apache.hadoop.ipc.protobuf.RpcHeaderProtos; + + +/** + * An alignment context shared by all connections in a {@link ConnectionPool}. + * There is a distinct connection pool for each [namespace,UGI] pairing. + *

    + * {@link #sharedGlobalStateId} is a reference to a + * shared {@link LongAccumulator} object in the {@link RouterStateIdContext}. + * {@link #poolLocalStateId} is specific to each PoolAlignmentContext. + *

    + * The shared {@link #sharedGlobalStateId} is updated only using + * responses from NameNodes, so clients cannot poison it. + * {@link #poolLocalStateId} is used to propagate client observed + * state into NameNode requests. A misbehaving client can poison this but the effect is only + * visible to other clients with the same UGI and accessing the same namespace. + */ +public class PoolAlignmentContext implements AlignmentContext { + private LongAccumulator sharedGlobalStateId; + private LongAccumulator poolLocalStateId; + + PoolAlignmentContext(RouterStateIdContext routerStateIdContext, String namespaceId) { + sharedGlobalStateId = routerStateIdContext.getNamespaceStateId(namespaceId); + poolLocalStateId = new LongAccumulator(Math::max, Long.MIN_VALUE); + } + + /** + * Client side implementation only receives state alignment info. + * It does not provide state alignment info therefore this does nothing. + */ + @Override + public void updateResponseState(RpcHeaderProtos.RpcResponseHeaderProto.Builder header) { + // Do nothing. + } + + /** + * Router updates a globally shared value using response from + * namenodes. + */ + @Override + public void receiveResponseState(RpcHeaderProtos.RpcResponseHeaderProto header) { + sharedGlobalStateId.accumulate(header.getStateId()); + } + + /** + * Client side implementation for routers to provide state info in requests to + * namenodes. + */ + @Override + public void updateRequestState(RpcHeaderProtos.RpcRequestHeaderProto.Builder header) { + long maxStateId = Long.max(poolLocalStateId.get(), sharedGlobalStateId.get()); + header.setStateId(maxStateId); + } + + /** + * Client side implementation only provides state alignment info in requests. + * Client does not receive RPC requests therefore this does nothing. + */ + @Override + public long receiveRequestState(RpcHeaderProtos.RpcRequestHeaderProto header, long threshold) + throws IOException { + // Do nothing. + return 0; + } + + @Override + public long getLastSeenStateId() { + return sharedGlobalStateId.get(); + } + + @Override + public boolean isCoordinatedCall(String protocolName, String method) { + throw new UnsupportedOperationException( + "Client should not be checking uncoordinated call"); + } + + public void advanceClientStateId(Long clientStateId) { + poolLocalStateId.accumulate(clientStateId); + } +} diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RBFConfigKeys.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RBFConfigKeys.java index c0a9e3f294cd8..c598076f636e7 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RBFConfigKeys.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RBFConfigKeys.java @@ -96,6 +96,10 @@ public class RBFConfigKeys extends CommonConfigurationKeysPublic { FEDERATION_ROUTER_PREFIX + "heartbeat.interval"; public static final long DFS_ROUTER_HEARTBEAT_INTERVAL_MS_DEFAULT = TimeUnit.SECONDS.toMillis(5); + public static final String DFS_ROUTER_HEALTH_MONITOR_TIMEOUT = + FEDERATION_ROUTER_PREFIX + "health.monitor.timeout"; + public static final long DFS_ROUTER_HEALTH_MONITOR_TIMEOUT_DEFAULT = + TimeUnit.SECONDS.toMillis(30); public static final String DFS_ROUTER_MONITOR_NAMENODE = FEDERATION_ROUTER_PREFIX + "monitor.namenode"; public static final String DFS_ROUTER_MONITOR_NAMENODE_RESOLUTION_ENABLED = @@ -135,6 +139,12 @@ public class RBFConfigKeys extends CommonConfigurationKeysPublic { FEDERATION_ROUTER_PREFIX + "connection.clean.ms"; public static final long DFS_ROUTER_NAMENODE_CONNECTION_CLEAN_MS_DEFAULT = TimeUnit.SECONDS.toMillis(10); + public static final String DFS_ROUTER_NAMENODE_ENABLE_MULTIPLE_SOCKET_KEY = + FEDERATION_ROUTER_PREFIX + "enable.multiple.socket"; + public static final boolean DFS_ROUTER_NAMENODE_ENABLE_MULTIPLE_SOCKET_DEFAULT = false; + public static final String DFS_ROUTER_MAX_CONCURRENCY_PER_CONNECTION_KEY = + FEDERATION_ROUTER_PREFIX + "max.concurrency.per.connection"; + public static final int DFS_ROUTER_MAX_CONCURRENCY_PER_CONNECTION_DEFAULT = 1; // HDFS Router RPC client public static final String DFS_ROUTER_CLIENT_THREADS_SIZE = @@ -181,6 +191,16 @@ public class RBFConfigKeys extends CommonConfigurationKeysPublic { FEDERATION_STORE_PREFIX + "enable"; public static final boolean DFS_ROUTER_STORE_ENABLE_DEFAULT = true; + public static final String DFS_ROUTER_OBSERVER_READ_DEFAULT_KEY = + FEDERATION_ROUTER_PREFIX + "observer.read.default"; + public static final boolean DFS_ROUTER_OBSERVER_READ_DEFAULT_VALUE = false; + public static final String DFS_ROUTER_OBSERVER_READ_OVERRIDES = + FEDERATION_ROUTER_PREFIX + "observer.read.overrides"; + + public static final String DFS_ROUTER_OBSERVER_FEDERATED_STATE_PROPAGATION_MAXSIZE = + FEDERATION_ROUTER_PREFIX + "observer.federated.state.propagation.maxsize"; + public static final int DFS_ROUTER_OBSERVER_FEDERATED_STATE_PROPAGATION_MAXSIZE_DEFAULT = 5; + public static final String FEDERATION_STORE_SERIALIZER_CLASS = FEDERATION_STORE_PREFIX + "serializer"; public static final Class @@ -315,6 +335,9 @@ public class RBFConfigKeys extends CommonConfigurationKeysPublic { FEDERATION_ROUTER_PREFIX + "dn-report.cache-expire"; public static final long DN_REPORT_CACHE_EXPIRE_MS_DEFAULT = TimeUnit.SECONDS.toMillis(10); + public static final String DFS_ROUTER_ENABLE_GET_DN_USAGE_KEY = + FEDERATION_ROUTER_PREFIX + "enable.get.dn.usage"; + public static final boolean DFS_ROUTER_ENABLE_GET_DN_USAGE_DEFAULT = true; // HDFS Router-based federation quota public static final String DFS_ROUTER_QUOTA_ENABLE = @@ -354,6 +377,10 @@ public class RBFConfigKeys extends CommonConfigurationKeysPublic { NoRouterRpcFairnessPolicyController.class; public static final String DFS_ROUTER_FAIR_HANDLER_COUNT_KEY_PREFIX = FEDERATION_ROUTER_FAIRNESS_PREFIX + "handler.count."; + public static final String DFS_ROUTER_FAIRNESS_ACQUIRE_TIMEOUT = + FEDERATION_ROUTER_FAIRNESS_PREFIX + "acquire.timeout"; + public static final long DFS_ROUTER_FAIRNESS_ACQUIRE_TIMEOUT_DEFAULT = + TimeUnit.SECONDS.toMillis(1); // HDFS Router Federation Rename. public static final String DFS_ROUTER_FEDERATION_RENAME_PREFIX = diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/Router.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/Router.java index 6d52bc26cdf63..7cca21dccf196 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/Router.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/Router.java @@ -757,7 +757,7 @@ public ActiveNamenodeResolver getNamenodeResolver() { /** * Get the state store interface for the router heartbeats. * - * @return FederationRouterStateStore state store API handle. + * @return RouterStore state store API handle. */ public RouterStore getRouterStateManager() { if (this.routerStateManager == null && this.stateStore != null) { diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterClientProtocol.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterClientProtocol.java index c1dafec92203b..a5f83c95b7baf 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterClientProtocol.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterClientProtocol.java @@ -115,6 +115,7 @@ import java.util.Set; import java.util.TreeMap; import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; /** * Module that implements all the RPC calls in {@link ClientProtocol} in the @@ -291,8 +292,10 @@ public HdfsFileStatus create(String src, FsPermission masked, RemoteLocation createLocation = null; try { createLocation = rpcServer.getCreateLocation(src, locations); - return rpcClient.invokeSingle(createLocation, method, + HdfsFileStatus status = rpcClient.invokeSingle(createLocation, method, HdfsFileStatus.class); + status.setNamespace(createLocation.getNameserviceId()); + return status; } catch (IOException ioe) { final List newLocations = checkFaultTolerantRetry( method, src, ioe, createLocation, locations); @@ -377,8 +380,11 @@ public LastBlockWithStatus append(String src, final String clientName, RemoteMethod method = new RemoteMethod("append", new Class[] {String.class, String.class, EnumSetWritable.class}, new RemoteParam(), clientName, flag); - return rpcClient.invokeSequential( - locations, method, LastBlockWithStatus.class, null); + RemoteResult result = rpcClient.invokeSequential( + method, locations, LastBlockWithStatus.class, null); + LastBlockWithStatus lbws = (LastBlockWithStatus) result.getResult(); + lbws.getFileStatus().setNamespace(result.getLocation().getNameserviceId()); + return lbws; } @Override @@ -759,14 +765,49 @@ public boolean mkdirs(String src, FsPermission masked, boolean createParent) } } + private Map getAvailableNamespaces() + throws IOException { + Map allAvailableNamespaces = + new HashMap<>(); + namenodeResolver.getNamespaces().forEach( + k -> allAvailableNamespaces.put(k.getNameserviceId(), k)); + return allAvailableNamespaces; + } + + /** + * Try to get a list of FederationNamespaceInfo for renewLease RPC. + */ + private List getRenewLeaseNSs(List namespaces) + throws IOException { + if (namespaces == null || namespaces.isEmpty()) { + return new ArrayList<>(namenodeResolver.getNamespaces()); + } + List result = new ArrayList<>(); + Map allAvailableNamespaces = + getAvailableNamespaces(); + for (String namespace : namespaces) { + if (!allAvailableNamespaces.containsKey(namespace)) { + return new ArrayList<>(namenodeResolver.getNamespaces()); + } else { + result.add(allAvailableNamespaces.get(namespace)); + } + } + return result; + } + @Override - public void renewLease(String clientName) throws IOException { + public void renewLease(String clientName, List namespaces) + throws IOException { rpcServer.checkOperation(NameNode.OperationCategory.WRITE); RemoteMethod method = new RemoteMethod("renewLease", - new Class[] {String.class}, clientName); - Set nss = namenodeResolver.getNamespaces(); - rpcClient.invokeConcurrent(nss, method, false, false); + new Class[] {String.class, List.class}, clientName, null); + List nss = getRenewLeaseNSs(namespaces); + if (nss.size() == 1) { + rpcClient.invokeSingle(nss.get(0).getNameserviceId(), method); + } else { + rpcClient.invokeConcurrent(nss, method, false, false); + } } @Override @@ -895,19 +936,22 @@ public BatchedDirectoryListing getBatchedListing(String[] srcs, public HdfsFileStatus getFileInfo(String src) throws IOException { rpcServer.checkOperation(NameNode.OperationCategory.READ); - final List locations = - rpcServer.getLocationsForPath(src, false, false); - RemoteMethod method = new RemoteMethod("getFileInfo", - new Class[] {String.class}, new RemoteParam()); - HdfsFileStatus ret = null; - // If it's a directory, we check in all locations - if (rpcServer.isPathAll(src)) { - ret = getFileInfoAll(locations, method); - } else { - // Check for file information sequentially - ret = rpcClient.invokeSequential( - locations, method, HdfsFileStatus.class, null); + IOException noLocationException = null; + try { + final List locations = rpcServer.getLocationsForPath(src, false, false); + RemoteMethod method = new RemoteMethod("getFileInfo", + new Class[] {String.class}, new RemoteParam()); + + // If it's a directory, we check in all locations + if (rpcServer.isPathAll(src)) { + ret = getFileInfoAll(locations, method); + } else { + // Check for file information sequentially + ret = rpcClient.invokeSequential(locations, method, HdfsFileStatus.class, null); + } + } catch (NoLocationException | RouterResolveException e) { + noLocationException = e; } // If there is no real path, check mount points @@ -926,6 +970,12 @@ public HdfsFileStatus getFileInfo(String src) throws IOException { } } + // Can't find mount point for path and the path didn't contain any sub monit points, + // throw the NoLocationException to client. + if (ret == null && noLocationException != null) { + throw noLocationException; + } + return ret; } @@ -1202,14 +1252,93 @@ public void setBalancerBandwidth(long bandwidth) throws IOException { rpcClient.invokeConcurrent(nss, method, true, false); } + /** + * Recursively get all the locations for the path. + * For example, there are some mount points: + * /a -> ns0 -> /a + * /a/b -> ns1 -> /a/b + * /a/b/c -> ns2 -> /a/b/c + * When the path is '/a', the result of locations should be + * {ns0 -> [RemoteLocation(/a)], ns1 -> [RemoteLocation(/a/b)], ns2 -> [RemoteLocation(/a/b/c)]} + * @param path the path to get the locations. + * @return a map to store all the locations and key is namespace id. + * @throws IOException + */ + @VisibleForTesting + Map> getAllLocations(String path) throws IOException { + Map> locations = new HashMap<>(); + try { + List parentLocations = rpcServer.getLocationsForPath(path, false, false); + parentLocations.forEach( + l -> locations.computeIfAbsent(l.getNameserviceId(), k -> new ArrayList<>()).add(l)); + } catch (NoLocationException | RouterResolveException e) { + LOG.debug("Cannot find locations for {}.", path); + } + + final List children = subclusterResolver.getMountPoints(path); + if (children != null) { + for (String child : children) { + String childPath = new Path(path, child).toUri().getPath(); + Map> childLocations = getAllLocations(childPath); + childLocations.forEach( + (k, v) -> locations.computeIfAbsent(k, l -> new ArrayList<>()).addAll(v)); + } + } + return locations; + } + + /** + * Get all the locations of the path for {@link this#getContentSummary(String)}. + * For example, there are some mount points: + * /a -> ns0 -> /a + * /a/b -> ns0 -> /a/b + * /a/b/c -> ns1 -> /a/b/c + * When the path is '/a', the result of locations should be + * [RemoteLocation('/a', ns0, '/a'), RemoteLocation('/a/b/c', ns1, '/a/b/c')] + * When the path is '/b', will throw NoLocationException. + * @param path the path to get content summary + * @return one list contains all the remote location + * @throws IOException + */ + @VisibleForTesting + List getLocationsForContentSummary(String path) throws IOException { + // Try to get all the locations of the path. + final Map> ns2Locations = getAllLocations(path); + if (ns2Locations.isEmpty()) { + throw new NoLocationException(path, subclusterResolver.getClass()); + } + + final List locations = new ArrayList<>(); + // remove the redundancy remoteLocation order by destination. + ns2Locations.forEach((k, v) -> { + List sortedList = v.stream().sorted().collect(Collectors.toList()); + int size = sortedList.size(); + for (int i = size - 1; i > -1; i--) { + RemoteLocation currentLocation = sortedList.get(i); + if (i == 0) { + locations.add(currentLocation); + } else { + RemoteLocation preLocation = sortedList.get(i - 1); + if (!currentLocation.getDest().startsWith(preLocation.getDest() + Path.SEPARATOR)) { + locations.add(currentLocation); + } else { + LOG.debug("Ignore redundant location {}, because there is an ancestor location {}", + currentLocation, preLocation); + } + } + } + }); + + return locations; + } + @Override public ContentSummary getContentSummary(String path) throws IOException { rpcServer.checkOperation(NameNode.OperationCategory.READ); // Get the summaries from regular files final Collection summaries = new ArrayList<>(); - final List locations = - rpcServer.getLocationsForPath(path, false, false); + final List locations = getLocationsForContentSummary(path); final RemoteMethod method = new RemoteMethod("getContentSummary", new Class[] {String.class}, new RemoteParam()); final List> results = @@ -1229,24 +1358,6 @@ public ContentSummary getContentSummary(String path) throws IOException { } } - // Add mount points at this level in the tree - final List children = subclusterResolver.getMountPoints(path); - if (children != null) { - for (String child : children) { - Path childPath = new Path(path, child); - try { - ContentSummary mountSummary = getContentSummary( - childPath.toString()); - if (mountSummary != null) { - summaries.add(mountSummary); - } - } catch (Exception e) { - LOG.error("Cannot get content summary for mount {}: {}", - childPath, e.getMessage()); - } - } - } - // Throw original exception if no original nor mount points if (summaries.isEmpty() && notFoundException != null) { throw notFoundException; @@ -1807,7 +1918,10 @@ public BatchedEntries listOpenFiles(long prevId, @Override public void msync() throws IOException { - rpcServer.checkOperation(NameNode.OperationCategory.READ, false); + rpcServer.checkOperation(NameNode.OperationCategory.READ, true); + Set nss = namenodeResolver.getNamespaces(); + RemoteMethod method = new RemoteMethod("msync"); + rpcClient.invokeConcurrent(nss, method); } @Override @@ -2165,7 +2279,7 @@ private List> getListingInt( .invokeConcurrent(locations, method, false, -1, DirectoryListing.class); return listings; - } catch (RouterResolveException e) { + } catch (NoLocationException | RouterResolveException e) { LOG.debug("Cannot get locations for {}, {}.", src, e.getMessage()); return new ArrayList<>(); } diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterFsckServlet.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterFsckServlet.java index 41216fce702ba..a439e5c0ce84e 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterFsckServlet.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterFsckServlet.java @@ -25,20 +25,19 @@ import java.util.Map; import javax.servlet.ServletContext; -import javax.servlet.http.HttpServlet; import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hdfs.server.common.JspHelper; +import org.apache.hadoop.hdfs.server.namenode.DfsServlet; import org.apache.hadoop.security.UserGroupInformation; /** * This class is used in Namesystem's web server to do fsck on namenode. */ @InterfaceAudience.Private -public class RouterFsckServlet extends HttpServlet { +public class RouterFsckServlet extends DfsServlet { /** for java.io.Serializable. */ private static final long serialVersionUID = 1L; @@ -67,15 +66,4 @@ public void doGet(HttpServletRequest request, HttpServletResponse response) } } - /** - * Copy from {@link org.apache.hadoop.hdfs.server.namenode.DfsServlet}. - * @param request Http request from the user - * @param conf configuration - * @return ugi of the requested user - * @throws IOException failed to get ugi - */ - protected UserGroupInformation getUGI(HttpServletRequest request, - Configuration conf) throws IOException { - return JspHelper.getUGI(getServletContext(), request, conf); - } } diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterHeartbeatService.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterHeartbeatService.java index 1316cf7184988..19d7442acb6ac 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterHeartbeatService.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterHeartbeatService.java @@ -63,12 +63,7 @@ public RouterHeartbeatService(Router router) { * Trigger the update of the Router state asynchronously. */ protected void updateStateAsync() { - Thread thread = new Thread(new Runnable() { - @Override - public void run() { - updateStateStore(); - } - }, "Router Heartbeat Async"); + Thread thread = new Thread(this::updateStateStore, "Router Heartbeat Async"); thread.setDaemon(true); thread.start(); } diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcClient.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcClient.java index ff90854ebb7ec..62ae4b0b95de7 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcClient.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcClient.java @@ -22,6 +22,7 @@ import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_CALLER_CONTEXT_SEPARATOR_KEY; import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.IPC_CLIENT_CONNECT_MAX_RETRIES_ON_SOCKET_TIMEOUTS_KEY; import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.IPC_CLIENT_CONNECT_TIMEOUT_KEY; +import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_IP_PROXY_USERS; import static org.apache.hadoop.hdfs.server.federation.fairness.RouterRpcFairnessConstants.CONCURRENT_NS; import java.io.EOFException; @@ -37,6 +38,7 @@ import java.util.Arrays; import java.util.Collection; import java.util.Collections; +import java.util.HashSet; import java.util.LinkedHashMap; import java.util.LinkedList; import java.util.List; @@ -69,17 +71,21 @@ import org.apache.hadoop.hdfs.server.federation.resolver.FederationNamenodeContext; import org.apache.hadoop.hdfs.server.federation.resolver.FederationNamenodeServiceState; import org.apache.hadoop.hdfs.server.federation.resolver.RemoteLocation; +import org.apache.hadoop.hdfs.server.namenode.ha.ReadOnly; import org.apache.hadoop.io.retry.RetryPolicies; import org.apache.hadoop.io.retry.RetryPolicy; import org.apache.hadoop.io.retry.RetryPolicy.RetryAction.RetryDecision; import org.apache.hadoop.ipc.CallerContext; +import org.apache.hadoop.ipc.ObserverRetryOnActiveException; import org.apache.hadoop.ipc.RemoteException; import org.apache.hadoop.ipc.RetriableException; import org.apache.hadoop.ipc.Server; import org.apache.hadoop.ipc.Server.Call; import org.apache.hadoop.ipc.StandbyException; import org.apache.hadoop.net.ConnectTimeoutException; +import org.apache.hadoop.net.NetUtils; import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.util.StringUtils; import org.eclipse.jetty.util.ajax.JSON; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -126,6 +132,10 @@ public class RouterRpcClient { private final RouterRpcMonitor rpcMonitor; /** Field separator of CallerContext. */ private final String contextFieldSeparator; + /** Observer read enabled. Default for all nameservices. */ + private final boolean observerReadEnabledDefault; + /** Nameservice specific overrides of the default setting for enabling observer reads. */ + private HashSet observerReadEnabledOverrides = new HashSet<>(); /** Pattern to parse a stack trace line. */ private static final Pattern STACK_TRACE_PATTERN = @@ -136,6 +146,8 @@ public class RouterRpcClient { private Map rejectedPermitsPerNs = new ConcurrentHashMap<>(); private Map acceptedPermitsPerNs = new ConcurrentHashMap<>(); + private final boolean enableProxyUser; + /** * Create a router RPC client to manage remote procedure calls to NNs. * @@ -145,7 +157,8 @@ public class RouterRpcClient { * @param monitor Optional performance monitor. */ public RouterRpcClient(Configuration conf, Router router, - ActiveNamenodeResolver resolver, RouterRpcMonitor monitor) { + ActiveNamenodeResolver resolver, RouterRpcMonitor monitor, + RouterStateIdContext routerStateIdContext) { this.router = router; this.namenodeResolver = resolver; @@ -154,7 +167,7 @@ public RouterRpcClient(Configuration conf, Router router, this.contextFieldSeparator = clientConf.get(HADOOP_CALLER_CONTEXT_SEPARATOR_KEY, HADOOP_CALLER_CONTEXT_SEPARATOR_DEFAULT); - this.connectionManager = new ConnectionManager(clientConf); + this.connectionManager = new ConnectionManager(clientConf, routerStateIdContext); this.connectionManager.start(); this.routerRpcFairnessPolicyController = FederationUtil.newFairnessPolicyController(conf); @@ -193,6 +206,18 @@ public RouterRpcClient(Configuration conf, Router router, this.retryPolicy = RetryPolicies.failoverOnNetworkException( RetryPolicies.TRY_ONCE_THEN_FAIL, maxFailoverAttempts, maxRetryAttempts, failoverSleepBaseMillis, failoverSleepMaxMillis); + String[] ipProxyUsers = conf.getStrings(DFS_NAMENODE_IP_PROXY_USERS); + this.enableProxyUser = ipProxyUsers != null && ipProxyUsers.length > 0; + this.observerReadEnabledDefault = conf.getBoolean( + RBFConfigKeys.DFS_ROUTER_OBSERVER_READ_DEFAULT_KEY, + RBFConfigKeys.DFS_ROUTER_OBSERVER_READ_DEFAULT_VALUE); + String[] observerReadOverrides = conf.getStrings(RBFConfigKeys.DFS_ROUTER_OBSERVER_READ_OVERRIDES); + if (observerReadOverrides != null) { + observerReadEnabledOverrides.addAll(Arrays.asList(observerReadOverrides)); + } + if (this.observerReadEnabledDefault) { + LOG.info("Observer read is enabled for router."); + } } /** @@ -362,13 +387,13 @@ private ConnectionContext getConnection(UserGroupInformation ugi, String nsId, // TODO Add tokens from the federated UGI UserGroupInformation connUGI = ugi; - if (UserGroupInformation.isSecurityEnabled()) { + if (UserGroupInformation.isSecurityEnabled() || this.enableProxyUser) { UserGroupInformation routerUser = UserGroupInformation.getLoginUser(); connUGI = UserGroupInformation.createProxyUser( ugi.getUserName(), routerUser); } connection = this.connectionManager.getConnection( - connUGI, rpcAddress, proto); + connUGI, rpcAddress, proto, nsId); LOG.debug("User {} NN {} is using connection {}", ugi.getUserName(), rpcAddress, connection); } catch (Exception ex) { @@ -444,6 +469,7 @@ private RetryDecision shouldRetry(final IOException ioe, final int retryCount, * @param ugi User group information. * @param namenodes A prioritized list of namenodes within the same * nameservice. + * @param useObserver Whether to use observer namenodes. * @param method Remote ClientProtocol method to invoke. * @param params Variable list of parameters matching the method. * @return The result of invoking the method. @@ -455,6 +481,7 @@ private RetryDecision shouldRetry(final IOException ioe, final int retryCount, public Object invokeMethod( final UserGroupInformation ugi, final List namenodes, + boolean useObserver, final Class protocol, final Method method, final Object... params) throws ConnectException, StandbyException, IOException { @@ -464,15 +491,19 @@ public Object invokeMethod( + router.getRouterId()); } - addClientIpToCallerContext(); + addClientInfoToCallerContext(); Object ret = null; if (rpcMonitor != null) { rpcMonitor.proxyOp(); } boolean failover = false; + boolean shouldUseObserver = useObserver; Map ioes = new LinkedHashMap<>(); for (FederationNamenodeContext namenode : namenodes) { + if (!shouldUseObserver && (namenode.getState() == FederationNamenodeServiceState.OBSERVER)) { + continue; + } ConnectionContext connection = null; String nsId = namenode.getNameserviceId(); String rpcAddress = namenode.getRpcAddress(); @@ -482,13 +513,14 @@ public Object invokeMethod( final Object proxy = client.getProxy(); ret = invoke(nsId, 0, method, proxy, params); - if (failover) { + if (failover && + FederationNamenodeServiceState.OBSERVER != namenode.getState()) { // Success on alternate server, update InetSocketAddress address = client.getAddress(); namenodeResolver.updateActiveNamenode(nsId, address); } if (this.rpcMonitor != null) { - this.rpcMonitor.proxyOpComplete(true, nsId); + this.rpcMonitor.proxyOpComplete(true, nsId, namenode.getState()); } if (this.router.getRouterClientMetrics() != null) { this.router.getRouterClientMetrics().incInvokedMethod(method); @@ -496,7 +528,11 @@ public Object invokeMethod( return ret; } catch (IOException ioe) { ioes.put(namenode, ioe); - if (ioe instanceof StandbyException) { + if (ioe instanceof ObserverRetryOnActiveException) { + LOG.info("Encountered ObserverRetryOnActiveException from {}." + + " Retry active namenode directly.", namenode); + shouldUseObserver = false; + } else if (ioe instanceof StandbyException) { // Fail over indicated by retry policy and/or NN if (this.rpcMonitor != null) { this.rpcMonitor.proxyOpFailureStandby(nsId); @@ -506,10 +542,15 @@ public Object invokeMethod( if (this.rpcMonitor != null) { this.rpcMonitor.proxyOpFailureCommunicate(nsId); } - failover = true; + if (FederationNamenodeServiceState.OBSERVER == namenode.getState()) { + namenodeResolver.updateUnavailableNamenode(nsId, + NetUtils.createSocketAddr(namenode.getRpcAddress())); + } else { + failover = true; + } } else if (ioe instanceof RemoteException) { if (this.rpcMonitor != null) { - this.rpcMonitor.proxyOpComplete(true, nsId); + this.rpcMonitor.proxyOpComplete(true, nsId, namenode.getState()); } RemoteException re = (RemoteException) ioe; ioe = re.unwrapRemoteException(); @@ -539,7 +580,7 @@ public Object invokeMethod( // Communication retries are handled by the retry policy if (this.rpcMonitor != null) { this.rpcMonitor.proxyOpFailureCommunicate(nsId); - this.rpcMonitor.proxyOpComplete(false, nsId); + this.rpcMonitor.proxyOpComplete(false, nsId, namenode.getState()); } throw ioe; } @@ -550,7 +591,7 @@ public Object invokeMethod( } } if (this.rpcMonitor != null) { - this.rpcMonitor.proxyOpComplete(false, null); + this.rpcMonitor.proxyOpComplete(false, null, null); } // All namenodes were unavailable or in standby @@ -584,12 +625,13 @@ public Object invokeMethod( } /** - * For tracking which is the actual client address. - * It adds trace info "clientIp:ip" and "clientPort:port" + * For tracking some information about the actual client. + * It adds trace info "clientIp:ip", "clientPort:port", + * "clientId:id" and "clientCallId:callId" * in the caller context, removing the old values if they were * already present. */ - private void addClientIpToCallerContext() { + private void addClientInfoToCallerContext() { CallerContext ctx = CallerContext.getCurrent(); String origContext = ctx == null ? null : ctx.getContext(); byte[] origSignature = ctx == null ? null : ctx.getSignature(); @@ -598,6 +640,10 @@ private void addClientIpToCallerContext() { .append(CallerContext.CLIENT_IP_STR, Server.getRemoteAddress()) .append(CallerContext.CLIENT_PORT_STR, Integer.toString(Server.getRemotePort())) + .append(CallerContext.CLIENT_ID_STR, + StringUtils.byteToHexString(Server.getClientId())) + .append(CallerContext.CLIENT_CALL_ID_STR, + Integer.toString(Server.getCallId())) .setSignature(origSignature); // Append the original caller context if (origContext != null) { @@ -628,16 +674,12 @@ private void addClientIpToCallerContext() { * @param params Variable parameters * @return Response from the remote server * @throws IOException - * @throws InterruptedException */ private Object invoke(String nsId, int retryCount, final Method method, final Object obj, final Object... params) throws IOException { try { return method.invoke(obj, params); - } catch (IllegalAccessException e) { - LOG.error("Unexpected exception while proxying API", e); - return null; - } catch (IllegalArgumentException e) { + } catch (IllegalAccessException | IllegalArgumentException e) { LOG.error("Unexpected exception while proxying API", e); return null; } catch (InvocationTargetException e) { @@ -701,7 +743,7 @@ public static boolean isUnavailableException(IOException ioe) { */ private boolean isClusterUnAvailable(String nsId) throws IOException { List nnState = this.namenodeResolver - .getNamenodesForNameserviceId(nsId); + .getNamenodesForNameserviceId(nsId, false); if (nnState != null) { for (FederationNamenodeContext nnContext : nnState) { @@ -832,13 +874,13 @@ public Object invokeSingle(final String nsId, RemoteMethod method) RouterRpcFairnessPolicyController controller = getRouterRpcFairnessPolicyController(); acquirePermit(nsId, ugi, method, controller); try { - List nns = - getNamenodesForNameservice(nsId); + boolean isObserverRead = isObserverReadEligible(nsId, method.getMethod()); + List nns = getOrderedNamenodes(nsId, isObserverRead); RemoteLocationContext loc = new RemoteLocation(nsId, "/", "/"); Class proto = method.getProtocol(); Method m = method.getMethod(); Object[] params = method.getParams(loc); - return invokeMethod(ugi, nns, proto, m, params); + return invokeMethod(ugi, nns, isObserverRead, proto, m, params); } finally { releasePermit(nsId, ugi, method, controller); } @@ -915,7 +957,7 @@ public T invokeSingle(final RemoteLocationContext location, * @throws IOException if the success condition is not met and one of the RPC * calls generated a remote exception. */ - public Object invokeSequential( + public T invokeSequential( final List locations, final RemoteMethod remoteMethod) throws IOException { return invokeSequential(locations, remoteMethod, null, null); @@ -1000,12 +1042,14 @@ public RemoteResult invokeSequential( for (final RemoteLocationContext loc : locations) { String ns = loc.getNameserviceId(); acquirePermit(ns, ugi, remoteMethod, controller); + boolean isObserverRead = isObserverReadEligible(ns, m); List namenodes = - getNamenodesForNameservice(ns); + getOrderedNamenodes(ns, isObserverRead); try { Class proto = remoteMethod.getProtocol(); Object[] params = remoteMethod.getParams(loc); - Object result = invokeMethod(ugi, namenodes, proto, m, params); + Object result = invokeMethod( + ugi, namenodes, isObserverRead, proto, m, params); // Check if the result is what we expected if (isExpectedClass(expectedResultClass, result) && isExpectedValue(expectedResultValue, result)) { @@ -1361,12 +1405,14 @@ public Map invokeConcurrent( String ns = location.getNameserviceId(); RouterRpcFairnessPolicyController controller = getRouterRpcFairnessPolicyController(); acquirePermit(ns, ugi, method, controller); + boolean isObserverRead = isObserverReadEligible(ns, m); final List namenodes = - getNamenodesForNameservice(ns); + getOrderedNamenodes(ns, isObserverRead); try { Class proto = method.getProtocol(); Object[] paramList = method.getParams(location); - R result = (R) invokeMethod(ugi, namenodes, proto, m, paramList); + R result = (R) invokeMethod( + ugi, namenodes, isObserverRead, proto, m, paramList); RemoteResult remoteResult = new RemoteResult<>(location, result); return Collections.singletonList(remoteResult); } catch (IOException ioe) { @@ -1384,8 +1430,9 @@ public Map invokeConcurrent( final CallerContext originContext = CallerContext.getCurrent(); for (final T location : locations) { String nsId = location.getNameserviceId(); + boolean isObserverRead = isObserverReadEligible(nsId, m); final List namenodes = - getNamenodesForNameservice(nsId); + getOrderedNamenodes(nsId, isObserverRead); final Class proto = method.getProtocol(); final Object[] paramList = method.getParams(location); if (standby) { @@ -1402,7 +1449,8 @@ public Map invokeConcurrent( callables.add( () -> { transferThreadLocalContext(originCall, originContext); - return invokeMethod(ugi, nnList, proto, m, paramList); + return invokeMethod( + ugi, nnList, isObserverRead, proto, m, paramList); }); } } else { @@ -1411,7 +1459,8 @@ public Map invokeConcurrent( callables.add( () -> { transferThreadLocalContext(originCall, originContext); - return invokeMethod(ugi, namenodes, proto, m, paramList); + return invokeMethod( + ugi, namenodes, isObserverRead, proto, m, paramList); }); } } @@ -1500,27 +1549,6 @@ private void transferThreadLocalContext( CallerContext.setCurrent(originContext); } - /** - * Get a prioritized list of NNs that share the same nameservice ID (in the - * same namespace). NNs that are reported as ACTIVE will be first in the list. - * - * @param nsId The nameservice ID for the namespace. - * @return A prioritized list of NNs to use for communication. - * @throws IOException If a NN cannot be located for the nameservice ID. - */ - private List getNamenodesForNameservice( - final String nsId) throws IOException { - - final List namenodes = - namenodeResolver.getNamenodesForNameserviceId(nsId); - - if (namenodes == null || namenodes.isEmpty()) { - throw new IOException("Cannot locate a registered namenode for " + nsId + - " from " + router.getRouterId()); - } - return namenodes; - } - /** * Get a prioritized list of NNs that share the same block pool ID (in the * same namespace). NNs that are reported as ACTIVE will be first in the list. @@ -1630,7 +1658,7 @@ public Long getAcceptedPermitForNs(String ns) { /** * Refreshes/changes the fairness policy controller implementation if possible - * and returns the controller class name + * and returns the controller class name. * @param conf Configuration * @return New controller class name if successfully refreshed, else old controller class name */ @@ -1658,4 +1686,48 @@ private String getCurrentFairnessPolicyControllerClassName() { } return null; } + + /** + * Get a prioritized list of NNs that share the same nameservice ID (in the + * same namespace). + * In observer read case, OBSERVER NNs will be first in the list. + * Otherwise, ACTIVE NNs will be first in the list. + * + * @param nsId The nameservice ID for the namespace. + * @param isObserverRead Read on observer namenode. + * @return A prioritized list of NNs to use for communication. + * @throws IOException If a NN cannot be located for the nameservice ID. + */ + private List getOrderedNamenodes(String nsId, + boolean isObserverRead) throws IOException { + final List namenodes; + + if (RouterStateIdContext.getClientStateIdFromCurrentCall(nsId) > Long.MIN_VALUE) { + namenodes = namenodeResolver.getNamenodesForNameserviceId(nsId, isObserverRead); + } else { + namenodes = namenodeResolver.getNamenodesForNameserviceId(nsId, false); + } + + if (namenodes == null || namenodes.isEmpty()) { + throw new IOException("Cannot locate a registered namenode for " + nsId + + " from " + router.getRouterId()); + } + return namenodes; + } + + private boolean isObserverReadEligible(String nsId, Method method) { + boolean isReadEnabledForNamespace = observerReadEnabledDefault != observerReadEnabledOverrides.contains(nsId); + return isReadEnabledForNamespace && isReadCall(method); + } + + /** + * Check if a method is read-only. + * @return whether the 'method' is a read-only operation. + */ + private static boolean isReadCall(Method method) { + if (!method.isAnnotationPresent(ReadOnly.class)) { + return false; + } + return !method.getAnnotationsByType(ReadOnly.class)[0].activeOnly(); + } } diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcMonitor.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcMonitor.java index 039b40ae2e585..256f03f12ff38 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcMonitor.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcMonitor.java @@ -19,6 +19,7 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hdfs.server.federation.metrics.FederationRPCMetrics; +import org.apache.hadoop.hdfs.server.federation.resolver.FederationNamenodeServiceState; import org.apache.hadoop.hdfs.server.federation.store.StateStoreService; /** @@ -61,8 +62,9 @@ void init( /** * Mark a proxy operation as completed. * @param success If the operation was successful. + * @param state proxy namenode state. */ - void proxyOpComplete(boolean success, String nsId); + void proxyOpComplete(boolean success, String nsId, FederationNamenodeServiceState state); /** * Failed to proxy an operation to a Namenode because it was in standby. diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcServer.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcServer.java index 58181dcc346cf..c4173163436ce 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcServer.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcServer.java @@ -252,18 +252,18 @@ public class RouterRpcServer extends AbstractService implements ClientProtocol, /** * Construct a router RPC server. * - * @param configuration HDFS Configuration. + * @param conf HDFS Configuration. * @param router A router using this RPC server. * @param nnResolver The NN resolver instance to determine active NNs in HA. * @param fileResolver File resolver to resolve file paths to subclusters. * @throws IOException If the RPC server could not be created. */ - public RouterRpcServer(Configuration configuration, Router router, + public RouterRpcServer(Configuration conf, Router router, ActiveNamenodeResolver nnResolver, FileSubclusterResolver fileResolver) throws IOException { super(RouterRpcServer.class.getName()); - this.conf = configuration; + this.conf = conf; this.router = router; this.namenodeResolver = nnResolver; this.subclusterResolver = fileResolver; @@ -321,6 +321,7 @@ public RouterRpcServer(Configuration configuration, Router router, // Create security manager this.securityManager = new RouterSecurityManager(this.conf); + RouterStateIdContext routerStateIdContext = new RouterStateIdContext(conf); this.rpcServer = new RPC.Builder(this.conf) .setProtocol(ClientNamenodeProtocolPB.class) @@ -331,6 +332,7 @@ public RouterRpcServer(Configuration configuration, Router router, .setnumReaders(readerCount) .setQueueSizePerHandler(handlerQueueSize) .setVerbose(false) + .setAlignmentContext(routerStateIdContext) .setSecretManager(this.securityManager.getSecretManager()) .build(); @@ -384,7 +386,7 @@ public RouterRpcServer(Configuration configuration, Router router, // Create the client this.rpcClient = new RouterRpcClient(this.conf, this.router, - this.namenodeResolver, this.rpcMonitor); + this.namenodeResolver, this.rpcMonitor, routerStateIdContext); // Initialize modules this.quotaCall = new Quota(this.router, this); @@ -405,7 +407,7 @@ public RouterRpcServer(Configuration configuration, Router router, .asMap() .keySet() .parallelStream() - .forEach((key) -> this.dnCache.refresh(key)), + .forEach(this.dnCache::refresh), 0, dnCacheExpire, TimeUnit.MILLISECONDS); initRouterFedRename(); @@ -581,7 +583,7 @@ public InetSocketAddress getRpcAddress() { * @param op Category of the operation to check. * @param supported If the operation is supported or not. If not, it will * throw an UnsupportedOperationException. - * @throws SafeModeException If the Router is in safe mode and cannot serve + * @throws StandbyException If the Router is in safe mode and cannot serve * client requests. * @throws UnsupportedOperationException If the operation is not supported. */ @@ -604,7 +606,7 @@ void checkOperation(OperationCategory op, boolean supported) * UNCHECKED. This function should be called by all ClientProtocol functions. * * @param op Category of the operation to check. - * @throws SafeModeException If the Router is in safe mode and cannot serve + * @throws StandbyException If the Router is in safe mode and cannot serve * client requests. */ void checkOperation(OperationCategory op) @@ -980,8 +982,9 @@ public boolean mkdirs(String src, FsPermission masked, boolean createParent) } @Override // ClientProtocol - public void renewLease(String clientName) throws IOException { - clientProto.renewLease(clientName); + public void renewLease(String clientName, List namespaces) + throws IOException { + clientProto.renewLease(clientName, namespaces); } @Override // ClientProtocol @@ -1328,7 +1331,7 @@ public void modifyAclEntries(String src, List aclSpec) clientProto.modifyAclEntries(src, aclSpec); } - @Override // ClienProtocol + @Override // ClientProtocol public void removeAclEntries(String src, List aclSpec) throws IOException { clientProto.removeAclEntries(src, aclSpec); @@ -1730,8 +1733,7 @@ protected List getLocationsForPath(String path, final PathLocation location = this.subclusterResolver.getDestinationForPath(path); if (location == null) { - throw new IOException("Cannot find locations for " + path + " in " + - this.subclusterResolver.getClass().getSimpleName()); + throw new NoLocationException(path, this.subclusterResolver.getClass()); } // We may block some write operations @@ -1764,6 +1766,9 @@ protected List getLocationsForPath(String path, locs.add(loc); } } + if (locs.isEmpty()) { + throw new NoLocationException(path, this.subclusterResolver.getClass()); + } return locs; } catch (IOException ioe) { if (this.rpcMonitor != null) { diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterStateIdContext.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterStateIdContext.java new file mode 100644 index 0000000000000..9d2b75b0b552b --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterStateIdContext.java @@ -0,0 +1,168 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hdfs.server.federation.router; + +import java.lang.reflect.Method; +import java.util.Collections; +import java.util.HashSet; + +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.atomic.LongAccumulator; +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos; +import org.apache.hadoop.hdfs.protocol.ClientProtocol; +import org.apache.hadoop.hdfs.server.namenode.ha.ReadOnly; +import org.apache.hadoop.ipc.AlignmentContext; +import org.apache.hadoop.ipc.RetriableException; +import org.apache.hadoop.ipc.Server; +import org.apache.hadoop.ipc.protobuf.RpcHeaderProtos.RpcRequestHeaderProto; +import org.apache.hadoop.ipc.protobuf.RpcHeaderProtos.RpcResponseHeaderProto; +import org.apache.hadoop.thirdparty.protobuf.ByteString; +import org.apache.hadoop.thirdparty.protobuf.InvalidProtocolBufferException; + + +/** + * This is the router implementation to hold the state Ids for all + * namespaces. This object is only updated by responses from NameNodes. + */ +@InterfaceAudience.Private +@InterfaceStability.Evolving +class RouterStateIdContext implements AlignmentContext { + + private final HashSet coordinatedMethods; + /** + * Collection of last-seen namespace state Ids for a set of namespaces. + * Each value is globally shared by all outgoing connections to a particular namespace, + * so updates should only be performed using reliable responses from NameNodes. + */ + private final ConcurrentHashMap namespaceIdMap; + // Size limit for the map of state Ids to send to clients. + private final int maxSizeOfFederatedStateToPropagate; + + RouterStateIdContext(Configuration conf) { + this.coordinatedMethods = new HashSet<>(); + // For now, only ClientProtocol methods can be coordinated, so only checking + // against ClientProtocol. + for (Method method : ClientProtocol.class.getDeclaredMethods()) { + if (method.isAnnotationPresent(ReadOnly.class) + && method.getAnnotationsByType(ReadOnly.class)[0].isCoordinated()) { + coordinatedMethods.add(method.getName()); + } + } + + namespaceIdMap = new ConcurrentHashMap<>(); + + maxSizeOfFederatedStateToPropagate = + conf.getInt(RBFConfigKeys.DFS_ROUTER_OBSERVER_FEDERATED_STATE_PROPAGATION_MAXSIZE, + RBFConfigKeys.DFS_ROUTER_OBSERVER_FEDERATED_STATE_PROPAGATION_MAXSIZE_DEFAULT); + } + + /** + * Adds the {@link #namespaceIdMap} to the response header that will be sent to a client. + */ + public void setResponseHeaderState(RpcResponseHeaderProto.Builder headerBuilder) { + if (namespaceIdMap.isEmpty()) { + return; + } + HdfsServerFederationProtos.RouterFederatedStateProto.Builder federatedStateBuilder = + HdfsServerFederationProtos.RouterFederatedStateProto.newBuilder(); + namespaceIdMap.forEach((k, v) -> federatedStateBuilder.putNamespaceStateIds(k, v.get())); + headerBuilder.setRouterFederatedState(federatedStateBuilder.build().toByteString()); + } + + public LongAccumulator getNamespaceStateId(String nsId) { + return namespaceIdMap.computeIfAbsent(nsId, key -> new LongAccumulator(Math::max, Long.MIN_VALUE)); + } + + public void removeNamespaceStateId(String nsId) { + namespaceIdMap.remove(nsId); + } + + /** + * Utility function to parse routerFederatedState field in RPC headers. + */ + public static Map getRouterFederatedStateMap(ByteString byteString) { + if (byteString != null) { + HdfsServerFederationProtos.RouterFederatedStateProto federatedState; + try { + federatedState = HdfsServerFederationProtos.RouterFederatedStateProto.parseFrom(byteString); + } catch (InvalidProtocolBufferException e) { + throw new RuntimeException(e); + } + return federatedState.getNamespaceStateIdsMap(); + } else { + return Collections.emptyMap(); + } + } + + public static long getClientStateIdFromCurrentCall(String nsId) { + Long clientStateID = Long.MIN_VALUE; + Server.Call call = Server.getCurCall().get(); + if (call != null) { + ByteString callFederatedNamespaceState = call.getFederatedNamespaceState(); + if (callFederatedNamespaceState != null) { + Map clientFederatedStateIds = getRouterFederatedStateMap(callFederatedNamespaceState); + clientStateID = clientFederatedStateIds.getOrDefault(nsId, Long.MIN_VALUE); + } + } + return clientStateID; + } + + @Override + public void updateResponseState(RpcResponseHeaderProto.Builder header) { + if (namespaceIdMap.size() <= maxSizeOfFederatedStateToPropagate) { + setResponseHeaderState(header); + } + } + + @Override + public void receiveResponseState(RpcResponseHeaderProto header) { + // Do nothing. + } + + @Override + public void updateRequestState(RpcRequestHeaderProto.Builder header) { + // Do nothing. + } + + /** + * Routers do not update their state using information from clients + * to avoid clients interfering with one another. + */ + @Override + public long receiveRequestState(RpcRequestHeaderProto header, + long clientWaitTime) throws RetriableException { + // Do nothing. + return 0; + } + + @Override + public long getLastSeenStateId() { + return 0; + } + + @Override + public boolean isCoordinatedCall(String protocolName, String methodName) { + return protocolName.equals(ClientProtocol.class.getCanonicalName()) + && coordinatedMethods.contains(methodName); + } +} diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/MembershipStore.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/MembershipStore.java index 4352ae19bde18..c6545d7425c92 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/MembershipStore.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/MembershipStore.java @@ -48,7 +48,7 @@ * StateStoreDriver}, NameNode registrations are cached until the next query. * The fetched registration data is aggregated using a quorum to determine the * best/most accurate state for each NameNode. The cache is periodically updated - * by the @{link StateStoreCacheUpdateService}. + * by the {@link StateStoreCacheUpdateService}. */ @InterfaceAudience.Private @InterfaceStability.Evolving diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/driver/impl/StateStoreZooKeeperImpl.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/driver/impl/StateStoreZooKeeperImpl.java index c6441caf82119..45442da0ab570 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/driver/impl/StateStoreZooKeeperImpl.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/driver/impl/StateStoreZooKeeperImpl.java @@ -50,6 +50,7 @@ * |--- MEMBERSHIP * |--- REBALANCER * |--- ROUTERS + * |--- DISABLE_NAMESERVICE */ public class StateStoreZooKeeperImpl extends StateStoreSerializableImpl { diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/package-info.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/package-info.java index 6b3e55f6d98fd..9b45f28396de6 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/package-info.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/package-info.java @@ -31,19 +31,21 @@ * StateStoreDriver} to handle querying, updating and deleting data records. The * data storage driver is initialized and maintained by the {@link * org.apache.hadoop.hdfs.server.federation.store.StateStoreService - * FederationStateStoreService}. The state store + * StateStoreService}. The state store * supports fetching all records of a type, filtering by column values or * fetching a single record by its primary key. *

    * The state store contains several API interfaces, one for each data records * type. *

      - *
    • FederationMembershipStateStore: state of all Namenodes in the federation. + *
    • MembershipStore: state of all Namenodes in the federation. * Uses the MembershipState record. - *
    • FederationMountTableStore: Mount table mapping paths in the global + *
    • MountTableStore: Mount table mapping paths in the global * namespace to individual subcluster paths. Uses the MountTable record. - *
    • RouterStateStore: State of all routers in the federation. Uses the + *
    • RouterStore: State of all routers in the federation. Uses the * RouterState record. + *
    • DisabledNameserviceStore: state of all disabled nameservice in the federation. + * Uses the DisabledNameservice record. *
    * Each API is defined in a separate interface. The implementations of these * interfaces are responsible for accessing the {@link diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/records/MembershipState.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/records/MembershipState.java index 4add8fa99c6eb..80889b3d4aa4a 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/records/MembershipState.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/records/MembershipState.java @@ -32,7 +32,7 @@ /** * Data schema for storing NN registration information in the * {@link org.apache.hadoop.hdfs.server.federation.store.StateStoreService - * FederationStateStoreService}. + * StateStoreService}. */ public abstract class MembershipState extends BaseRecord implements FederationNamenodeContext { diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/records/MountTable.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/records/MountTable.java index bedf37b64c8e8..a80866780129d 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/records/MountTable.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/records/MountTable.java @@ -42,9 +42,9 @@ /** * Data schema for {@link * org.apache.hadoop.hdfs.server.federation.store.MountTableStore - * FederationMountTableStore} data stored in the {@link + * MountTableStore} data stored in the {@link * org.apache.hadoop.hdfs.server.federation.store.StateStoreService - * FederationStateStoreService}. Supports string serialization. + * StateStoreService}. Supports string serialization. */ public abstract class MountTable extends BaseRecord { diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/records/RouterState.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/records/RouterState.java index 761e2a4872e9a..337a58c359812 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/records/RouterState.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/records/RouterState.java @@ -31,7 +31,7 @@ * Entry to log the state of a * {@link org.apache.hadoop.hdfs.server.federation.router.Router Router} in the * {@link org.apache.hadoop.hdfs.server.federation.store.StateStoreService - * FederationStateStoreService}. + * StateStoreService}. */ public abstract class RouterState extends BaseRecord { diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/proto/FederationProtocol.proto b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/proto/FederationProtocol.proto index 336130e419a3a..7f61d80fe1aa8 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/proto/FederationProtocol.proto +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/proto/FederationProtocol.proto @@ -312,3 +312,16 @@ message GetDisabledNameservicesRequestProto { message GetDisabledNameservicesResponseProto { repeated string nameServiceIds = 1; } + +///////////////////////////////////////////////// +// Alignment state for namespaces. +///////////////////////////////////////////////// + +/** + * Clients should receive this message in RPC responses and forward it + * in RPC requests without interpreting it. It should be encoded + * as an obscure byte array when being sent to clients. + */ +message RouterFederatedStateProto { + map namespaceStateIds = 1; // Last seen state IDs for multiple namespaces. +} \ No newline at end of file diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/resources/hdfs-rbf-default.xml b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/resources/hdfs-rbf-default.xml index fcf6a28475fbd..52a1e3a3bd1e1 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/resources/hdfs-rbf-default.xml +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/resources/hdfs-rbf-default.xml @@ -134,6 +134,33 @@ + + dfs.federation.router.enable.multiple.socket + false + + If enable multiple downstream socket or not. If true, ConnectionPool + will use a new socket when creating a new connection for the same user, + and RouterRPCClient will get a better throughput. It's best used with + dfs.federation.router.max.concurrency.per.connection together to get + a better throughput with fewer sockets. Such as enable + dfs.federation.router.enable.multiple.socket and + set dfs.federation.router.max.concurrency.per.connection = 20. + + + + + dfs.federation.router.max.concurrency.per.connection + 1 + + The maximum number of requests that a connection can handle concurrently. + When the number of requests being processed by a socket is less than this value, + new request will be processed by this socket. When enable + dfs.federation.router.enable.multiple.socket, it's best + set this value greater than 1, such as 20, to avoid frequent + creation and idle sockets in the case of a NS with jitter requests. + + + dfs.federation.router.connection.pool.clean.ms 60000 @@ -168,6 +195,16 @@ + + dfs.federation.router.enable.get.dn.usage + true + + If true, the getNodeUsage method in RBFMetrics will return an up-to-date + result collecting from downstream nameservices. But it will take a long + time and take up thread resources. If false, it will return a mock result with all 0. + + + dfs.federation.router.metrics.class org.apache.hadoop.hdfs.server.federation.metrics.FederationRPCPerformanceMonitor @@ -385,6 +422,14 @@ + + dfs.federation.router.health.monitor.timeout + 30s + + Time out for Router to obtain HAServiceStatus from NameNode. + + + dfs.federation.router.heartbeat-state.interval 5s @@ -723,6 +768,14 @@ + + dfs.federation.router.fairness.acquire.timeout + 1s + + The maximum time to wait for a permit. + + + dfs.federation.router.federation.rename.bandwidth 10 @@ -781,4 +834,34 @@ (delete the source path directly) and skip (skip both trash and deletion). + + + dfs.federation.router.observer.read.default + false + + Whether observer reads are enabled. This is a default for all nameservices. + The default can be inverted for individual namespace by adding them to + dfs.federation.router.observer.read.overrides. + + + + + dfs.federation.router.observer.read.overrides + + + Commas separated list of namespaces for which to invert the default configuration, + dfs.federation.router.observer.read.default, for whether to enable observer reads. + + + + + dfs.federation.router.observer.federated.state.propagation.maxsize + 5 + + The maximum size of the federated state to send in the RPC header. Sending the federated + state removes the need to msync on every read call, but at the expense of having a larger + header. The cost tradeoff between the larger header and always msync'ing depends on the number + of namespaces in use and the latency of the msync requests. + + diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/webapps/router/federationhealth.html b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/webapps/router/federationhealth.html index f45da5605f734..9fa43a308584b 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/webapps/router/federationhealth.html +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/webapps/router/federationhealth.html @@ -509,5 +509,6 @@ + diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/site/markdown/HDFSRouterFederation.md b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/site/markdown/HDFSRouterFederation.md index ff2cea5761273..5a9c2fd42855c 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/site/markdown/HDFSRouterFederation.md +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/site/markdown/HDFSRouterFederation.md @@ -46,7 +46,6 @@ This approach has the same architecture as [YARN federation](../../hadoop-yarn/h ### Example flow The simplest configuration deploys a Router on each NameNode machine. The Router monitors the local NameNode and its state and heartbeats to the State Store. -The Router monitors the local NameNode and heartbeats the state to the State Store. When a regular DFS client contacts any of the Routers to access a file in the federated filesystem, the Router checks the Mount Table in the State Store (i.e., the local cache) to find out which subcluster contains the file. Then it checks the Membership table in the State Store (i.e., the local cache) for the NameNode responsible for the subcluster. After it has identified the correct NameNode, the Router proxies the request. @@ -416,11 +415,13 @@ The RPC server to receive connections from the clients. The Router forwards the client requests to the NameNodes. It uses a pool of connections to reduce the latency of creating them. -| Property | Default | Description| +| Property | Default | Description | |:---- |:---- |:---- | | dfs.federation.router.connection.pool-size | 1 | Size of the pool of connections from the router to namenodes. | | dfs.federation.router.connection.clean.ms | 10000 | Time interval, in milliseconds, to check if the connection pool should remove unused connections. | | dfs.federation.router.connection.pool.clean.ms | 60000 | Time interval, in milliseconds, to check if the connection manager should remove unused connection pools. | +| dfs.federation.router.enable.multiple.socket | false | If true, ConnectionPool will use a new socket when creating a new connection for the same user. And it's best used with dfs.federation.router.max.concurrency.per.connection together. | +| dfs.federation.router.max.concurrency.per.connection | 1 | The maximum number of requests that a connection can handle concurrently. | ### Admin server diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/FederationTestUtils.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/FederationTestUtils.java index e758eee4fda7c..b0a897d9f4bb2 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/FederationTestUtils.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/FederationTestUtils.java @@ -92,8 +92,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.util.function.Supplier; - /** * Helper utilities for testing HDFS Federation. */ @@ -174,26 +172,23 @@ public static void waitNamenodeRegistered( final String nsId, final String nnId, final FederationNamenodeServiceState state) throws Exception { - GenericTestUtils.waitFor(new Supplier() { - @Override - public Boolean get() { - try { - List namenodes = - resolver.getNamenodesForNameserviceId(nsId); - if (namenodes != null) { - for (FederationNamenodeContext namenode : namenodes) { - // Check if this is the Namenode we are checking - if (namenode.getNamenodeId() == nnId || - namenode.getNamenodeId().equals(nnId)) { - return state == null || namenode.getState().equals(state); - } + GenericTestUtils.waitFor(() -> { + try { + List namenodes = + resolver.getNamenodesForNameserviceId(nsId, false); + if (namenodes != null) { + for (FederationNamenodeContext namenode : namenodes) { + // Check if this is the Namenode we are checking + if (namenode.getNamenodeId() == nnId || + namenode.getNamenodeId().equals(nnId)) { + return state == null || namenode.getState().equals(state); } } - } catch (IOException e) { - // Ignore } - return false; + } catch (IOException e) { + // Ignore } + return false; }, 1000, 60 * 1000); } @@ -209,22 +204,19 @@ public static void waitNamenodeRegistered( final ActiveNamenodeResolver resolver, final String nsId, final FederationNamenodeServiceState state) throws Exception { - GenericTestUtils.waitFor(new Supplier() { - @Override - public Boolean get() { - try { - List nns = - resolver.getNamenodesForNameserviceId(nsId); - for (FederationNamenodeContext nn : nns) { - if (nn.getState().equals(state)) { - return true; - } + GenericTestUtils.waitFor(() -> { + try { + List nns = + resolver.getNamenodesForNameserviceId(nsId, false); + for (FederationNamenodeContext nn : nns) { + if (nn.getState().equals(state)) { + return true; } - } catch (IOException e) { - // Ignore } - return false; + } catch (IOException e) { + // Ignore } + return false; }, 1000, 20 * 1000); } @@ -361,19 +353,16 @@ public Object answer(InvocationOnMock invocation) throws Throwable { */ public static void waitRouterRegistered(RouterStore stateManager, long routerCount, int timeout) throws Exception { - GenericTestUtils.waitFor(new Supplier() { - @Override - public Boolean get() { - try { - List cachedRecords = stateManager.getCachedRecords(); - if (cachedRecords.size() == routerCount) { - return true; - } - } catch (IOException e) { - // Ignore + GenericTestUtils.waitFor(() -> { + try { + List cachedRecords = stateManager.getCachedRecords(); + if (cachedRecords.size() == routerCount) { + return true; } - return false; + } catch (IOException e) { + // Ignore } + return false; }, 100, timeout); } @@ -390,15 +379,13 @@ public static void simulateThrowExceptionRouterRpcServer( ConnectionManager connectionManager = new ConnectionManager(server.getConfig()); ConnectionManager spyConnectionManager = spy(connectionManager); - doAnswer(new Answer() { - @Override - public Object answer(InvocationOnMock invocation) throws Throwable { - LOG.info("Simulating connectionManager throw IOException {}", - invocation.getMock()); - throw new IOException("Simulate connectionManager throw IOException"); - } + doAnswer(invocation -> { + LOG.info("Simulating connectionManager throw IOException {}", + invocation.getMock()); + throw new IOException("Simulate connectionManager throw IOException"); }).when(spyConnectionManager).getConnection( - any(UserGroupInformation.class), any(String.class), any(Class.class)); + any(UserGroupInformation.class), any(String.class), any(Class.class), + any(String.class)); Whitebox.setInternalState(rpcClient, "connectionManager", spyConnectionManager); diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/MiniRouterDFSCluster.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/MiniRouterDFSCluster.java index 87b99e5d9523c..4fcdf6595e4ad 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/MiniRouterDFSCluster.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/MiniRouterDFSCluster.java @@ -237,12 +237,8 @@ public DFSClient getClient(UserGroupInformation user) throws IOException, URISyntaxException, InterruptedException { LOG.info("Connecting to router at {}", fileSystemUri); - return user.doAs(new PrivilegedExceptionAction() { - @Override - public DFSClient run() throws IOException { - return new DFSClient(fileSystemUri, conf); - } - }); + return user.doAs((PrivilegedExceptionAction) + () -> new DFSClient(fileSystemUri, conf)); } public RouterClient getAdminClient() throws IOException { @@ -384,12 +380,8 @@ public DFSClient getClient(UserGroupInformation user) throws IOException, URISyntaxException, InterruptedException { LOG.info("Connecting to namenode at {}", fileSystemUri); - return user.doAs(new PrivilegedExceptionAction() { - @Override - public DFSClient run() throws IOException { - return new DFSClient(fileSystemUri, conf); - } - }); + return user.doAs((PrivilegedExceptionAction) + () -> new DFSClient(fileSystemUri, conf)); } public DFSClient getClient() throws IOException, URISyntaxException { @@ -814,6 +806,7 @@ public void startCluster(Configuration overrideConf) { .numDataNodes(numDNs) .nnTopology(topology) .dataNodeConfOverlays(dnConfs) + .checkExitOnShutdown(false) .storageTypes(storageTypes) .racks(racks) .build(); @@ -1046,6 +1039,27 @@ public void switchToStandby(String nsId, String nnId) { } } + /** + * Switch a namenode in a nameservice to be the observer. + * @param nsId Nameservice identifier. + * @param nnId Namenode identifier. + */ + public void switchToObserver(String nsId, String nnId) { + try { + int total = cluster.getNumNameNodes(); + NameNodeInfo[] nns = cluster.getNameNodeInfos(); + for (int i = 0; i < total; i++) { + NameNodeInfo nn = nns[i]; + if (nn.getNameserviceId().equals(nsId) && + nn.getNamenodeId().equals(nnId)) { + cluster.transitionToObserver(i); + } + } + } catch (Throwable e) { + LOG.error("Cannot transition to active", e); + } + } + /** * Stop the federated HDFS cluster. */ diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/MockNamenode.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/MockNamenode.java index a4755c20fcae4..27fcf8726b6c8 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/MockNamenode.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/MockNamenode.java @@ -156,15 +156,12 @@ protected void setupMock() throws IOException { NamespaceInfo nsInfo = new NamespaceInfo(1, this.nsId, this.nsId, 1); when(mockNn.versionRequest()).thenReturn(nsInfo); - when(mockNn.getServiceStatus()).thenAnswer(new Answer() { - @Override - public HAServiceStatus answer(InvocationOnMock invocation) - throws Throwable { - HAServiceStatus haStatus = new HAServiceStatus(getHAServiceState()); - haStatus.setNotReadyToBecomeActive(""); - return haStatus; - } - }); + when(mockNn.getServiceStatus()). + thenAnswer((Answer) invocation -> { + HAServiceStatus haStatus = new HAServiceStatus(getHAServiceState()); + haStatus.setNotReadyToBecomeActive(""); + return haStatus; + }); } /** diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/MockResolver.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/MockResolver.java index 1519bad74b5c1..4aaa8e7569e88 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/MockResolver.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/MockResolver.java @@ -23,6 +23,7 @@ import java.util.Collections; import java.util.HashMap; import java.util.HashSet; +import java.util.Iterator; import java.util.LinkedList; import java.util.List; import java.util.Map; @@ -59,6 +60,7 @@ public class MockResolver private String defaultNamespace = null; private boolean disableDefaultNamespace = false; private volatile boolean disableRegistration = false; + private TreeSet disableNamespaces = new TreeSet<>(); public MockResolver() { this.cleanRegistrations(); @@ -118,12 +120,24 @@ public void setDisableRegistration(boolean isDisable) { disableRegistration = isDisable; } + @Override public void updateUnavailableNamenode(String ns, + InetSocketAddress failedAddress) throws IOException { + updateNameNodeState(ns, failedAddress, + FederationNamenodeServiceState.UNAVAILABLE); + } + @Override public void updateActiveNamenode( String nsId, InetSocketAddress successfulAddress) { + updateNameNodeState(nsId, successfulAddress, + FederationNamenodeServiceState.ACTIVE); + } - String address = successfulAddress.getHostName() + ":" + - successfulAddress.getPort(); + private void updateNameNodeState(String nsId, + InetSocketAddress iAddr, + FederationNamenodeServiceState state) { + String sAddress = iAddr.getHostName() + ":" + + iAddr.getPort(); String key = nsId; if (key != null) { // Update the active entry @@ -131,9 +145,9 @@ public void updateActiveNamenode( List namenodes = (List) this.resolver.get(key); for (FederationNamenodeContext namenode : namenodes) { - if (namenode.getRpcAddress().equals(address)) { + if (namenode.getRpcAddress().equals(sAddress)) { MockNamenodeContext nn = (MockNamenodeContext) namenode; - nn.setState(FederationNamenodeServiceState.ACTIVE); + nn.setState(state); break; } } @@ -146,14 +160,39 @@ public void updateActiveNamenode( @Override public synchronized List - getNamenodesForNameserviceId(String nameserviceId) { + getNamenodesForNameserviceId(String nameserviceId, boolean observerRead) { // Return a copy of the list because it is updated periodically List namenodes = this.resolver.get(nameserviceId); if (namenodes == null) { namenodes = new ArrayList<>(); } - return Collections.unmodifiableList(new ArrayList<>(namenodes)); + + List ret = new ArrayList<>(); + + if (observerRead) { + Iterator iterator = namenodes + .iterator(); + List observerNN = new ArrayList<>(); + List nonObserverNN = new ArrayList<>(); + while (iterator.hasNext()) { + FederationNamenodeContext membership = iterator.next(); + if (membership.getState() == FederationNamenodeServiceState.OBSERVER) { + observerNN.add(membership); + } else { + nonObserverNN.add(membership); + } + } + Collections.shuffle(observerNN); + Collections.sort(nonObserverNN, new NamenodePriorityComparator()); + ret.addAll(observerNN); + ret.addAll(nonObserverNN); + } else { + ret.addAll(namenodes); + Collections.sort(ret, new NamenodePriorityComparator()); + } + + return Collections.unmodifiableList(ret); } @Override @@ -300,9 +339,17 @@ public synchronized Set getNamespaces() return Collections.unmodifiableSet(this.namespaces); } + public void clearDisableNamespaces() { + this.disableNamespaces.clear(); + } + + public void disableNamespace(String nsId) { + this.disableNamespaces.add(nsId); + } + @Override public Set getDisabledNamespaces() throws IOException { - return new TreeSet<>(); + return this.disableNamespaces; } @Override diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/fairness/TestRouterHandlersFairness.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/fairness/TestRouterHandlersFairness.java index 2d27c66e37ee0..8fc9de0cb261f 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/fairness/TestRouterHandlersFairness.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/fairness/TestRouterHandlersFairness.java @@ -194,7 +194,7 @@ private void invokeSequential(ClientProtocol routerProto) throws IOException { private void invokeConcurrent(ClientProtocol routerProto, String clientName) throws IOException { - routerProto.renewLease(clientName); + routerProto.renewLease(clientName, null); } private int getTotalRejectedPermits(RouterContext routerContext) { diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/fairness/TestRouterRefreshFairnessPolicyController.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/fairness/TestRouterRefreshFairnessPolicyController.java index dfda47b9a53f4..0741f1aed441a 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/fairness/TestRouterRefreshFairnessPolicyController.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/fairness/TestRouterRefreshFairnessPolicyController.java @@ -133,9 +133,8 @@ public void testConcurrentRefreshRequests() throws InterruptedException { // Spawn 100 concurrent refresh requests Thread[] threads = new Thread[100]; for (int i = 0; i < 100; i++) { - threads[i] = new Thread(() -> { - client.refreshFairnessPolicyController(routerContext.getConf()); - }); + threads[i] = new Thread(() -> + client.refreshFairnessPolicyController(routerContext.getConf())); } for (Thread thread : threads) { @@ -162,7 +161,8 @@ public void testRefreshStaticChangeHandlers() throws Exception { Thread.sleep(sleepTime); return null; }).when(client) - .invokeMethod(Mockito.any(), Mockito.any(), Mockito.any(), Mockito.any(), Mockito.any()); + .invokeMethod(Mockito.any(), Mockito.any(), Mockito.anyBoolean(), + Mockito.any(), Mockito.any(), Mockito.any()); // No calls yet assertEquals("{}", @@ -182,9 +182,8 @@ public void testRefreshStaticChangeHandlers() throws Exception { final int newNs1Permits = 4; conf.setInt(DFS_ROUTER_FAIR_HANDLER_COUNT_KEY_PREFIX + "ns0", newNs0Permits); conf.setInt(DFS_ROUTER_FAIR_HANDLER_COUNT_KEY_PREFIX + "ns1", newNs1Permits); - Thread threadRefreshController = new Thread(() -> { - client.refreshFairnessPolicyController(routerContext.getConf()); - }); + Thread threadRefreshController = new Thread(() -> client. + refreshFairnessPolicyController(routerContext.getConf())); threadRefreshController.start(); threadRefreshController.join(); diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/fairness/TestRouterRpcFairnessPolicyController.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/fairness/TestRouterRpcFairnessPolicyController.java index 8307f666b5d1c..1f5770b1ddaa3 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/fairness/TestRouterRpcFairnessPolicyController.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/fairness/TestRouterRpcFairnessPolicyController.java @@ -23,10 +23,14 @@ import org.apache.hadoop.hdfs.server.federation.router.FederationUtil; import org.apache.hadoop.hdfs.server.federation.router.RBFConfigKeys; import org.apache.hadoop.test.GenericTestUtils; +import org.apache.hadoop.util.Time; import org.junit.Test; import org.slf4j.LoggerFactory; +import java.util.concurrent.TimeUnit; + import static org.apache.hadoop.hdfs.server.federation.fairness.RouterRpcFairnessConstants.CONCURRENT_NS; +import static org.apache.hadoop.hdfs.server.federation.router.RBFConfigKeys.DFS_ROUTER_FAIRNESS_ACQUIRE_TIMEOUT; import static org.apache.hadoop.hdfs.server.federation.router.RBFConfigKeys.DFS_ROUTER_HANDLER_COUNT_KEY; import static org.apache.hadoop.hdfs.server.federation.router.RBFConfigKeys.DFS_ROUTER_MONITOR_NAMENODE; import static org.apache.hadoop.hdfs.server.federation.router.RBFConfigKeys.DFS_ROUTER_FAIR_HANDLER_COUNT_KEY_PREFIX; @@ -83,6 +87,26 @@ public void testHandlerAllocationPreconfigured() { assertFalse(routerRpcFairnessPolicyController.acquirePermit(CONCURRENT_NS)); } + @Test + public void testAcquireTimeout() { + Configuration conf = createConf(40); + conf.setInt(DFS_ROUTER_FAIR_HANDLER_COUNT_KEY_PREFIX + "ns1", 30); + conf.setTimeDuration(DFS_ROUTER_FAIRNESS_ACQUIRE_TIMEOUT, 100, TimeUnit.MILLISECONDS); + RouterRpcFairnessPolicyController routerRpcFairnessPolicyController = + FederationUtil.newFairnessPolicyController(conf); + + // ns1 should have 30 permits allocated + for (int i = 0; i < 30; i++) { + assertTrue(routerRpcFairnessPolicyController.acquirePermit("ns1")); + } + long acquireBeginTimeMs = Time.monotonicNow(); + assertFalse(routerRpcFairnessPolicyController.acquirePermit("ns1")); + long acquireTimeMs = Time.monotonicNow() - acquireBeginTimeMs; + + // There are some other operations, so acquireTimeMs >= 100ms. + assertTrue(acquireTimeMs >= 100); + } + @Test public void testAllocationErrorWithZeroHandlers() { Configuration conf = createConf(0); diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/metrics/TestRouterClientMetrics.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/metrics/TestRouterClientMetrics.java index da16c05910785..3397718745ffb 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/metrics/TestRouterClientMetrics.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/metrics/TestRouterClientMetrics.java @@ -156,7 +156,7 @@ public void testGetQuota() throws Exception { @Test public void testRenewLease() throws Exception { - router.getRpcServer().renewLease("test"); + router.getRpcServer().renewLease("test", null); assertCounter("RenewLeaseOps", 2L, getMetrics(ROUTER_METRICS)); assertCounter("ConcurrentRenewLeaseOps", 1L, getMetrics(ROUTER_METRICS)); } diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/resolver/TestNamenodeResolver.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/resolver/TestNamenodeResolver.java index ed10a3a87317d..b602a27c95f60 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/resolver/TestNamenodeResolver.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/resolver/TestNamenodeResolver.java @@ -129,7 +129,7 @@ private void verifyFirstRegistration(String nsId, String nnId, int resultsCount, FederationNamenodeServiceState state) throws IOException { List namenodes = - namenodeResolver.getNamenodesForNameserviceId(nsId); + namenodeResolver.getNamenodesForNameserviceId(nsId, false); if (resultsCount == 0) { assertNull(namenodes); } else { @@ -291,8 +291,8 @@ public void testCacheUpdateOnNamenodeStateUpdate() throws IOException { HAServiceState.STANDBY))); stateStore.refreshCaches(true); // Check whether the namenpde state is reported correct as standby. - FederationNamenodeContext namenode = - namenodeResolver.getNamenodesForNameserviceId(NAMESERVICES[0]).get(0); + FederationNamenodeContext namenode = namenodeResolver + .getNamenodesForNameserviceId(NAMESERVICES[0], false).get(0); assertEquals(FederationNamenodeServiceState.STANDBY, namenode.getState()); String rpcAddr = namenode.getRpcAddress(); InetSocketAddress inetAddr = getInetSocketAddress(rpcAddr); @@ -301,8 +301,8 @@ public void testCacheUpdateOnNamenodeStateUpdate() throws IOException { // RouterRpcClient calls updateActiveNamenode to update the state to active, // Check whether correct updated state is returned post update. namenodeResolver.updateActiveNamenode(NAMESERVICES[0], inetAddr); - FederationNamenodeContext namenode1 = - namenodeResolver.getNamenodesForNameserviceId(NAMESERVICES[0]).get(0); + FederationNamenodeContext namenode1 = namenodeResolver + .getNamenodesForNameserviceId(NAMESERVICES[0], false).get(0); assertEquals("The namenode state should be ACTIVE post update.", FederationNamenodeServiceState.ACTIVE, namenode1.getState()); } @@ -318,8 +318,8 @@ public void testCacheUpdateOnNamenodeStateUpdateWithIp() InetSocketAddress inetAddr = getInetSocketAddress(rpcAddress); namenodeResolver.updateActiveNamenode(NAMESERVICES[0], inetAddr); - FederationNamenodeContext namenode = - namenodeResolver.getNamenodesForNameserviceId(NAMESERVICES[0]).get(0); + FederationNamenodeContext namenode = namenodeResolver + .getNamenodesForNameserviceId(NAMESERVICES[0], false).get(0); assertEquals("The namenode state should be ACTIVE post update.", FederationNamenodeServiceState.ACTIVE, namenode.getState()); } diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/resolver/order/TestLocalResolver.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/resolver/order/TestLocalResolver.java index 08e75b2d30918..0625bbadf7d42 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/resolver/order/TestLocalResolver.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/resolver/order/TestLocalResolver.java @@ -43,7 +43,6 @@ import org.apache.hadoop.hdfs.server.federation.store.records.MembershipState; import org.apache.hadoop.hdfs.server.federation.store.records.MountTable; import org.junit.Test; -import org.mockito.invocation.InvocationOnMock; import org.mockito.stubbing.Answer; /** @@ -78,12 +77,8 @@ public void testLocalResolver() throws IOException { StringBuilder sb = new StringBuilder("clientX"); LocalResolver localResolver = new LocalResolver(conf, router); LocalResolver spyLocalResolver = spy(localResolver); - doAnswer(new Answer() { - @Override - public String answer(InvocationOnMock invocation) throws Throwable { - return sb.toString(); - } - }).when(spyLocalResolver).getClientAddr(); + doAnswer((Answer) invocation -> sb.toString() + ).when(spyLocalResolver).getClientAddr(); // Add the mocks to the resolver MultipleDestinationMountTableResolver resolver = diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestConnectionManager.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestConnectionManager.java index e397692e9a86d..067d43dabd5fa 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestConnectionManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestConnectionManager.java @@ -36,6 +36,7 @@ import java.util.concurrent.BlockingQueue; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; import static org.junit.Assert.assertNotNull; @@ -80,15 +81,15 @@ public void shutdown() { public void testCleanup() throws Exception { Map poolMap = connManager.getPools(); - ConnectionPool pool1 = new ConnectionPool( - conf, TEST_NN_ADDRESS, TEST_USER1, 0, 10, 0.5f, ClientProtocol.class); + ConnectionPool pool1 = new ConnectionPool(conf, TEST_NN_ADDRESS, TEST_USER1, + 0, 10, 0.5f, ClientProtocol.class, null); addConnectionsToPool(pool1, 9, 4); poolMap.put( new ConnectionPoolId(TEST_USER1, TEST_NN_ADDRESS, ClientProtocol.class), pool1); - ConnectionPool pool2 = new ConnectionPool( - conf, TEST_NN_ADDRESS, TEST_USER2, 0, 10, 0.5f, ClientProtocol.class); + ConnectionPool pool2 = new ConnectionPool(conf, TEST_NN_ADDRESS, TEST_USER2, + 0, 10, 0.5f, ClientProtocol.class, null); addConnectionsToPool(pool2, 10, 10); poolMap.put( new ConnectionPoolId(TEST_USER2, TEST_NN_ADDRESS, ClientProtocol.class), @@ -110,8 +111,8 @@ public void testCleanup() throws Exception { checkPoolConnections(TEST_USER2, 10, 10); // Make sure the number of connections doesn't go below minSize - ConnectionPool pool3 = new ConnectionPool( - conf, TEST_NN_ADDRESS, TEST_USER3, 2, 10, 0.5f, ClientProtocol.class); + ConnectionPool pool3 = new ConnectionPool(conf, TEST_NN_ADDRESS, TEST_USER3, + 2, 10, 0.5f, ClientProtocol.class, null); addConnectionsToPool(pool3, 8, 0); poolMap.put( new ConnectionPoolId(TEST_USER3, TEST_NN_ADDRESS, ClientProtocol.class), @@ -131,12 +132,50 @@ public void testCleanup() throws Exception { checkPoolConnections(TEST_USER3, 4, 2); } + @Test + public void testGetConnectionWithConcurrency() throws Exception { + Map poolMap = connManager.getPools(); + Configuration copyConf = new Configuration(conf); + copyConf.setInt(RBFConfigKeys.DFS_ROUTER_MAX_CONCURRENCY_PER_CONNECTION_KEY, 20); + + ConnectionPool pool = new ConnectionPool( + copyConf, TEST_NN_ADDRESS, TEST_USER1, 1, 10, 0.5f, + ClientProtocol.class, null); + poolMap.put( + new ConnectionPoolId(TEST_USER1, TEST_NN_ADDRESS, ClientProtocol.class), + pool); + assertEquals(1, pool.getNumConnections()); + // one connection can process the maximum number of requests concurrently. + for (int i = 0; i < 20; i++) { + ConnectionContext cc = pool.getConnection(); + assertTrue(cc.isUsable()); + cc.getClient(); + } + assertEquals(1, pool.getNumConnections()); + + // Ask for more and this returns an unusable connection + ConnectionContext cc1 = pool.getConnection(); + assertTrue(cc1.isActive()); + assertFalse(cc1.isUsable()); + + // add a new connection into pool + pool.addConnection(pool.newConnection()); + // will return the new connection + ConnectionContext cc2 = pool.getConnection(); + assertTrue(cc2.isUsable()); + cc2.getClient(); + + assertEquals(2, pool.getNumConnections()); + + checkPoolConnections(TEST_USER1, 2, 2); + } + @Test public void testConnectionCreatorWithException() throws Exception { // Create a bad connection pool pointing to unresolvable namenode address. ConnectionPool badPool = new ConnectionPool( - conf, UNRESOLVED_TEST_NN_ADDRESS, TEST_USER1, 0, 10, 0.5f, - ClientProtocol.class); + conf, UNRESOLVED_TEST_NN_ADDRESS, TEST_USER1, 0, 10, 0.5f, + ClientProtocol.class, null); BlockingQueue queue = new ArrayBlockingQueue<>(1); queue.add(badPool); ConnectionManager.ConnectionCreator connectionCreator = @@ -144,7 +183,7 @@ public void testConnectionCreatorWithException() throws Exception { connectionCreator.setDaemon(true); connectionCreator.start(); // Wait to make sure async thread is scheduled and picks - GenericTestUtils.waitFor(()->queue.isEmpty(), 50, 5000); + GenericTestUtils.waitFor(queue::isEmpty, 50, 5000); // At this point connection creation task should be definitely picked up. assertTrue(queue.isEmpty()); // At this point connection thread should still be alive. @@ -162,7 +201,7 @@ public void testGetConnectionWithException() throws Exception { // Create a bad connection pool pointing to unresolvable namenode address. ConnectionPool badPool = new ConnectionPool( conf, UNRESOLVED_TEST_NN_ADDRESS, TEST_USER1, 1, 10, 0.5f, - ClientProtocol.class); + ClientProtocol.class, null); } @Test @@ -171,8 +210,8 @@ public void testGetConnection() throws Exception { final int totalConns = 10; int activeConns = 5; - ConnectionPool pool = new ConnectionPool( - conf, TEST_NN_ADDRESS, TEST_USER1, 0, 10, 0.5f, ClientProtocol.class); + ConnectionPool pool = new ConnectionPool(conf, TEST_NN_ADDRESS, TEST_USER1, + 0, 10, 0.5f, ClientProtocol.class, null); addConnectionsToPool(pool, totalConns, activeConns); poolMap.put( new ConnectionPoolId(TEST_USER1, TEST_NN_ADDRESS, ClientProtocol.class), @@ -196,8 +235,8 @@ public void testGetConnection() throws Exception { @Test public void testValidClientIndex() throws Exception { - ConnectionPool pool = new ConnectionPool( - conf, TEST_NN_ADDRESS, TEST_USER1, 2, 2, 0.5f, ClientProtocol.class); + ConnectionPool pool = new ConnectionPool(conf, TEST_NN_ADDRESS, TEST_USER1, + 2, 2, 0.5f, ClientProtocol.class, null); for(int i = -3; i <= 3; i++) { pool.getClientIndex().set(i); ConnectionContext conn = pool.getConnection(); @@ -212,8 +251,8 @@ public void getGetConnectionNamenodeProtocol() throws Exception { final int totalConns = 10; int activeConns = 5; - ConnectionPool pool = new ConnectionPool( - conf, TEST_NN_ADDRESS, TEST_USER1, 0, 10, 0.5f, NamenodeProtocol.class); + ConnectionPool pool = new ConnectionPool(conf, TEST_NN_ADDRESS, TEST_USER1, + 0, 10, 0.5f, NamenodeProtocol.class, null); addConnectionsToPool(pool, totalConns, activeConns); poolMap.put( new ConnectionPoolId( @@ -286,7 +325,7 @@ private void testConnectionCleanup(float ratio, int totalConns, // Create one new connection pool tmpConnManager.getConnection(TEST_USER1, TEST_NN_ADDRESS, - NamenodeProtocol.class); + NamenodeProtocol.class, "ns0"); Map poolMap = tmpConnManager.getPools(); ConnectionPoolId connectionPoolId = new ConnectionPoolId(TEST_USER1, @@ -317,6 +356,6 @@ public void testUnsupportedProtoExceptionMsg() throws Exception { "Unsupported protocol for connection to NameNode: " + TestConnectionManager.class.getName(), () -> ConnectionPool.newConnection(conf, TEST_NN_ADDRESS, TEST_USER1, - TestConnectionManager.class)); + TestConnectionManager.class, false, 0, null)); } } diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestDisableNameservices.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestDisableNameservices.java index ae04150d70fa9..78f41c5d92a2d 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestDisableNameservices.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestDisableNameservices.java @@ -159,7 +159,7 @@ public void cleanup() throws IOException { public void testWithoutDisabling() throws IOException { // ns0 is slow and renewLease should take a long time long t0 = monotonicNow(); - routerProtocol.renewLease("client0"); + routerProtocol.renewLease("client0", null); long t = monotonicNow() - t0; assertTrue("It took too little: " + t + "ms", t > TimeUnit.SECONDS.toMillis(1)); @@ -178,7 +178,7 @@ public void testDisabling() throws Exception { // renewLease should be fast as we are skipping ns0 long t0 = monotonicNow(); - routerProtocol.renewLease("client0"); + routerProtocol.renewLease("client0", null); long t = monotonicNow() - t0; assertTrue("It took too long: " + t + "ms", t < TimeUnit.SECONDS.toMillis(1)); diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestObserverWithRouter.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestObserverWithRouter.java new file mode 100644 index 0000000000000..fbd731c073f4b --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestObserverWithRouter.java @@ -0,0 +1,425 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hdfs.server.federation.router; + +import static org.apache.hadoop.hdfs.server.federation.FederationTestUtils.NAMENODES; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotEquals; +import static org.junit.Assert.assertTrue; +import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_HA_NAMENODE_ID_KEY; +import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMESERVICE_ID; +import static org.apache.hadoop.hdfs.server.federation.router.RBFConfigKeys.DFS_ROUTER_MONITOR_NAMENODE; + +import java.io.IOException; +import java.util.List; +import java.util.concurrent.TimeUnit; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hdfs.DFSConfigKeys; +import org.apache.hadoop.hdfs.server.federation.MiniRouterDFSCluster; +import org.apache.hadoop.hdfs.server.federation.MiniRouterDFSCluster.RouterContext; +import org.apache.hadoop.hdfs.server.federation.RouterConfigBuilder; +import org.apache.hadoop.hdfs.server.federation.StateStoreDFSCluster; +import org.apache.hadoop.hdfs.server.federation.resolver.FederationNamenodeContext; +import org.apache.hadoop.hdfs.server.federation.resolver.FederationNamenodeServiceState; +import org.apache.hadoop.hdfs.server.federation.resolver.MembershipNamenodeResolver; +import org.apache.hadoop.hdfs.server.namenode.NameNode; +import org.junit.After; +import org.junit.Test; + +public class TestObserverWithRouter { + + private MiniRouterDFSCluster cluster; + + public void startUpCluster(int numberOfObserver) throws Exception { + startUpCluster(numberOfObserver, null); + } + + public void startUpCluster(int numberOfObserver, Configuration confOverrides) throws Exception { + int numberOfNamenode = 2 + numberOfObserver; + Configuration conf = new Configuration(false); + conf.setBoolean(RBFConfigKeys.DFS_ROUTER_OBSERVER_READ_DEFAULT_KEY, true); + conf.setBoolean(DFSConfigKeys.DFS_HA_TAILEDITS_INPROGRESS_KEY, true); + conf.set(DFSConfigKeys.DFS_HA_TAILEDITS_PERIOD_KEY, "0ms"); + if (confOverrides != null) { + conf.addResource(confOverrides); + } + cluster = new MiniRouterDFSCluster(true, 2, numberOfNamenode); + cluster.addNamenodeOverrides(conf); + // Start NNs and DNs and wait until ready + cluster.startCluster(); + + // Making one Namenode active per nameservice + if (cluster.isHighAvailability()) { + for (String ns : cluster.getNameservices()) { + cluster.switchToActive(ns, NAMENODES[0]); + cluster.switchToStandby(ns, NAMENODES[1]); + for (int i = 2; i < numberOfNamenode; i++) { + cluster.switchToObserver(ns, NAMENODES[i]); + } + } + } + + Configuration routerConf = new RouterConfigBuilder() + .metrics() + .rpc() + .build(); + + cluster.addRouterOverrides(conf); + cluster.addRouterOverrides(routerConf); + + // Start routers with only an RPC service + cluster.startRouters(); + + // Register and verify all NNs with all routers + cluster.registerNamenodes(); + cluster.waitNamenodeRegistration(); + // Setup the mount table + cluster.installMockLocations(); + + cluster.waitActiveNamespaces(); + } + + @After + public void teardown() throws IOException { + if (cluster != null) { + cluster.shutdown(); + cluster = null; + } + } + + @Test + public void testObserverRead() throws Exception { + startUpCluster(1); + RouterContext routerContext = cluster.getRandomRouter(); + List namenodes = routerContext + .getRouter().getNamenodeResolver() + .getNamenodesForNameserviceId(cluster.getNameservices().get(0), true); + assertEquals("First namenode should be observer", namenodes.get(0).getState(), + FederationNamenodeServiceState.OBSERVER); + FileSystem fileSystem = routerContext.getFileSystem(); + Path path = new Path("/testFile"); + // Send Create call to active + fileSystem.create(path).close(); + + // Send read request to observer + fileSystem.open(path).close(); + + long rpcCountForActive = routerContext.getRouter().getRpcServer() + .getRPCMetrics().getActiveProxyOps(); + // Create and complete calls should be sent to active + assertEquals("Two calls should be sent to active", 2, rpcCountForActive); + + long rpcCountForObserver = routerContext.getRouter().getRpcServer() + .getRPCMetrics().getObserverProxyOps(); + // getBlockLocations should be sent to observer + assertEquals("One call should be sent to observer", 1, rpcCountForObserver); + fileSystem.close(); + } + + @Test + public void testObserverReadWithoutFederatedStatePropagation() throws Exception { + Configuration confOverrides = new Configuration(false); + confOverrides.setInt(RBFConfigKeys.DFS_ROUTER_OBSERVER_FEDERATED_STATE_PROPAGATION_MAXSIZE, 0); + startUpCluster(1, confOverrides); + RouterContext routerContext = cluster.getRandomRouter(); + List namenodes = routerContext + .getRouter().getNamenodeResolver() + .getNamenodesForNameserviceId(cluster.getNameservices().get(0), true); + assertEquals("First namenode should be observer", namenodes.get(0).getState(), + FederationNamenodeServiceState.OBSERVER); + FileSystem fileSystem = routerContext.getFileSystem(); + Path path = new Path("/testFile"); + // Send Create call to active + fileSystem.create(path).close(); + + // Send read request to observer. The router will msync to the active namenode. + fileSystem.open(path).close(); + + long rpcCountForActive = routerContext.getRouter().getRpcServer() + .getRPCMetrics().getActiveProxyOps(); + // Create, complete and getBlockLocations calls should be sent to active + assertEquals("Three calls should be sent to active", 3, rpcCountForActive); + + long rpcCountForObserver = routerContext.getRouter().getRpcServer() + .getRPCMetrics().getObserverProxyOps(); + assertEquals("No call should be sent to observer", 0, rpcCountForObserver); + fileSystem.close(); + } + + @Test + public void testDisablingObserverReadUsingNameserviceOverride() throws Exception { + // Disable observer reads using per-nameservice override + Configuration confOverrides = new Configuration(false); + confOverrides.set(RBFConfigKeys.DFS_ROUTER_OBSERVER_READ_OVERRIDES, "ns0"); + startUpCluster(1, confOverrides); + + RouterContext routerContext = cluster.getRandomRouter(); + FileSystem fileSystem = routerContext.getFileSystem(); + Path path = new Path("/testFile"); + fileSystem.create(path).close(); + fileSystem.open(path).close(); + fileSystem.close(); + + long rpcCountForActive = routerContext.getRouter().getRpcServer() + .getRPCMetrics().getActiveProxyOps(); + // Create, complete and read calls should be sent to active + assertEquals("Three calls should be sent to active", 3, rpcCountForActive); + + long rpcCountForObserver = routerContext.getRouter().getRpcServer() + .getRPCMetrics().getObserverProxyOps(); + assertEquals("Zero calls should be sent to observer", 0, rpcCountForObserver); + } + + @Test + public void testReadWhenObserverIsDown() throws Exception { + startUpCluster(1); + RouterContext routerContext = cluster.getRandomRouter(); + FileSystem fileSystem = routerContext.getFileSystem(); + Path path = new Path("/testFile1"); + // Send Create call to active + fileSystem.create(path).close(); + + // Stop observer NN + int nnIndex = stopObserver(1); + + assertNotEquals("No observer found", 3, nnIndex); + + // Send read request + fileSystem.open(path).close(); + + long rpcCountForActive = routerContext.getRouter().getRpcServer() + .getRPCMetrics().getActiveProxyOps(); + // Create, complete and getBlockLocation calls should be sent to active + assertEquals("Three calls should be sent to active", 3, + rpcCountForActive); + + long rpcCountForObserver = routerContext.getRouter().getRpcServer() + .getRPCMetrics().getObserverProxyOps(); + assertEquals("No call should send to observer", 0, + rpcCountForObserver); + fileSystem.close(); + } + + @Test + public void testMultipleObserver() throws Exception { + startUpCluster(2); + RouterContext routerContext = cluster.getRandomRouter(); + FileSystem fileSystem = routerContext.getFileSystem(); + Path path = new Path("/testFile1"); + // Send Create call to active + fileSystem.create(path).close(); + + // Stop one observer NN + stopObserver(1); + + // Send read request + fileSystem.open(path).close(); + + long rpcCountForActive = routerContext.getRouter().getRpcServer() + .getRPCMetrics().getActiveProxyOps(); + + long expectedActiveRpc = 2; + long expectedObserverRpc = 1; + + // Create and complete calls should be sent to active + assertEquals("Two calls should be sent to active", + expectedActiveRpc, rpcCountForActive); + + long rpcCountForObserver = routerContext.getRouter() + .getRpcServer().getRPCMetrics().getObserverProxyOps(); + // getBlockLocation call should send to observer + assertEquals("Read should be success with another observer", + expectedObserverRpc, rpcCountForObserver); + + // Stop one observer NN + stopObserver(1); + + // Send read request + fileSystem.open(path).close(); + + rpcCountForActive = routerContext.getRouter() + .getRpcServer().getRPCMetrics().getActiveProxyOps(); + + // getBlockLocation call should be sent to active + expectedActiveRpc += 1; + assertEquals("One call should be sent to active", expectedActiveRpc, + rpcCountForActive); + expectedObserverRpc += 0; + rpcCountForObserver = routerContext.getRouter() + .getRpcServer().getRPCMetrics().getObserverProxyOps(); + assertEquals("No call should send to observer", + expectedObserverRpc, rpcCountForObserver); + fileSystem.close(); + } + + private int stopObserver(int num) { + int nnIndex; + for (nnIndex = 0; nnIndex < cluster.getNamenodes().size(); nnIndex++) { + NameNode nameNode = cluster.getCluster().getNameNode(nnIndex); + if (nameNode != null && nameNode.isObserverState()) { + cluster.getCluster().shutdownNameNode(nnIndex); + num--; + if (num == 0) { + break; + } + } + } + return nnIndex; + } + + // test router observer with multiple to know which observer NN received + // requests + @Test + public void testMultipleObserverRouter() throws Exception { + StateStoreDFSCluster innerCluster; + RouterContext routerContext; + MembershipNamenodeResolver resolver; + + String ns0; + String ns1; + //create 4NN, One Active One Standby and Two Observers + innerCluster = new StateStoreDFSCluster(true, 4, 4, TimeUnit.SECONDS.toMillis(5), + TimeUnit.SECONDS.toMillis(5)); + Configuration routerConf = + new RouterConfigBuilder().stateStore().admin().rpc() + .enableLocalHeartbeat(true).heartbeat().build(); + + StringBuilder sb = new StringBuilder(); + ns0 = innerCluster.getNameservices().get(0); + MiniRouterDFSCluster.NamenodeContext context = + innerCluster.getNamenodes(ns0).get(1); + routerConf.set(DFS_NAMESERVICE_ID, ns0); + routerConf.set(DFS_HA_NAMENODE_ID_KEY, context.getNamenodeId()); + + // Specify namenodes (ns1.nn0,ns1.nn1) to monitor + ns1 = innerCluster.getNameservices().get(1); + for (MiniRouterDFSCluster.NamenodeContext ctx : innerCluster.getNamenodes(ns1)) { + String suffix = ctx.getConfSuffix(); + if (sb.length() != 0) { + sb.append(","); + } + sb.append(suffix); + } + routerConf.set(DFS_ROUTER_MONITOR_NAMENODE, sb.toString()); + routerConf.setBoolean(RBFConfigKeys.DFS_ROUTER_OBSERVER_READ_DEFAULT_KEY, true); + routerConf.setBoolean(DFSConfigKeys.DFS_HA_TAILEDITS_INPROGRESS_KEY, true); + routerConf.set(DFSConfigKeys.DFS_HA_TAILEDITS_PERIOD_KEY, "0ms"); + + innerCluster.addNamenodeOverrides(routerConf); + innerCluster.addRouterOverrides(routerConf); + innerCluster.startCluster(); + + if (innerCluster.isHighAvailability()) { + for (String ns : innerCluster.getNameservices()) { + innerCluster.switchToActive(ns, NAMENODES[0]); + innerCluster.switchToStandby(ns, NAMENODES[1]); + for (int i = 2; i < 4; i++) { + innerCluster.switchToObserver(ns, NAMENODES[i]); + } + } + } + innerCluster.startRouters(); + innerCluster.waitClusterUp(); + + routerContext = innerCluster.getRandomRouter(); + resolver = (MembershipNamenodeResolver) routerContext.getRouter() + .getNamenodeResolver(); + + resolver.loadCache(true); + List namespaceInfo0 = + resolver.getNamenodesForNameserviceId(ns0, true); + List namespaceInfo1 = + resolver.getNamenodesForNameserviceId(ns1, true); + assertEquals(namespaceInfo0.get(0).getState(), + FederationNamenodeServiceState.OBSERVER); + assertEquals(namespaceInfo0.get(1).getState(), + FederationNamenodeServiceState.OBSERVER); + assertNotEquals(namespaceInfo0.get(0).getNamenodeId(), + namespaceInfo0.get(1).getNamenodeId()); + assertEquals(namespaceInfo1.get(0).getState(), + FederationNamenodeServiceState.OBSERVER); + } + + @Test + public void testUnavailableObserverNN() throws Exception { + startUpCluster(2); + RouterContext routerContext = cluster.getRandomRouter(); + FileSystem fileSystem = routerContext.getFileSystem(); + + stopObserver(2); + + Path path = new Path("/testFile"); + // Send Create call to active + fileSystem.create(path).close(); + + // Send read request. + fileSystem.open(path).close(); + + long rpcCountForActive = routerContext.getRouter().getRpcServer() + .getRPCMetrics().getActiveProxyOps(); + + // Create, complete and getBlockLocations + // calls should be sent to active. + assertEquals("Three calls should be send to active", + 3, rpcCountForActive); + + + boolean hasUnavailable = false; + for(String ns : cluster.getNameservices()) { + List nns = routerContext.getRouter() + .getNamenodeResolver().getNamenodesForNameserviceId(ns, false); + for(FederationNamenodeContext nn : nns) { + if(FederationNamenodeServiceState.UNAVAILABLE == nn.getState()) { + hasUnavailable = true; + } + } + } + // After attempting to communicate with unavailable observer namenode, + // its state is updated to unavailable. + assertTrue("There must be unavailable namenodes", hasUnavailable); + } + + @Test + public void testRouterMsync() throws Exception { + startUpCluster(1); + RouterContext routerContext = cluster.getRandomRouter(); + + FileSystem fileSystem = routerContext.getFileSystem(); + Path path = new Path("/testFile"); + + // Send Create call to active + fileSystem.create(path).close(); + long rpcCountForActive = routerContext.getRouter().getRpcServer() + .getRPCMetrics().getActiveProxyOps(); + // Create and complete calls should be sent to active + assertEquals("Two calls should be sent to active", 2, + rpcCountForActive); + + // Send msync + fileSystem.msync(); + rpcCountForActive = routerContext.getRouter().getRpcServer() + .getRPCMetrics().getActiveProxyOps(); + // 2 msync calls should be sent. One to each active namenode in the two namespaces. + assertEquals("Four calls should be sent to active", 4, + rpcCountForActive); + fileSystem.close(); + } +} \ No newline at end of file diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRenewLeaseWithSameINodeId.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRenewLeaseWithSameINodeId.java new file mode 100644 index 0000000000000..76d64e4eb5583 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRenewLeaseWithSameINodeId.java @@ -0,0 +1,106 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hdfs.server.federation.router; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FSDataOutputStream; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hdfs.DistributedFileSystem; +import org.apache.hadoop.hdfs.protocol.HdfsFileStatus; +import org.apache.hadoop.hdfs.server.federation.MiniRouterDFSCluster; +import org.apache.hadoop.hdfs.server.federation.MockResolver; +import org.apache.hadoop.hdfs.server.federation.RouterConfigBuilder; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; + +import java.io.IOException; + +import static org.junit.Assert.assertEquals; + +/** + * Testing DFSClient renewLease with same INodeId. + */ +public class TestRenewLeaseWithSameINodeId { + + /** Federated HDFS cluster. */ + private static MiniRouterDFSCluster cluster; + + /** The first Router Context for this federated cluster. */ + private static MiniRouterDFSCluster.RouterContext routerContext; + + @BeforeClass + public static void globalSetUp() throws Exception { + cluster = new MiniRouterDFSCluster(false, 2); + cluster.setNumDatanodesPerNameservice(3); + cluster.startCluster(); + + Configuration routerConf = new RouterConfigBuilder() + .metrics() + .rpc() + .quota() + .build(); + cluster.addRouterOverrides(routerConf); + cluster.startRouters(); + + // Register and verify all NNs with all routers + cluster.registerNamenodes(); + cluster.waitNamenodeRegistration(); + + routerContext = cluster.getRouters().get(0); + } + + @AfterClass + public static void tearDown() throws Exception { + cluster.shutdown(); + } + + /** + * Testing case: + * 1. One Router DFSClient writing multi files from different namespace with same iNodeId. + * 2. DFSClient Lease Renewer should work well. + */ + @Test + public void testRenewLeaseWithSameINodeId() throws IOException { + // Add mount point "/ns0" and "/ns1" + Router router = cluster.getRouters().get(0).getRouter(); + MockResolver resolver = (MockResolver) router.getSubclusterResolver(); + resolver.addLocation("/ns0", cluster.getNameservices().get(0), "/ns0"); + resolver.addLocation("/ns1", cluster.getNameservices().get(1), "/ns1"); + + DistributedFileSystem fs = (DistributedFileSystem) routerContext.getFileSystem(); + + Path path1 = new Path("/ns0/file"); + Path path2 = new Path("/ns1/file"); + + try (FSDataOutputStream ignored1 = fs.create(path1); + FSDataOutputStream ignored2 = fs.create(path2)) { + HdfsFileStatus fileStatus1 = fs.getClient().getFileInfo(path1.toUri().getPath()); + HdfsFileStatus fileStatus2 = fs.getClient().getFileInfo(path2.toUri().getPath()); + + // The fileId of the files from different new namespaces should be same. + assertEquals(fileStatus2.getFileId(), fileStatus1.getFileId()); + + // The number of fileBeingWritten of this DFSClient should be two. + assertEquals(2, fs.getClient().getNumOfFilesBeingWritten()); + } + + // The number of fileBeingWritten of this DFSClient should be zero. + assertEquals(0, fs.getClient().getNumOfFilesBeingWritten()); + } +} diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterClientRejectOverload.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterClientRejectOverload.java index 71ec747af4c30..8d776546801ba 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterClientRejectOverload.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterClientRejectOverload.java @@ -206,33 +206,29 @@ private void testOverloaded(int expOverloadMin, int expOverloadMax, for (int i = 0; i < numOps; i++) { // Stagger the operations a little (50ms) final int sleepTime = i * 50; - Future future = exec.submit(new Runnable() { - @Override - public void run() { - DFSClient routerClient = null; - try { - Thread.sleep(sleepTime); - routerClient = new DFSClient(address, conf); - String clientName = routerClient.getClientName(); - ClientProtocol routerProto = routerClient.getNamenode(); - routerProto.renewLease(clientName); - } catch (RemoteException re) { - IOException ioe = re.unwrapRemoteException(); - assertTrue("Wrong exception: " + ioe, - ioe instanceof StandbyException); - assertExceptionContains("is overloaded", ioe); - overloadException.incrementAndGet(); - } catch (IOException e) { - fail("Unexpected exception: " + e); - } catch (InterruptedException e) { - fail("Cannot sleep: " + e); - } finally { - if (routerClient != null) { - try { - routerClient.close(); - } catch (IOException e) { - LOG.error("Cannot close the client"); - } + Future future = exec.submit(() -> { + DFSClient routerClient = null; + try { + Thread.sleep(sleepTime); + routerClient = new DFSClient(address, conf); + String clientName = routerClient.getClientName(); + ClientProtocol routerProto = routerClient.getNamenode(); + routerProto.renewLease(clientName, null); + } catch (RemoteException re) { + IOException ioe = re.unwrapRemoteException(); + assertTrue("Wrong exception: " + ioe, ioe instanceof StandbyException); + assertExceptionContains("is overloaded", ioe); + overloadException.incrementAndGet(); + } catch (IOException e) { + fail("Unexpected exception: " + e); + } catch (InterruptedException e) { + fail("Cannot sleep: " + e); + } finally { + if (routerClient != null) { + try { + routerClient.close(); + } catch (IOException e) { + LOG.error("Cannot close the client"); } } } @@ -390,7 +386,7 @@ public void testAsyncCallerPoolMetrics() throws Exception { cluster.getRouterClientConf()); String clientName = routerClient.getClientName(); ClientProtocol routerProto = routerClient.getNamenode(); - routerProto.renewLease(clientName); + routerProto.renewLease(clientName, null); } catch (Exception e) { fail("Client request failed: " + e); } finally { diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterFaultTolerant.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterFaultTolerant.java index ef5322ba218ed..34d50937b2bf6 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterFaultTolerant.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterFaultTolerant.java @@ -168,7 +168,7 @@ public void cleanup() throws Exception { } namenodes.clear(); - routers.forEach(router -> router.stop()); + routers.forEach(Router::stop); routers.clear(); if (service != null) { diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterFederatedState.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterFederatedState.java new file mode 100644 index 0000000000000..2bc8cfc21b230 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterFederatedState.java @@ -0,0 +1,104 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hdfs.server.federation.router; + +import java.util.HashMap; +import java.util.Map; +import org.apache.hadoop.ipc.AlignmentContext; +import org.apache.hadoop.ipc.ClientId; +import org.apache.hadoop.ipc.RPC; +import org.apache.hadoop.ipc.RpcConstants; +import org.apache.hadoop.ipc.protobuf.RpcHeaderProtos; +import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.RouterFederatedStateProto; +import org.apache.hadoop.thirdparty.protobuf.InvalidProtocolBufferException; +import org.apache.hadoop.util.ProtoUtil; +import org.junit.Test; + +import static org.junit.Assert.*; + + +public class TestRouterFederatedState { + + @Test + public void testRpcRouterFederatedState() throws InvalidProtocolBufferException { + byte[] uuid = ClientId.getClientId(); + Map expectedStateIds = new HashMap() { + { + put("namespace1", 11L); + put("namespace2", 22L); + } + }; + + AlignmentContext alignmentContext = new AlignmentContextWithRouterState(expectedStateIds); + + RpcHeaderProtos.RpcRequestHeaderProto header = ProtoUtil.makeRpcRequestHeader( + RPC.RpcKind.RPC_PROTOCOL_BUFFER, + RpcHeaderProtos.RpcRequestHeaderProto.OperationProto.RPC_FINAL_PACKET, + 0, + RpcConstants.INVALID_RETRY_COUNT, + uuid, + alignmentContext); + + Map stateIdsFromHeader = + RouterFederatedStateProto.parseFrom( + header.getRouterFederatedState().toByteArray() + ).getNamespaceStateIdsMap(); + + assertEquals(expectedStateIds, stateIdsFromHeader); + } + + private static class AlignmentContextWithRouterState implements AlignmentContext { + + private Map routerFederatedState; + + AlignmentContextWithRouterState(Map namespaceStates) { + this.routerFederatedState = namespaceStates; + } + + @Override + public void updateRequestState(RpcHeaderProtos.RpcRequestHeaderProto.Builder header) { + RouterFederatedStateProto fedState = RouterFederatedStateProto + .newBuilder() + .putAllNamespaceStateIds(routerFederatedState) + .build(); + + header.setRouterFederatedState(fedState.toByteString()); + } + + @Override + public void updateResponseState(RpcHeaderProtos.RpcResponseHeaderProto.Builder header) {} + + @Override + public void receiveResponseState(RpcHeaderProtos.RpcResponseHeaderProto header) {} + + @Override + public long receiveRequestState(RpcHeaderProtos.RpcRequestHeaderProto header, long threshold) { + return 0; + } + + @Override + public long getLastSeenStateId() { + return 0; + } + + @Override + public boolean isCoordinatedCall(String protocolName, String method) { + return false; + } + } +} \ No newline at end of file diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterMountTableWithoutDefaultNS.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterMountTableWithoutDefaultNS.java new file mode 100644 index 0000000000000..57d4c69db698f --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterMountTableWithoutDefaultNS.java @@ -0,0 +1,268 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hdfs.server.federation.router; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.ContentSummary; +import org.apache.hadoop.fs.FSDataOutputStream; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hdfs.protocol.ClientProtocol; +import org.apache.hadoop.hdfs.protocol.HdfsFileStatus; +import org.apache.hadoop.hdfs.server.federation.MiniRouterDFSCluster.RouterContext; +import org.apache.hadoop.hdfs.server.federation.RouterConfigBuilder; +import org.apache.hadoop.hdfs.server.federation.StateStoreDFSCluster; +import org.apache.hadoop.hdfs.server.federation.resolver.MountTableManager; +import org.apache.hadoop.hdfs.server.federation.resolver.MountTableResolver; +import org.apache.hadoop.hdfs.server.federation.resolver.RemoteLocation; +import org.apache.hadoop.hdfs.server.federation.resolver.RouterResolveException; +import org.apache.hadoop.hdfs.server.federation.store.protocol.AddMountTableEntryRequest; +import org.apache.hadoop.hdfs.server.federation.store.protocol.AddMountTableEntryResponse; +import org.apache.hadoop.hdfs.server.federation.store.protocol.GetMountTableEntriesRequest; +import org.apache.hadoop.hdfs.server.federation.store.protocol.GetMountTableEntriesResponse; +import org.apache.hadoop.hdfs.server.federation.store.protocol.RemoveMountTableEntryRequest; +import org.apache.hadoop.hdfs.server.federation.store.records.MountTable; +import org.apache.hadoop.test.LambdaTestUtils; +import org.junit.After; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; + +import java.io.IOException; +import java.util.Collections; +import java.util.List; +import java.util.Map; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +/** + * Test a router end-to-end including the MountTable without default nameservice. + */ +public class TestRouterMountTableWithoutDefaultNS { + private static StateStoreDFSCluster cluster; + private static RouterContext routerContext; + private static MountTableResolver mountTable; + private static ClientProtocol routerProtocol; + private static FileSystem nnFs0; + private static FileSystem nnFs1; + + @BeforeClass + public static void globalSetUp() throws Exception { + // Build and start a federated cluster + cluster = new StateStoreDFSCluster(false, 2); + Configuration conf = new RouterConfigBuilder() + .stateStore() + .admin() + .rpc() + .build(); + conf.setInt(RBFConfigKeys.DFS_ROUTER_ADMIN_MAX_COMPONENT_LENGTH_KEY, 20); + conf.setBoolean(RBFConfigKeys.DFS_ROUTER_DEFAULT_NAMESERVICE_ENABLE, false); + cluster.addRouterOverrides(conf); + cluster.startCluster(); + cluster.startRouters(); + cluster.waitClusterUp(); + + // Get the end points + nnFs0 = cluster.getNamenode("ns0", null).getFileSystem(); + nnFs1 = cluster.getNamenode("ns1", null).getFileSystem(); + routerContext = cluster.getRandomRouter(); + Router router = routerContext.getRouter(); + routerProtocol = routerContext.getClient().getNamenode(); + mountTable = (MountTableResolver) router.getSubclusterResolver(); + } + + @AfterClass + public static void tearDown() { + if (cluster != null) { + cluster.stopRouter(routerContext); + cluster.shutdown(); + cluster = null; + } + } + + @After + public void clearMountTable() throws IOException { + RouterClient client = routerContext.getAdminClient(); + MountTableManager mountTableManager = client.getMountTableManager(); + GetMountTableEntriesRequest req1 = GetMountTableEntriesRequest.newInstance("/"); + GetMountTableEntriesResponse response = mountTableManager.getMountTableEntries(req1); + for (MountTable entry : response.getEntries()) { + RemoveMountTableEntryRequest req2 = + RemoveMountTableEntryRequest.newInstance(entry.getSourcePath()); + mountTableManager.removeMountTableEntry(req2); + } + } + + /** + * Add a mount table entry to the mount table through the admin API. + * @param entry Mount table entry to add. + * @return If it was succesfully added. + * @throws IOException Problems adding entries. + */ + private boolean addMountTable(final MountTable entry) throws IOException { + RouterClient client = routerContext.getAdminClient(); + MountTableManager mountTableManager = client.getMountTableManager(); + AddMountTableEntryRequest addRequest = AddMountTableEntryRequest.newInstance(entry); + AddMountTableEntryResponse addResponse = mountTableManager.addMountTableEntry(addRequest); + + // Reload the Router cache + mountTable.loadCache(true); + + return addResponse.getStatus(); + } + + /** + * Verify that RBF that disable default nameservice should support + * get information about ancestor mount points. + */ + @Test + public void testGetFileInfoWithSubMountPoint() throws IOException { + MountTable addEntry = MountTable.newInstance("/testdir/1", + Collections.singletonMap("ns0", "/testdir/1")); + assertTrue(addMountTable(addEntry)); + HdfsFileStatus finfo = routerProtocol.getFileInfo("/testdir"); + assertNotNull(finfo); + assertEquals("supergroup", finfo.getGroup()); + assertTrue(finfo.isDirectory()); + } + + /** + * Verify that RBF doesn't support get the file information + * with no location and sub mount points. + */ + @Test + public void testGetFileInfoWithoutSubMountPoint() throws Exception { + MountTable addEntry = MountTable.newInstance("/testdir/1", + Collections.singletonMap("ns0", "/testdir/1")); + assertTrue(addMountTable(addEntry)); + LambdaTestUtils.intercept(RouterResolveException.class, + () -> routerContext.getRouter().getRpcServer().getFileInfo("/testdir2")); + } + + /** + * Verify that RBF that disable default nameservice should support + * get information about ancestor mount points. + */ + @Test + public void testGetContentSummaryWithSubMountPoint() throws IOException { + MountTable addEntry = MountTable.newInstance("/testdir/1/2", + Collections.singletonMap("ns0", "/testdir/1/2")); + assertTrue(addMountTable(addEntry)); + + try { + writeData(nnFs0, new Path("/testdir/1/2/3"), 10 * 1024 * 1024); + + RouterRpcServer routerRpcServer = routerContext.getRouterRpcServer(); + ContentSummary summaryFromRBF = routerRpcServer.getContentSummary("/testdir"); + assertNotNull(summaryFromRBF); + assertEquals(1, summaryFromRBF.getFileCount()); + assertEquals(10 * 1024 * 1024, summaryFromRBF.getLength()); + } finally { + nnFs0.delete(new Path("/testdir"), true); + } + } + + @Test + public void testGetAllLocations() throws IOException { + // Add mount table entry. + MountTable addEntry = MountTable.newInstance("/testA", + Collections.singletonMap("ns0", "/testA")); + assertTrue(addMountTable(addEntry)); + addEntry = MountTable.newInstance("/testA/testB", + Collections.singletonMap("ns1", "/testA/testB")); + assertTrue(addMountTable(addEntry)); + addEntry = MountTable.newInstance("/testA/testB/testC", + Collections.singletonMap("ns2", "/testA/testB/testC")); + assertTrue(addMountTable(addEntry)); + + RouterClientProtocol protocol = routerContext.getRouterRpcServer().getClientProtocolModule(); + Map> locations = protocol.getAllLocations("/testA"); + assertEquals(3, locations.size()); + } + + @Test + public void testGetLocationsForContentSummary() throws Exception { + // Add mount table entry. + MountTable addEntry = MountTable.newInstance("/testA/testB", + Collections.singletonMap("ns0", "/testA/testB")); + assertTrue(addMountTable(addEntry)); + addEntry = MountTable.newInstance("/testA/testB/testC", + Collections.singletonMap("ns1", "/testA/testB/testC")); + assertTrue(addMountTable(addEntry)); + + RouterClientProtocol protocol = routerContext.getRouterRpcServer().getClientProtocolModule(); + List locations = protocol.getLocationsForContentSummary("/testA"); + assertEquals(2, locations.size()); + + for (RemoteLocation location : locations) { + String nsId = location.getNameserviceId(); + if ("ns0".equals(nsId)) { + assertEquals("/testA/testB", location.getDest()); + } else if ("ns1".equals(nsId)) { + assertEquals("/testA/testB/testC", location.getDest()); + } else { + fail("Unexpected NS " + nsId); + } + } + + LambdaTestUtils.intercept(NoLocationException.class, + () -> protocol.getLocationsForContentSummary("/testB")); + } + + @Test + public void testGetContentSummary() throws Exception { + try { + // Add mount table entry. + MountTable addEntry = MountTable.newInstance("/testA", + Collections.singletonMap("ns0", "/testA")); + assertTrue(addMountTable(addEntry)); + addEntry = MountTable.newInstance("/testA/testB", + Collections.singletonMap("ns0", "/testA/testB")); + assertTrue(addMountTable(addEntry)); + addEntry = MountTable.newInstance("/testA/testB/testC", + Collections.singletonMap("ns1", "/testA/testB/testC")); + assertTrue(addMountTable(addEntry)); + + writeData(nnFs0, new Path("/testA/testB/file1"), 1024 * 1024); + writeData(nnFs1, new Path("/testA/testB/testC/file2"), 1024 * 1024); + writeData(nnFs1, new Path("/testA/testB/testC/file3"), 1024 * 1024); + + RouterRpcServer routerRpcServer = routerContext.getRouterRpcServer(); + ContentSummary summary = routerRpcServer.getContentSummary("/testA"); + assertEquals(3, summary.getFileCount()); + assertEquals(1024 * 1024 * 3, summary.getLength()); + + LambdaTestUtils.intercept(NoLocationException.class, + () -> routerRpcServer.getContentSummary("/testB")); + } finally { + nnFs0.delete(new Path("/testA"), true); + nnFs1.delete(new Path("/testA"), true); + } + } + + void writeData(FileSystem fs, Path path, int fileLength) throws IOException { + try (FSDataOutputStream outputStream = fs.create(path)) { + for (int writeSize = 0; writeSize < fileLength; writeSize++) { + outputStream.write(writeSize); + } + } + } +} diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterNamenodeHeartbeat.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterNamenodeHeartbeat.java index 94f2baeaed136..04b4b58bcb6e3 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterNamenodeHeartbeat.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterNamenodeHeartbeat.java @@ -167,7 +167,7 @@ public void testHearbeat() throws InterruptedException, IOException { // Verify the locator has matching NN entries for each NS for (String ns : cluster.getNameservices()) { List nns = - namenodeResolver.getNamenodesForNameserviceId(ns); + namenodeResolver.getNamenodesForNameserviceId(ns, false); // Active FederationNamenodeContext active = nns.get(0); @@ -191,7 +191,7 @@ public void testHearbeat() throws InterruptedException, IOException { // Verify the locator has recorded the failover for the failover NS List failoverNSs = - namenodeResolver.getNamenodesForNameserviceId(failoverNS); + namenodeResolver.getNamenodesForNameserviceId(failoverNS, false); // Active FederationNamenodeContext active = failoverNSs.get(0); assertEquals(NAMENODES[1], active.getNamenodeId()); @@ -202,7 +202,7 @@ public void testHearbeat() throws InterruptedException, IOException { // Verify the locator has the same records for the other ns List normalNss = - namenodeResolver.getNamenodesForNameserviceId(normalNs); + namenodeResolver.getNamenodesForNameserviceId(normalNs, false); // Active active = normalNss.get(0); assertEquals(NAMENODES[0], active.getNamenodeId()); diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterNamenodeMonitoring.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterNamenodeMonitoring.java index 4fae86b01d399..bae2dea3ceabf 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterNamenodeMonitoring.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterNamenodeMonitoring.java @@ -204,7 +204,7 @@ public void testNamenodeMonitoring() throws Exception { final List namespaceInfo = new ArrayList<>(); for (String nsId : nns.keySet()) { List nnReports = - resolver.getNamenodesForNameserviceId(nsId); + resolver.getNamenodesForNameserviceId(nsId, false); namespaceInfo.addAll(nnReports); } for (FederationNamenodeContext nnInfo : namespaceInfo) { diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterNamenodeWebScheme.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterNamenodeWebScheme.java index ab507aaf9ecd4..f23b02092a299 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterNamenodeWebScheme.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterNamenodeWebScheme.java @@ -194,7 +194,7 @@ private void testWebScheme(HttpConfig.Policy httpPolicy, final List namespaceInfo = new ArrayList<>(); for (String nsId : nns.keySet()) { List nnReports = - resolver.getNamenodesForNameserviceId(nsId); + resolver.getNamenodesForNameserviceId(nsId, false); namespaceInfo.addAll(nnReports); } for (FederationNamenodeContext nnInfo : namespaceInfo) { diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterQuota.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterQuota.java index b69004198eb48..aa3d547056134 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterQuota.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterQuota.java @@ -74,8 +74,6 @@ import org.junit.Before; import org.junit.Test; -import java.util.function.Supplier; - /** * Tests quota behaviors in Router-based Federation. */ @@ -210,21 +208,17 @@ public void testStorageSpaceQuotaExceed() throws Exception { routerClient.create("/ssquota/file", true).close(); routerClient.create("/ssquota/subdir/file", true).close(); - GenericTestUtils.waitFor(new Supplier() { - - @Override - public Boolean get() { - boolean isDsQuotaViolated = false; - try { - // append data to trigger NSQuotaExceededException - appendData("/ssquota/file", routerClient, BLOCK_SIZE); - appendData("/ssquota/subdir/file", routerClient, BLOCK_SIZE); - } catch (DSQuotaExceededException e) { - isDsQuotaViolated = true; - } catch (IOException ignored) { - } - return isDsQuotaViolated; + GenericTestUtils.waitFor(() -> { + boolean isDsQuotaViolated = false; + try { + // append data to trigger NSQuotaExceededException + appendData("/ssquota/file", routerClient, BLOCK_SIZE); + appendData("/ssquota/subdir/file", routerClient, BLOCK_SIZE); + } catch (DSQuotaExceededException e) { + isDsQuotaViolated = true; + } catch (IOException ignored) { } + return isDsQuotaViolated; }, 5000, 60000); // append data to destination path in real FileSystem should be okay diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterRPCClientRetries.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterRPCClientRetries.java index 73803d9805203..1054e5ac8cf97 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterRPCClientRetries.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterRPCClientRetries.java @@ -54,8 +54,6 @@ import org.junit.Test; import org.junit.rules.Timeout; -import java.util.function.Supplier; - /** * Test retry behavior of the Router RPC Client. */ @@ -153,7 +151,7 @@ public void testRetryWhenOneNameServiceDown() throws Exception { DFSClient client = nnContext1.getClient(); // Renew lease for the DFS client, it will succeed. - routerProtocol.renewLease(client.getClientName()); + routerProtocol.renewLease(client.getClientName(), null); // Verify the retry times, it will retry one time for ns0. FederationRPCMetrics rpcMetrics = routerContext.getRouter() @@ -168,7 +166,7 @@ public void testRetryWhenOneNameServiceDown() throws Exception { private void registerInvalidNameReport() throws IOException { String ns0 = cluster.getNameservices().get(0); List origin = resolver - .getNamenodesForNameserviceId(ns0); + .getNamenodesForNameserviceId(ns0, false); FederationNamenodeContext nnInfo = origin.get(0); NamenodeStatusReport report = new NamenodeStatusReport(ns0, nnInfo.getNamenodeId(), nnInfo.getRpcAddress(), @@ -237,11 +235,6 @@ private static int getNumDatanodes(final String jsonString) private static void waitUpdateLiveNodes( final String oldValue, final NamenodeBeanMetrics metrics) throws Exception { - waitFor(new Supplier() { - @Override - public Boolean get() { - return !oldValue.equals(metrics.getLiveNodes()); - } - }, 500, 5 * 1000); + waitFor(() -> !oldValue.equals(metrics.getLiveNodes()), 500, 5 * 1000); } } diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterRetryCache.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterRetryCache.java new file mode 100644 index 0000000000000..e498874812a9f --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterRetryCache.java @@ -0,0 +1,182 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hdfs.server.federation.router; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.permission.FsPermission; +import org.apache.hadoop.ha.HAServiceProtocol; +import org.apache.hadoop.hdfs.server.federation.MiniRouterDFSCluster; +import org.apache.hadoop.hdfs.server.namenode.NameNode; +import org.apache.hadoop.io.retry.RetryInvocationHandler; +import org.apache.hadoop.ipc.Client; +import org.apache.hadoop.security.UserGroupInformation; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import java.io.IOException; +import java.security.PrivilegedExceptionAction; + +import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_IP_PROXY_USERS; +import static org.apache.hadoop.hdfs.server.federation.FederationTestUtils.NAMENODES; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; + +public class TestRouterRetryCache { + /** Federated HDFS cluster. */ + private MiniRouterDFSCluster cluster; + + @Before + public void setup() throws Exception { + UserGroupInformation routerUser = UserGroupInformation.getLoginUser(); + Configuration conf = new Configuration(); + String adminUser = routerUser.getUserName(); + conf.set("hadoop.proxyuser." + adminUser + ".hosts", "*"); + conf.set("hadoop.proxyuser." + adminUser + ".groups", "*"); + conf.set("hadoop.proxyuser.fake_joe.hosts", "*"); + conf.set("hadoop.proxyuser.fake_joe.groups", "*"); + conf.set(DFS_NAMENODE_IP_PROXY_USERS, routerUser.getShortUserName()); + cluster = new MiniRouterDFSCluster(true, 1, conf); + cluster.addNamenodeOverrides(conf); + + // Start NNs and DNs and wait until ready + cluster.startCluster(); + + // Start routers with only an RPC service + cluster.startRouters(); + + // Register and verify all NNs with all routers + cluster.registerNamenodes(); + cluster.waitNamenodeRegistration(); + + // Setup the mount table + cluster.installMockLocations(); + + // Making one Namenodes active per nameservice + if (cluster.isHighAvailability()) { + for (String ns : cluster.getNameservices()) { + cluster.switchToActive(ns, NAMENODES[0]); + cluster.switchToStandby(ns, NAMENODES[1]); + } + } + cluster.waitActiveNamespaces(); + } + + @After + public void teardown() throws IOException { + if (cluster != null) { + cluster.shutdown(); + cluster = null; + } + } + + @Test + public void testRetryCacheWithOneLevelProxyUser() throws Exception { + internalTestRetryCache(false); + } + + @Test + public void testRetryCacheWithTwoLevelProxyUser() throws Exception { + internalTestRetryCache(true); + } + + /** + * Test RetryCache through RBF with proxyUser and non-ProxyUser respectively. + * + * 1. Start cluster with current user. + * 2. Create one test directory by the admin user. + * 3. Create one Router FileSystem with one mocked user, one proxyUser or non-ProxyUser. + * 4. Try to create one test directory by the router fileSystem. + * 5. Try to rename the new test directory to one test destination directory + * 6. Then failover the active to the standby + * 7. Try to rename the source directory to the destination directory again with the same callId + * 8. Try to + */ + private void internalTestRetryCache(boolean twoLevelProxyUGI) throws Exception { + RetryInvocationHandler.SET_CALL_ID_FOR_TEST.set(false); + FileSystem routerFS = cluster.getRandomRouter().getFileSystem(); + Path testDir = new Path("/target-ns0/testdir"); + routerFS.mkdirs(testDir); + routerFS.setPermission(testDir, FsPermission.getDefault()); + + // Run as fake joe to authorize the test + UserGroupInformation joe = UserGroupInformation.createUserForTesting("fake_joe", + new String[] {"fake_group"}); + if (twoLevelProxyUGI) { + joe = UserGroupInformation.createProxyUser("fake_proxy_joe", joe); + } + FileSystem joeFS = joe.doAs((PrivilegedExceptionAction) () -> + FileSystem.newInstance(routerFS.getUri(), routerFS.getConf())); + + Path renameSrc = new Path(testDir, "renameSrc"); + Path renameDst = new Path(testDir, "renameDst"); + joeFS.mkdirs(renameSrc); + + assertEquals(HAServiceProtocol.HAServiceState.ACTIVE, + cluster.getCluster().getNamesystem(0).getState()); + + int callId = Client.nextCallId(); + Client.setCallIdAndRetryCount(callId, 0, null); + assertTrue(joeFS.rename(renameSrc, renameDst)); + + Client.setCallIdAndRetryCount(callId, 0, null); + assertTrue(joeFS.rename(renameSrc, renameDst)); + + String ns0 = cluster.getNameservices().get(0); + cluster.switchToStandby(ns0, NAMENODES[0]); + cluster.switchToActive(ns0, NAMENODES[1]); + + assertEquals(HAServiceProtocol.HAServiceState.ACTIVE, + cluster.getCluster().getNamesystem(1).getState()); + + Client.setCallIdAndRetryCount(callId, 0, null); + assertTrue(joeFS.rename(renameSrc, renameDst)); + + FileStatus fileStatus = joeFS.getFileStatus(renameDst); + if (twoLevelProxyUGI) { + assertEquals("fake_proxy_joe", fileStatus.getOwner()); + } else { + assertEquals("fake_joe", fileStatus.getOwner()); + } + + joeFS.delete(renameDst, true); + } + + @Test + public void testParseSpecialValue() { + String mockContent = "mockContent,clientIp:127.0.0.1," + + "clientCallId:12345,clientId:mockClientId"; + String clientIp = NameNode.parseSpecialValue(mockContent, "clientIp:"); + assertEquals("127.0.0.1", clientIp); + + String clientCallId = NameNode.parseSpecialValue( + mockContent, "clientCallId:"); + assertEquals("12345", clientCallId); + + String clientId = NameNode.parseSpecialValue(mockContent, "clientId:"); + assertEquals("mockClientId", clientId); + + String clientRetryNum = NameNode.parseSpecialValue( + mockContent, "clientRetryNum:"); + assertNull(clientRetryNum); + } +} diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterRpc.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterRpc.java index 4aeb2ec9b8f30..35b640d40678a 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterRpc.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterRpc.java @@ -58,6 +58,7 @@ import org.apache.hadoop.crypto.CryptoProtocolVersion; import org.apache.hadoop.fs.ContentSummary; import org.apache.hadoop.fs.CreateFlag; +import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.fs.FileContext; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; @@ -107,6 +108,7 @@ import org.apache.hadoop.hdfs.server.federation.MiniRouterDFSCluster.RouterContext; import org.apache.hadoop.hdfs.server.federation.MockResolver; import org.apache.hadoop.hdfs.server.federation.RouterConfigBuilder; +import org.apache.hadoop.hdfs.server.federation.metrics.FederationRPCMetrics; import org.apache.hadoop.hdfs.server.federation.metrics.NamenodeBeanMetrics; import org.apache.hadoop.hdfs.server.federation.metrics.RBFMetrics; import org.apache.hadoop.hdfs.server.federation.resolver.FileSubclusterResolver; @@ -129,6 +131,7 @@ import org.apache.hadoop.service.Service.STATE; import org.apache.hadoop.test.GenericTestUtils; import org.apache.hadoop.test.LambdaTestUtils; +import org.codehaus.jettison.json.JSONException; import org.codehaus.jettison.json.JSONObject; import org.junit.AfterClass; import org.junit.Before; @@ -1450,6 +1453,119 @@ public void testProxyRestoreFailedStorage() throws Exception { assertEquals(nnSuccess, routerSuccess); } + private void testRenewLeaseInternal(DistributedFileSystem dfs, + FederationRPCMetrics rpcMetrics, Path testPath, boolean createFlag) + throws Exception { + FSDataOutputStream outputStream = null; + try { + if (createFlag) { + outputStream = dfs.create(testPath); + } else { + outputStream = dfs.append(testPath); + } + outputStream.write("hello world. \n".getBytes()); + long proxyOpBeforeRenewLease = rpcMetrics.getProxyOps(); + assertTrue(dfs.getClient().renewLease()); + long proxyOpAfterRenewLease = rpcMetrics.getProxyOps(); + assertEquals((proxyOpBeforeRenewLease + 1), proxyOpAfterRenewLease); + } finally { + if (outputStream != null) { + outputStream.close(); + } + } + } + + @Test + public void testRenewLeaseForECFile() throws Exception { + String ecName = "RS-6-3-1024k"; + FederationRPCMetrics metrics = router.getRouterRpcServer().getRPCMetrics(); + // Install a mount point to a different path to check + MockResolver resolver = + (MockResolver)router.getRouter().getSubclusterResolver(); + String ns0 = cluster.getNameservices().get(0); + resolver.addLocation("/testRenewLease0", ns0, "/testRenewLease0"); + + // Stop LeaseRenewer + DistributedFileSystem routerDFS = (DistributedFileSystem) routerFS; + routerDFS.getClient().getLeaseRenewer().interruptAndJoin(); + + Path testECPath = new Path("/testRenewLease0/ecDirectory/test_ec.txt"); + routerDFS.mkdirs(testECPath.getParent()); + routerDFS.setErasureCodingPolicy( + testECPath.getParent(), ecName); + testRenewLeaseInternal(routerDFS, metrics, testECPath, true); + + ErasureCodingPolicy ecPolicy = routerDFS.getErasureCodingPolicy(testECPath); + assertNotNull(ecPolicy); + assertEquals(ecName, ecPolicy.getName()); + } + + + @Test + public void testRenewLeaseForReplicaFile() throws Exception { + FederationRPCMetrics metrics = router.getRouterRpcServer().getRPCMetrics(); + // Install a mount point to a different path to check + MockResolver resolver = + (MockResolver)router.getRouter().getSubclusterResolver(); + String ns0 = cluster.getNameservices().get(0); + resolver.addLocation("/testRenewLease0", ns0, "/testRenewLease0"); + + // Stop LeaseRenewer + DistributedFileSystem routerDFS = (DistributedFileSystem) routerFS; + routerDFS.getClient().getLeaseRenewer().interruptAndJoin(); + + // Test Replica File + Path testPath = new Path("/testRenewLease0/test_replica.txt"); + testRenewLeaseInternal(routerDFS, metrics, testPath, true); + testRenewLeaseInternal(routerDFS, metrics, testPath, false); + } + + @Test + public void testRenewLeaseWithMultiStream() throws Exception { + FederationRPCMetrics metrics = router.getRouterRpcServer().getRPCMetrics(); + // Install a mount point to a different path to check + MockResolver resolver = + (MockResolver)router.getRouter().getSubclusterResolver(); + String ns0 = cluster.getNameservices().get(0); + String ns1 = cluster.getNameservices().get(1); + resolver.addLocation("/testRenewLease0", ns0, "/testRenewLease0"); + resolver.addLocation("/testRenewLease1", ns1, "/testRenewLease1"); + + // Stop LeaseRenewer + DistributedFileSystem routerDFS = (DistributedFileSystem) routerFS; + routerDFS.getClient().getLeaseRenewer().interruptAndJoin(); + + Path newTestPath0 = new Path("/testRenewLease0/test1.txt"); + Path newTestPath1 = new Path("/testRenewLease1/test1.txt"); + try (FSDataOutputStream outStream1 = routerDFS.create(newTestPath0); + FSDataOutputStream outStream2 = routerDFS.create(newTestPath1)) { + outStream1.write("hello world \n".getBytes()); + outStream2.write("hello world \n".getBytes()); + long proxyOpBeforeRenewLease2 = metrics.getProxyOps(); + assertTrue(routerDFS.getClient().renewLease()); + long proxyOpAfterRenewLease2 = metrics.getProxyOps(); + assertEquals((proxyOpBeforeRenewLease2 + 2), proxyOpAfterRenewLease2); + } + } + + @Test + public void testMkdirWithDisableNameService() throws Exception { + MockResolver resolver = (MockResolver)router.getRouter().getSubclusterResolver(); + String ns0 = cluster.getNameservices().get(0); + resolver.addLocation("/mnt", ns0, "/"); + MockResolver activeNamenodeResolver = (MockResolver)router.getRouter().getNamenodeResolver(); + activeNamenodeResolver.disableNamespace(ns0); + + try { + FsPermission permission = new FsPermission("777"); + RouterRpcServer rpcServer = router.getRouter().getRpcServer(); + LambdaTestUtils.intercept(NoLocationException.class, + () -> rpcServer.mkdirs("/mnt/folder0/folder1", permission, true)); + } finally { + activeNamenodeResolver.clearDisableNamespaces(); + } + } + @Test public void testProxyExceptionMessages() throws IOException { @@ -1956,6 +2072,8 @@ public void testMkdirsWithCallerContext() throws IOException { final String logOutput = auditlog.getOutput(); assertTrue(logOutput.contains("callerContext=clientIp:")); assertTrue(logOutput.contains(",clientContext")); + assertTrue(logOutput.contains(",clientId")); + assertTrue(logOutput.contains(",clientCallId")); assertTrue(verifyFileExists(routerFS, dirPath)); } @@ -2006,6 +2124,41 @@ public void testAddClientIpPortToCallerContext() throws IOException { assertFalse(auditLog.getOutput().contains("clientPort:1234")); } + @Test + public void testAddClientIdAndCallIdToCallerContext() throws IOException { + GenericTestUtils.LogCapturer auditLog = + GenericTestUtils.LogCapturer.captureLogs(FSNamesystem.auditLog); + + // 1. ClientId and ClientCallId are not set on the client. + // Set client context. + CallerContext.setCurrent( + new CallerContext.Builder("clientContext").build()); + + // Create a directory via the router. + String dirPath = "/test"; + routerProtocol.mkdirs(dirPath, new FsPermission("755"), false); + + // The audit log should contains "clientId:" and "clientCallId:". + assertTrue(auditLog.getOutput().contains("clientId:")); + assertTrue(auditLog.getOutput().contains("clientCallId:")); + assertTrue(verifyFileExists(routerFS, dirPath)); + auditLog.clearOutput(); + + // 2. ClientId and ClientCallId are set on the client. + // Reset client context. + CallerContext.setCurrent( + new CallerContext.Builder( + "clientContext,clientId:mockClientId,clientCallId:4321").build()); + + // Create a directory via the router. + routerProtocol.getFileInfo(dirPath); + + // The audit log should not contain the original clientId and clientCallId + // set by client. + assertFalse(auditLog.getOutput().contains("clientId:mockClientId")); + assertFalse(auditLog.getOutput().contains("clientCallId:4321")); + } + @Test public void testContentSummaryWithSnapshot() throws Exception { DistributedFileSystem routerDFS = (DistributedFileSystem) routerFS; @@ -2047,4 +2200,34 @@ public void testContentSummaryWithSnapshot() throws Exception { routerDFS.delete(dirPath, true); } } + + @Test + public void testDisableNodeUsageInRBFMetrics() throws JSONException { + RBFMetrics rbfMetrics = router.getRouter().getMetrics(); + FederationRPCMetrics federationRPCMetrics = router.getRouter().getRpcServer().getRPCMetrics(); + + long proxyOpBefore = federationRPCMetrics.getProxyOps(); + String nodeUsageEnable = router.getRouter().getMetrics().getNodeUsage(); + assertNotNull(nodeUsageEnable); + long proxyOpAfterWithEnable = federationRPCMetrics.getProxyOps(); + assertEquals(proxyOpBefore + 2, proxyOpAfterWithEnable); + + rbfMetrics.setEnableGetDNUsage(false); + String nodeUsageDisable = rbfMetrics.getNodeUsage(); + assertNotNull(nodeUsageDisable); + long proxyOpAfterWithDisable = federationRPCMetrics.getProxyOps(); + assertEquals(proxyOpAfterWithEnable, proxyOpAfterWithDisable); + JSONObject jsonObject = new JSONObject(nodeUsageDisable); + JSONObject json = jsonObject.getJSONObject("nodeUsage"); + assertEquals("0.00%", json.get("min")); + assertEquals("0.00%", json.get("median")); + assertEquals("0.00%", json.get("max")); + assertEquals("0.00%", json.get("stdDev")); + + rbfMetrics.setEnableGetDNUsage(true); + String nodeUsageWithReEnable = rbfMetrics.getNodeUsage(); + assertNotNull(nodeUsageWithReEnable); + long proxyOpAfterWithReEnable = federationRPCMetrics.getProxyOps(); + assertEquals(proxyOpAfterWithDisable + 2, proxyOpAfterWithReEnable); + } } diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/store/TestStateStoreMembershipState.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/store/TestStateStoreMembershipState.java index 45a240b866b85..533a1d97daf7f 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/store/TestStateStoreMembershipState.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/store/TestStateStoreMembershipState.java @@ -586,8 +586,6 @@ private MembershipState getExpiredNamenodeRegistration( /** * Register a namenode heartbeat with the state store. * - * @param store FederationMembershipStateStore instance to retrieve the - * membership data records. * @param namenode A fully populated namenode membership record to be * committed to the data store. * @return True if successful, false otherwise. diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/store/TestStateStoreMountTable.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/store/TestStateStoreMountTable.java index 6e5bd9ca85ffb..d4dfcc4863fdb 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/store/TestStateStoreMountTable.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/store/TestStateStoreMountTable.java @@ -227,7 +227,6 @@ private MountTable getMountTableEntry(String mount) throws IOException { /** * Fetch all mount table records beneath a root path. * - * @param store FederationMountTableStore instance to commit the data. * @param mount The root search path, enter "/" to return all mount table * records. * diff --git a/hadoop-hdfs-project/hadoop-hdfs/dev-support/jdiff/Apache_Hadoop_HDFS_3.2.4.xml b/hadoop-hdfs-project/hadoop-hdfs/dev-support/jdiff/Apache_Hadoop_HDFS_3.2.4.xml new file mode 100644 index 0000000000000..2aa6ef1cdb5be --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs/dev-support/jdiff/Apache_Hadoop_HDFS_3.2.4.xml @@ -0,0 +1,674 @@ + + + + + + + + + + + A distributed implementation of {@link +org.apache.hadoop.fs.FileSystem}. This is loosely modelled after +Google's GFS.

    + +

    The most important difference is that unlike GFS, Hadoop DFS files +have strictly one writer at any one time. Bytes are always appended +to the end of the writer's stream. There is no notion of "record appends" +or "mutations" that are then checked or reordered. Writers simply emit +a byte stream. That byte stream is guaranteed to be stored in the +order written.

    ]]> +     +     + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + This method must return as quickly as possible, since it's called + in a critical section of the NameNode's operation. + + @param succeeded Whether authorization succeeded. + @param userName Name of the user executing the request. + @param addr Remote address of the request. + @param cmd The requested command. + @param src Path of affected source file. + @param dst Path of affected destination file (if any). + @param stat File information for operations that change the file's + metadata (permissions, owner, times, etc).]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +     diff --git a/hadoop-hdfs-project/hadoop-hdfs/dev-support/jdiff/Apache_Hadoop_HDFS_3.3.4.xml b/hadoop-hdfs-project/hadoop-hdfs/dev-support/jdiff/Apache_Hadoop_HDFS_3.3.4.xml new file mode 100644 index 0000000000000..b3978b01a6994 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs/dev-support/jdiff/Apache_Hadoop_HDFS_3.3.4.xml @@ -0,0 +1,835 @@ + + + + + + + + + + + A distributed implementation of {@link +org.apache.hadoop.fs.FileSystem}. This is loosely modelled after +Google's GFS.

    + +

    The most important difference is that unlike GFS, Hadoop DFS files +have strictly one writer at any one time. Bytes are always appended +to the end of the writer's stream. There is no notion of "record appends" +or "mutations" that are then checked or reordered. Writers simply emit +a byte stream. That byte stream is guaranteed to be stored in the +order written.

    ]]> +     +     + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + This method must return as quickly as possible, since it's called + in a critical section of the NameNode's operation. + + @param succeeded Whether authorization succeeded. + @param userName Name of the user executing the request. + @param addr Remote address of the request. + @param cmd The requested command. + @param src Path of affected source file. + @param dst Path of affected destination file (if any). + @param stat File information for operations that change the file's + metadata (permissions, owner, times, etc).]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +     diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java index 9e1333f95295b..8ff46e37aee54 100755 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java @@ -557,7 +557,9 @@ public class DFSConfigKeys extends CommonConfigurationKeys { // This value uses the times of heartbeat interval to define the minimum value for stale interval. public static final String DFS_NAMENODE_STALE_DATANODE_MINIMUM_INTERVAL_KEY = "dfs.namenode.stale.datanode.minimum.interval"; public static final int DFS_NAMENODE_STALE_DATANODE_MINIMUM_INTERVAL_DEFAULT = 3; // i.e. min_interval is 3 * heartbeat_interval = 9s - + public static final String DFS_NAMENODE_REMOVE_DEAD_DATANODE_BATCHNUM_KEY + = "dfs.namenode.remove.dead.datanode.batchnum"; + public static final int DFS_NAMENODE_REMOVE_BAD_BATCH_NUM_DEFAULT = 10; // When the percentage of stale datanodes reaches this ratio, // allow writing to stale nodes to prevent hotspots. public static final String DFS_NAMENODE_USE_STALE_DATANODE_FOR_WRITE_RATIO_KEY = "dfs.namenode.write.stale.datanode.ratio"; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolServerSideTranslatorPB.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolServerSideTranslatorPB.java index 0164f25460dc6..79c122cf5bae0 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolServerSideTranslatorPB.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolServerSideTranslatorPB.java @@ -818,7 +818,7 @@ public GetBatchedListingResponseProto getBatchedListing( public RenewLeaseResponseProto renewLease(RpcController controller, RenewLeaseRequestProto req) throws ServiceException { try { - server.renewLease(req.getClientName()); + server.renewLease(req.getClientName(), req.getNamespacesList()); return VOID_RENEWLEASE_RESPONSE; } catch (IOException e) { throw new ServiceException(e); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java index ce27342729ba3..b5f7b9c80f25f 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java @@ -1049,11 +1049,17 @@ public static BlockECReconstructionInfo convertBlockECReconstructionInfo( byte[] liveBlkIndices = blockEcReconstructionInfoProto.getLiveBlockIndices() .toByteArray(); + byte[] excludeReconstructedIndices = + blockEcReconstructionInfoProto.hasExcludeReconstructedIndices() ? + blockEcReconstructionInfoProto.getExcludeReconstructedIndices() + .toByteArray() : new byte[0]; ErasureCodingPolicy ecPolicy = PBHelperClient.convertErasureCodingPolicy( blockEcReconstructionInfoProto.getEcPolicy()); - return new BlockECReconstructionInfo(block, sourceDnInfos, targetDnInfos, - targetStorageUuids, convertStorageTypes, liveBlkIndices, ecPolicy); + return new BlockECReconstructionInfo( + block, sourceDnInfos, targetDnInfos, + targetStorageUuids, convertStorageTypes, liveBlkIndices, + excludeReconstructedIndices, ecPolicy); } public static BlockECReconstructionInfoProto convertBlockECRecoveryInfo( @@ -1079,6 +1085,10 @@ public static BlockECReconstructionInfoProto convertBlockECRecoveryInfo( byte[] liveBlockIndices = blockEcRecoveryInfo.getLiveBlockIndices(); builder.setLiveBlockIndices(PBHelperClient.getByteString(liveBlockIndices)); + byte[] excludeReconstructedIndices = blockEcRecoveryInfo.getExcludeReconstructedIndices(); + builder.setExcludeReconstructedIndices( + PBHelperClient.getByteString(excludeReconstructedIndices)); + builder.setEcPolicy(PBHelperClient.convertErasureCodingPolicy( blockEcRecoveryInfo.getErasureCodingPolicy())); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/IPCLoggerChannel.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/IPCLoggerChannel.java index 58c5ad39b99ab..4b7e59c51f13e 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/IPCLoggerChannel.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/IPCLoggerChannel.java @@ -23,7 +23,6 @@ import java.net.URI; import java.net.URL; import java.security.PrivilegedExceptionAction; -import java.util.concurrent.Callable; import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; @@ -67,6 +66,7 @@ import org.apache.hadoop.thirdparty.com.google.common.util.concurrent.MoreExecutors; import org.apache.hadoop.thirdparty.com.google.common.util.concurrent.ThreadFactoryBuilder; import org.apache.hadoop.thirdparty.com.google.common.util.concurrent.UncaughtExceptionHandlers; +import org.apache.hadoop.util.Time; /** * Channel to a remote JournalNode using Hadoop IPC. @@ -154,26 +154,15 @@ public class IPCLoggerChannel implements AsyncLogger { private static final long WARN_JOURNAL_MILLIS_THRESHOLD = 1000; - static final Factory FACTORY = new AsyncLogger.Factory() { - @Override - public AsyncLogger createLogger(Configuration conf, NamespaceInfo nsInfo, - String journalId, String nameServiceId, InetSocketAddress addr) { - return new IPCLoggerChannel(conf, nsInfo, journalId, nameServiceId, addr); - } - }; + static final Factory FACTORY = IPCLoggerChannel::new; - public IPCLoggerChannel(Configuration conf, - NamespaceInfo nsInfo, - String journalId, - InetSocketAddress addr) { + public IPCLoggerChannel(Configuration conf, NamespaceInfo nsInfo, + String journalId, InetSocketAddress addr) { this(conf, nsInfo, journalId, null, addr); } - public IPCLoggerChannel(Configuration conf, - NamespaceInfo nsInfo, - String journalId, - String nameServiceId, - InetSocketAddress addr) { + public IPCLoggerChannel(Configuration conf, NamespaceInfo nsInfo, + String journalId, String nameServiceId, InetSocketAddress addr) { this.conf = conf; this.nsInfo = nsInfo; this.journalId = journalId; @@ -202,7 +191,7 @@ public synchronized void setCommittedTxId(long txid) { "Trying to move committed txid backwards in client " + "old: %s new: %s", committedTxId, txid); this.committedTxId = txid; - this.lastCommitNanos = System.nanoTime(); + this.lastCommitNanos = Time.monotonicNowNanos(); } @Override @@ -229,25 +218,19 @@ protected QJournalProtocol createProxy() throws IOException { final Configuration confCopy = new Configuration(conf); // Need to set NODELAY or else batches larger than MTU can trigger - // 40ms nagling delays. - confCopy.setBoolean( - CommonConfigurationKeysPublic.IPC_CLIENT_TCPNODELAY_KEY, - true); - + // 40ms nailing delays. + confCopy.setBoolean(CommonConfigurationKeysPublic.IPC_CLIENT_TCPNODELAY_KEY, true); RPC.setProtocolEngine(confCopy, QJournalProtocolPB.class, ProtobufRpcEngine2.class); return SecurityUtil.doAsLoginUser( - new PrivilegedExceptionAction() { - @Override - public QJournalProtocol run() throws IOException { - RPC.setProtocolEngine(confCopy, - QJournalProtocolPB.class, ProtobufRpcEngine2.class); - QJournalProtocolPB pbproxy = RPC.getProxy( - QJournalProtocolPB.class, - RPC.getProtocolVersion(QJournalProtocolPB.class), - addr, confCopy); - return new QJournalProtocolTranslatorPB(pbproxy); - } + (PrivilegedExceptionAction) () -> { + RPC.setProtocolEngine(confCopy, + QJournalProtocolPB.class, ProtobufRpcEngine2.class); + QJournalProtocolPB pbproxy = RPC.getProxy( + QJournalProtocolPB.class, + RPC.getProtocolVersion(QJournalProtocolPB.class), + addr, confCopy); + return new QJournalProtocolTranslatorPB(pbproxy); }); } @@ -260,10 +243,8 @@ protected ExecutorService createSingleThreadExecutor() { return Executors.newSingleThreadExecutor( new ThreadFactoryBuilder() .setDaemon(true) - .setNameFormat("Logger channel (from single-thread executor) to " + - addr) - .setUncaughtExceptionHandler( - UncaughtExceptionHandlers.systemExit()) + .setNameFormat("Logger channel (from single-thread executor) to " + addr) + .setUncaughtExceptionHandler(UncaughtExceptionHandlers.systemExit()) .build()); } @@ -308,11 +289,6 @@ private synchronized RequestInfo createReqInfo() { epoch, ipcSerial++, committedTxId); } - @VisibleForTesting - synchronized long getNextIpcSerial() { - return ipcSerial; - } - public synchronized int getQueuedEditsSize() { return queuedEditsSizeBytes; } @@ -333,11 +309,7 @@ public synchronized boolean isOutOfSync() { @VisibleForTesting void waitForAllPendingCalls() throws InterruptedException { try { - singleThreadExecutor.submit(new Runnable() { - @Override - public void run() { - } - }).get(); + singleThreadExecutor.submit(() -> {}).get(); } catch (ExecutionException e) { // This can't happen! throw new AssertionError(e); @@ -346,36 +318,23 @@ public void run() { @Override public ListenableFuture isFormatted() { - return singleThreadExecutor.submit(new Callable() { - @Override - public Boolean call() throws IOException { - return getProxy().isFormatted(journalId, nameServiceId); - } - }); + return singleThreadExecutor.submit(() -> getProxy().isFormatted(journalId, nameServiceId)); } @Override public ListenableFuture getJournalState() { - return singleThreadExecutor.submit(new Callable() { - @Override - public GetJournalStateResponseProto call() throws IOException { - GetJournalStateResponseProto ret = - getProxy().getJournalState(journalId, nameServiceId); - constructHttpServerURI(ret); - return ret; - } + return singleThreadExecutor.submit(() -> { + GetJournalStateResponseProto ret = getProxy().getJournalState(journalId, nameServiceId); + constructHttpServerURI(ret); + return ret; }); } @Override public ListenableFuture newEpoch( final long epoch) { - return singleThreadExecutor.submit(new Callable() { - @Override - public NewEpochResponseProto call() throws IOException { - return getProxy().newEpoch(journalId, nameServiceId, nsInfo, epoch); - } - }); + return singleThreadExecutor.submit( + () -> getProxy().newEpoch(journalId, nameServiceId, nsInfo, epoch)); } @Override @@ -390,50 +349,43 @@ public ListenableFuture sendEdits( // When this batch is acked, we use its submission time in order // to calculate how far we are lagging. - final long submitNanos = System.nanoTime(); + final long submitNanos = Time.monotonicNowNanos(); ListenableFuture ret = null; try { - ret = singleThreadExecutor.submit(new Callable() { - @Override - public Void call() throws IOException { - throwIfOutOfSync(); - - long rpcSendTimeNanos = System.nanoTime(); - try { - getProxy().journal(createReqInfo(), - segmentTxId, firstTxnId, numTxns, data); - } catch (IOException e) { - QuorumJournalManager.LOG.warn( - "Remote journal " + IPCLoggerChannel.this + " failed to " + - "write txns " + firstTxnId + "-" + (firstTxnId + numTxns - 1) + - ". Will try to write to this JN again after the next " + - "log roll.", e); - synchronized (IPCLoggerChannel.this) { - outOfSync = true; - } - throw e; - } finally { - long now = System.nanoTime(); - long rpcTime = TimeUnit.MICROSECONDS.convert( - now - rpcSendTimeNanos, TimeUnit.NANOSECONDS); - long endToEndTime = TimeUnit.MICROSECONDS.convert( - now - submitNanos, TimeUnit.NANOSECONDS); - metrics.addWriteEndToEndLatency(endToEndTime); - metrics.addWriteRpcLatency(rpcTime); - if (rpcTime / 1000 > WARN_JOURNAL_MILLIS_THRESHOLD) { - QuorumJournalManager.LOG.warn( - "Took " + (rpcTime / 1000) + "ms to send a batch of " + - numTxns + " edits (" + data.length + " bytes) to " + - "remote journal " + IPCLoggerChannel.this); - } - } + ret = singleThreadExecutor.submit(() -> { + throwIfOutOfSync(); + + final long rpcSendTimeNanos = Time.monotonicNowNanos(); + try { + getProxy().journal(createReqInfo(), segmentTxId, firstTxnId, numTxns, data); + } catch (IOException e) { + QuorumJournalManager.LOG.warn("Remote journal {} failed to write txns {}-{}." + + " Will try to write to this JN again after the next log roll.", + IPCLoggerChannel.this, firstTxnId, (firstTxnId + numTxns - 1), e); synchronized (IPCLoggerChannel.this) { - highestAckedTxId = firstTxnId + numTxns - 1; - lastAckNanos = submitNanos; + outOfSync = true; } - return null; + throw e; + } finally { + final long nowNanos = Time.monotonicNowNanos(); + final long rpcTimeMicros = TimeUnit.MICROSECONDS.convert( + (nowNanos - rpcSendTimeNanos), TimeUnit.NANOSECONDS); + final long endToEndTimeMicros = TimeUnit.MICROSECONDS.convert( + (nowNanos - submitNanos), TimeUnit.NANOSECONDS); + metrics.addWriteEndToEndLatency(endToEndTimeMicros); + metrics.addWriteRpcLatency(rpcTimeMicros); + if (rpcTimeMicros / 1000 > WARN_JOURNAL_MILLIS_THRESHOLD) { + QuorumJournalManager.LOG.warn( + "Took {}ms to send a batch of {} edits ({} bytes) to remote journal {}.", + rpcTimeMicros / 1000, numTxns, data.length, IPCLoggerChannel.this); + } + } + synchronized (IPCLoggerChannel.this) { + highestAckedTxId = firstTxnId + numTxns - 1; + lastAckNanos = submitNanos; } + return null; }); } finally { if (ret == null) { @@ -460,14 +412,12 @@ public void onSuccess(Void t) { return ret; } - private void throwIfOutOfSync() - throws JournalOutOfSyncException, IOException { + private void throwIfOutOfSync() throws IOException { if (isOutOfSync()) { // Even if we're out of sync, it's useful to send an RPC // to the remote node in order to update its lag metrics, etc. heartbeatIfNecessary(); - throw new JournalOutOfSyncException( - "Journal disabled until next roll"); + throw new JournalOutOfSyncException("Journal disabled until next roll"); } } @@ -497,12 +447,10 @@ private void heartbeatIfNecessary() throws IOException { private synchronized void reserveQueueSpace(int size) throws LoggerTooFarBehindException { Preconditions.checkArgument(size >= 0); - if (queuedEditsSizeBytes + size > queueSizeLimitBytes && - queuedEditsSizeBytes > 0) { - QuorumJournalManager.LOG.warn("Pending edits to " + IPCLoggerChannel.this - + " is going to exceed limit size: " + queueSizeLimitBytes - + ", current queued edits size: " + queuedEditsSizeBytes - + ", will silently drop " + size + " bytes of edits!"); + if (queuedEditsSizeBytes + size > queueSizeLimitBytes && queuedEditsSizeBytes > 0) { + QuorumJournalManager.LOG.warn("Pending edits to {} is going to exceed limit size: {}" + + ", current queued edits size: {}, will silently drop {} bytes of edits!", + IPCLoggerChannel.class, queueSizeLimitBytes, queuedEditsSizeBytes, size); throw new LoggerTooFarBehindException(); } queuedEditsSizeBytes += size; @@ -514,203 +462,144 @@ private synchronized void unreserveQueueSpace(int size) { } @Override - public ListenableFuture format(final NamespaceInfo nsInfo, - final boolean force) { - return singleThreadExecutor.submit(new Callable() { - @Override - public Void call() throws Exception { - getProxy().format(journalId, nameServiceId, nsInfo, force); - return null; - } + public ListenableFuture format(final NamespaceInfo nsInfo, final boolean force) { + return singleThreadExecutor.submit(() -> { + getProxy().format(journalId, nameServiceId, nsInfo, force); + return null; }); } @Override - public ListenableFuture startLogSegment(final long txid, - final int layoutVersion) { - return singleThreadExecutor.submit(new Callable() { - @Override - public Void call() throws IOException { - getProxy().startLogSegment(createReqInfo(), txid, layoutVersion); - synchronized (IPCLoggerChannel.this) { - if (outOfSync) { - outOfSync = false; - QuorumJournalManager.LOG.info( - "Restarting previously-stopped writes to " + - IPCLoggerChannel.this + " in segment starting at txid " + - txid); - } + public ListenableFuture startLogSegment(final long txid, final int layoutVersion) { + return singleThreadExecutor.submit(() -> { + getProxy().startLogSegment(createReqInfo(), txid, layoutVersion); + synchronized (IPCLoggerChannel.this) { + if (outOfSync) { + outOfSync = false; + QuorumJournalManager.LOG.info( + "Restarting previously-stopped writes to {} in segment starting at txid {}.", + IPCLoggerChannel.class, txid); } - return null; } + return null; }); } @Override - public ListenableFuture finalizeLogSegment( - final long startTxId, final long endTxId) { - return singleThreadExecutor.submit(new Callable() { - @Override - public Void call() throws IOException { - throwIfOutOfSync(); - - getProxy().finalizeLogSegment(createReqInfo(), startTxId, endTxId); - return null; - } + public ListenableFuture finalizeLogSegment(final long startTxId, final long endTxId) { + return singleThreadExecutor.submit(() -> { + throwIfOutOfSync(); + getProxy().finalizeLogSegment(createReqInfo(), startTxId, endTxId); + return null; }); } @Override public ListenableFuture purgeLogsOlderThan(final long minTxIdToKeep) { - return singleThreadExecutor.submit(new Callable() { - @Override - public Void call() throws Exception { - getProxy().purgeLogsOlderThan(createReqInfo(), minTxIdToKeep); - return null; - } + return singleThreadExecutor.submit(() -> { + getProxy().purgeLogsOlderThan(createReqInfo(), minTxIdToKeep); + return null; }); } @Override public ListenableFuture getJournaledEdits( long fromTxnId, int maxTransactions) { - return parallelExecutor.submit( - new Callable() { - @Override - public GetJournaledEditsResponseProto call() throws IOException { - return getProxy().getJournaledEdits(journalId, nameServiceId, - fromTxnId, maxTransactions); - } - }); + return parallelExecutor.submit(() -> getProxy().getJournaledEdits( + journalId, nameServiceId, fromTxnId, maxTransactions)); } @Override public ListenableFuture getEditLogManifest( final long fromTxnId, final boolean inProgressOk) { - return parallelExecutor.submit(new Callable() { - @Override - public RemoteEditLogManifest call() throws IOException { - GetEditLogManifestResponseProto ret = getProxy().getEditLogManifest( - journalId, nameServiceId, fromTxnId, inProgressOk); - // Update the http port, since we need this to build URLs to any of the - // returned logs. - constructHttpServerURI(ret); - return PBHelper.convert(ret.getManifest()); - } + return parallelExecutor.submit(() -> { + GetEditLogManifestResponseProto ret = getProxy().getEditLogManifest( + journalId, nameServiceId, fromTxnId, inProgressOk); + // Update the http port, since we need this to build URLs to any of the + // returned logs. + constructHttpServerURI(ret); + return PBHelper.convert(ret.getManifest()); }); } @Override - public ListenableFuture prepareRecovery( - final long segmentTxId) { - return singleThreadExecutor.submit(new Callable() { - @Override - public PrepareRecoveryResponseProto call() throws IOException { - if (!hasHttpServerEndPoint()) { - // force an RPC call so we know what the HTTP port should be if it - // haven't done so. - GetJournalStateResponseProto ret = getProxy().getJournalState( - journalId, nameServiceId); - constructHttpServerURI(ret); - } - return getProxy().prepareRecovery(createReqInfo(), segmentTxId); + public ListenableFuture prepareRecovery(final long segmentTxId) { + return singleThreadExecutor.submit(() -> { + if (!hasHttpServerEndPoint()) { + // force an RPC call, so we know what the HTTP port should be if it + // hasn't done so. + GetJournalStateResponseProto ret = getProxy().getJournalState( + journalId, nameServiceId); + constructHttpServerURI(ret); } + return getProxy().prepareRecovery(createReqInfo(), segmentTxId); }); } @Override - public ListenableFuture acceptRecovery( - final SegmentStateProto log, final URL url) { - return singleThreadExecutor.submit(new Callable() { - @Override - public Void call() throws IOException { - getProxy().acceptRecovery(createReqInfo(), log, url); - return null; - } + public ListenableFuture acceptRecovery(final SegmentStateProto log, final URL url) { + return singleThreadExecutor.submit(() -> { + getProxy().acceptRecovery(createReqInfo(), log, url); + return null; }); } @Override public ListenableFuture doPreUpgrade() { - return singleThreadExecutor.submit(new Callable() { - @Override - public Void call() throws IOException { - getProxy().doPreUpgrade(journalId); - return null; - } + return singleThreadExecutor.submit(() -> { + getProxy().doPreUpgrade(journalId); + return null; }); } @Override public ListenableFuture doUpgrade(final StorageInfo sInfo) { - return singleThreadExecutor.submit(new Callable() { - @Override - public Void call() throws IOException { - getProxy().doUpgrade(journalId, sInfo); - return null; - } + return singleThreadExecutor.submit(() -> { + getProxy().doUpgrade(journalId, sInfo); + return null; }); } @Override public ListenableFuture doFinalize() { - return singleThreadExecutor.submit(new Callable() { - @Override - public Void call() throws IOException { - getProxy().doFinalize(journalId, nameServiceId); - return null; - } + return singleThreadExecutor.submit(() -> { + getProxy().doFinalize(journalId, nameServiceId); + return null; }); } @Override public ListenableFuture canRollBack(final StorageInfo storage, final StorageInfo prevStorage, final int targetLayoutVersion) { - return singleThreadExecutor.submit(new Callable() { - @Override - public Boolean call() throws IOException { - return getProxy().canRollBack(journalId, nameServiceId, - storage, prevStorage, targetLayoutVersion); - } - }); + return singleThreadExecutor.submit( + () -> getProxy().canRollBack(journalId, nameServiceId, + storage, prevStorage, targetLayoutVersion)); } @Override public ListenableFuture doRollback() { - return singleThreadExecutor.submit(new Callable() { - @Override - public Void call() throws IOException { - getProxy().doRollback(journalId, nameServiceId); - return null; - } + return singleThreadExecutor.submit(() -> { + getProxy().doRollback(journalId, nameServiceId); + return null; }); } @Override public ListenableFuture discardSegments(final long startTxId) { - return singleThreadExecutor.submit(new Callable() { - @Override - public Void call() throws IOException { - getProxy().discardSegments(journalId, nameServiceId, startTxId); - return null; - } + return singleThreadExecutor.submit(() -> { + getProxy().discardSegments(journalId, nameServiceId, startTxId); + return null; }); } @Override public ListenableFuture getJournalCTime() { - return singleThreadExecutor.submit(new Callable() { - @Override - public Long call() throws IOException { - return getProxy().getJournalCTime(journalId, nameServiceId); - } - }); + return singleThreadExecutor.submit(() -> getProxy().getJournalCTime(journalId, nameServiceId)); } @Override public String toString() { - return InetAddresses.toAddrString(addr.getAddress()) + ':' + - addr.getPort(); + return InetAddresses.toAddrString(addr.getAddress()) + ':' + addr.getPort(); } @Override @@ -778,5 +667,4 @@ private URL getHttpServerURI(String scheme, int port) { private boolean hasHttpServerEndPoint() { return httpServerURL != null; } - } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/QuorumJournalManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/QuorumJournalManager.java index e25485e6cd78e..faf71a7b545d9 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/QuorumJournalManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/QuorumJournalManager.java @@ -479,10 +479,9 @@ public void recoverUnfinalizedSegments() throws IOException { LOG.info("Successfully started new epoch " + loggers.getEpoch()); if (LOG.isDebugEnabled()) { - LOG.debug("newEpoch(" + loggers.getEpoch() + ") responses:\n" + - QuorumCall.mapToString(resps)); + LOG.debug("newEpoch({}) responses:\n{}", loggers.getEpoch(), QuorumCall.mapToString(resps)); } - + long mostRecentSegmentTxId = Long.MIN_VALUE; for (NewEpochResponseProto r : resps.values()) { if (r.hasLastSegmentTxId()) { @@ -518,10 +517,7 @@ public void selectInputStreams(Collection streams, // the cache used for RPC calls is not enabled; fall back to using the // streaming mechanism to serve such requests if (inProgressOk && inProgressTailingEnabled) { - if (LOG.isDebugEnabled()) { - LOG.debug("Tailing edits starting from txn ID " + fromTxnId + - " via RPC mechanism"); - } + LOG.debug("Tailing edits starting from txn ID {} via RPC mechanism", fromTxnId); try { Collection rpcStreams = new ArrayList<>(); selectRpcInputStreams(rpcStreams, fromTxnId, onlyDurableTxns); @@ -585,8 +581,8 @@ private void selectRpcInputStreams(Collection streams, int maxAllowedTxns = !onlyDurableTxns ? highestTxnCount : responseCounts.get(responseCounts.size() - loggers.getMajoritySize()); if (maxAllowedTxns == 0) { - LOG.debug("No new edits available in logs; requested starting from " + - "ID {}", fromTxnId); + LOG.debug("No new edits available in logs; requested starting from ID {}", + fromTxnId); return; } LogAction logAction = selectInputStreamLogHelper.record(fromTxnId); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/GetJournalEditServlet.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/GetJournalEditServlet.java index 81b3f8c1a1f1f..f726ff8f84de6 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/GetJournalEditServlet.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/GetJournalEditServlet.java @@ -27,11 +27,11 @@ import javax.servlet.ServletContext; import javax.servlet.ServletException; -import javax.servlet.http.HttpServlet; import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; import org.apache.commons.text.StringEscapeUtils; +import org.apache.hadoop.hdfs.server.namenode.DfsServlet; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.hadoop.classification.InterfaceAudience; @@ -64,7 +64,7 @@ * */ @InterfaceAudience.Private -public class GetJournalEditServlet extends HttpServlet { +public class GetJournalEditServlet extends DfsServlet { private static final long serialVersionUID = -4635891628211723009L; private static final Logger LOG = @@ -77,17 +77,11 @@ public class GetJournalEditServlet extends HttpServlet { protected boolean isValidRequestor(HttpServletRequest request, Configuration conf) throws IOException { - String remotePrincipal = request.getUserPrincipal().getName(); - String remoteShortName = request.getRemoteUser(); - if (remotePrincipal == null) { // This really shouldn't happen... - LOG.warn("Received null remoteUser while authorizing access to " + - "GetJournalEditServlet"); - return false; - } + UserGroupInformation ugi = getUGI(request, conf); if (LOG.isDebugEnabled()) { - LOG.debug("Validating request made by " + remotePrincipal + - " / " + remoteShortName + ". This user is: " + + LOG.debug("Validating request made by " + ugi.getUserName() + + " / " + ugi.getShortUserName() + ". This user is: " + UserGroupInformation.getLoginUser()); } @@ -115,9 +109,9 @@ protected boolean isValidRequestor(HttpServletRequest request, Configuration con for (String v : validRequestors) { if (LOG.isDebugEnabled()) LOG.debug("isValidRequestor is comparing to valid requestor: " + v); - if (v != null && v.equals(remotePrincipal)) { + if (v != null && v.equals(ugi.getUserName())) { if (LOG.isDebugEnabled()) - LOG.debug("isValidRequestor is allowing: " + remotePrincipal); + LOG.debug("isValidRequestor is allowing: " + ugi.getUserName()); return true; } } @@ -125,16 +119,16 @@ protected boolean isValidRequestor(HttpServletRequest request, Configuration con // Additionally, we compare the short name of the requestor to this JN's // username, because we want to allow requests from other JNs during // recovery, but we can't enumerate the full list of JNs. - if (remoteShortName.equals( + if (ugi.getShortUserName().equals( UserGroupInformation.getLoginUser().getShortUserName())) { if (LOG.isDebugEnabled()) LOG.debug("isValidRequestor is allowing other JN principal: " + - remotePrincipal); + ugi.getUserName()); return true; } if (LOG.isDebugEnabled()) - LOG.debug("isValidRequestor is rejecting: " + remotePrincipal); + LOG.debug("isValidRequestor is rejecting: " + ugi.getUserName()); return false; } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/Journal.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/Journal.java index 6b9b40871816b..ffa613018c6a6 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/Journal.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/Journal.java @@ -750,10 +750,23 @@ public GetJournaledEditsResponseProto getJournaledEdits(long sinceTxId, "is a requirement to fetch journaled edits via RPC. Please enable " + "it via " + DFSConfigKeys.DFS_HA_TAILEDITS_INPROGRESS_KEY); } - if (sinceTxId > getHighestWrittenTxId()) { - // Requested edits that don't exist yet; short-circuit the cache here + long highestTxId = getHighestWrittenTxId(); + if (sinceTxId == highestTxId + 1) { + // Requested edits that don't exist yet, but this is expected, + // because namenode always get the journaled edits with the sinceTxId + // equal to image.getLastAppliedTxId() + 1. Short-circuiting the cache here + // and returning a response with a count of 0. metrics.rpcEmptyResponses.incr(); return GetJournaledEditsResponseProto.newBuilder().setTxnCount(0).build(); + } else if (sinceTxId > highestTxId + 1) { + // Requested edits that don't exist yet and this is unexpected. Means that there is a lag + // in this journal that does not contain some edits that should exist. + // Throw one NewerTxnIdException to make namenode treat this response as an exception. + // More detailed info please refer to: HDFS-16659 and HDFS-16771. + metrics.rpcEmptyResponses.incr(); + throw new NewerTxnIdException( + "Highest txn ID available in the journal is %d, but requested txns starting at %d.", + highestTxId, sinceTxId); } try { List buffers = new ArrayList<>(); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/JournalNode.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/JournalNode.java index df1314ac1eb56..3e8831d8087c7 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/JournalNode.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/JournalNode.java @@ -124,6 +124,11 @@ synchronized Journal getOrCreateJournal(String jid, return journal; } + @VisibleForTesting + public JournalNodeSyncer getJournalSyncer(String jid) { + return journalSyncersById.get(jid); + } + @VisibleForTesting public boolean getJournalSyncerStatus(String jid) { if (journalSyncersById.get(jid) != null) { diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/JournalNodeRpcServer.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/JournalNodeRpcServer.java index ad67cf481ae70..ab909aef2ecd8 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/JournalNodeRpcServer.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/JournalNodeRpcServer.java @@ -114,6 +114,8 @@ public class JournalNodeRpcServer implements QJournalProtocol, .setVerbose(false) .build(); + this.server.addTerseExceptions(NewerTxnIdException.class); + this.server.addTerseExceptions(JournaledEditsCache.CacheMissException.class); //Adding InterQJournalProtocolPB to server InterQJournalProtocolServerSideTranslatorPB diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/JournalNodeSyncer.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/JournalNodeSyncer.java index fd29c849dfcb3..f451b46de7b37 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/JournalNodeSyncer.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/JournalNodeSyncer.java @@ -17,6 +17,7 @@ */ package org.apache.hadoop.hdfs.qjournal.server; +import org.apache.hadoop.classification.VisibleForTesting; import org.apache.hadoop.thirdparty.com.google.common.collect.ImmutableList; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.conf.Configuration; @@ -39,6 +40,7 @@ import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.util.Daemon; import org.apache.hadoop.util.Lists; +import org.apache.hadoop.util.Sets; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -50,10 +52,10 @@ import java.net.URISyntaxException; import java.net.URL; import java.security.PrivilegedExceptionAction; -import java.util.Arrays; import java.util.Collection; import java.util.HashSet; import java.util.List; +import java.util.Set; /** * A Journal Sync thread runs through the lifetime of the JN. It periodically @@ -153,6 +155,9 @@ private boolean getOtherJournalNodeProxies() { LOG.warn("Could not add proxy for Journal at addresss " + addr, e); } } + // Check if there are any other JournalNodes before starting the sync. Although some proxies + // may be unresolved now, the act of attempting to sync will instigate resolution when the + // servers become available. if (otherJNProxies.isEmpty()) { LOG.error("Cannot sync as there is no other JN available for sync."); return false; @@ -310,12 +315,24 @@ private List getOtherJournalNodeAddrs() { return null; } - private List getJournalAddrList(String uriStr) throws + @VisibleForTesting + protected List getJournalAddrList(String uriStr) throws URISyntaxException, IOException { URI uri = new URI(uriStr); - return Util.getLoggerAddresses(uri, - new HashSet<>(Arrays.asList(jn.getBoundIpcAddress())), conf); + + InetSocketAddress boundIpcAddress = jn.getBoundIpcAddress(); + Set excluded = Sets.newHashSet(boundIpcAddress); + List addrList = Util.getLoggerAddresses(uri, excluded, conf); + + // Exclude the current JournalNode instance (a local address and the same port). If the address + // is bound to a local address on the same port, then remove it to handle scenarios where a + // wildcard address (e.g. "0.0.0.0") is used. We can't simply exclude all local addresses + // since we may be running multiple servers on the same host. + addrList.removeIf(addr -> !addr.isUnresolved() && addr.getAddress().isAnyLocalAddress() + && boundIpcAddress.getPort() == addr.getPort()); + + return addrList; } private void getMissingLogSegments(List thisJournalEditLogs, diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/NewerTxnIdException.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/NewerTxnIdException.java new file mode 100644 index 0000000000000..ec691402719d7 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/NewerTxnIdException.java @@ -0,0 +1,31 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hdfs.qjournal.server; + +import java.io.IOException; + +/** + * Exception when no edits are available. + */ +public class NewerTxnIdException extends IOException { + private static final long serialVersionUID = 0L; + + public NewerTxnIdException(String msgFormat, Object... msgArgs) { + super(String.format(msgFormat, msgArgs)); + } +} diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java index 51e12ec43372c..dfe48f7bde1f4 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java @@ -949,7 +949,7 @@ private void dumpBlockMeta(Block block, PrintWriter out) { // source node returned is not used chooseSourceDatanodes(blockInfo, containingNodes, containingLiveReplicasNodes, numReplicas, new ArrayList(), - new ArrayList(), LowRedundancyBlocks.LEVEL); + new ArrayList(), new ArrayList(), LowRedundancyBlocks.LEVEL); // containingLiveReplicasNodes can include READ_ONLY_SHARED replicas which are // not included in the numReplicas.liveReplicas() count @@ -1067,6 +1067,26 @@ public void setBlocksReplWorkMultiplier(int newVal) { blocksReplWorkMultiplier = newVal; } + /** + * Updates the value used for pendingReconstruction timeout, which is set by + * {@code DFSConfigKeys. + * DFS_NAMENODE_RECONSTRUCTION_PENDING_TIMEOUT_SEC_KEY} initially. + * + * @param newVal - Must be a positive non-zero integer. + */ + public void setReconstructionPendingTimeout(int newVal) { + ensurePositiveInt(newVal, + DFSConfigKeys.DFS_NAMENODE_RECONSTRUCTION_PENDING_TIMEOUT_SEC_KEY); + pendingReconstruction.setTimeout(newVal * 1000L); + } + + /** Returns the current setting for pendingReconstruction timeout, set by + * {@code DFSConfigKeys.DFS_NAMENODE_RECONSTRUCTION_PENDING_TIMEOUT_SEC_KEY}. + */ + public int getReconstructionPendingTimeout() { + return (int)(pendingReconstruction.getTimeout() / 1000L); + } + public int getDefaultStorageNum(BlockInfo block) { switch (block.getBlockType()) { case STRIPED: return ((BlockInfoStriped) block).getRealTotalBlockNum(); @@ -1541,6 +1561,7 @@ public LocatedBlocks createLocatedBlocks(final BlockInfo[] blocks, if (LOG.isDebugEnabled()) { LOG.debug("blocks = {}", java.util.Arrays.asList(blocks)); } + final AccessMode mode = needBlockToken? BlockTokenIdentifier.AccessMode.READ: null; LocatedBlockBuilder locatedBlocks = providedStorageMap @@ -1873,8 +1894,7 @@ public void findAndMarkBlockAsCorrupt(final ExtendedBlock blk, } if (storage == null) { - blockLog.debug("BLOCK* findAndMarkBlockAsCorrupt: {} not found on {}", - blk, dn); + blockLog.debug("BLOCK* findAndMarkBlockAsCorrupt: {} not found on {}", blk, dn); return; } markBlockAsCorrupt(new BlockToMarkCorrupt(reportedBlock, storedBlock, @@ -1893,8 +1913,10 @@ private void markBlockAsCorrupt(BlockToMarkCorrupt b, DatanodeStorageInfo storageInfo, DatanodeDescriptor node) throws IOException { if (b.getStored().isDeleted()) { - blockLog.debug("BLOCK markBlockAsCorrupt: {} cannot be marked as" + - " corrupt as it does not belong to any file", b); + if (blockLog.isDebugEnabled()) { + blockLog.debug("BLOCK markBlockAsCorrupt: {} cannot be marked as" + + " corrupt as it does not belong to any file", b); + } addToInvalidates(b.getCorrupted(), node); return; } @@ -1975,10 +1997,12 @@ private boolean invalidateBlock(BlockToMarkCorrupt b, DatanodeInfo dn, // Check how many copies we have of the block if (nr.replicasOnStaleNodes() > 0 && !deleteCorruptReplicaImmediately) { - blockLog.debug("BLOCK* invalidateBlocks: postponing " + - "invalidation of {} on {} because {} replica(s) are located on " + - "nodes with potentially out-of-date block reports", b, dn, - nr.replicasOnStaleNodes()); + if (blockLog.isDebugEnabled()) { + blockLog.debug("BLOCK* invalidateBlocks: postponing " + + "invalidation of {} on {} because {} replica(s) are located on " + + "nodes with potentially out-of-date block reports", b, dn, + nr.replicasOnStaleNodes()); + } postponeBlock(b.getCorrupted()); return false; } else { @@ -1986,8 +2010,7 @@ private boolean invalidateBlock(BlockToMarkCorrupt b, DatanodeInfo dn, // function and know there are enough live replicas, so we can delete it. addToInvalidates(b.getCorrupted(), dn); removeStoredBlock(b.getStored(), node); - blockLog.debug("BLOCK* invalidateBlocks: {} on {} listed for deletion.", - b, dn); + blockLog.debug("BLOCK* invalidateBlocks: {} on {} listed for deletion.", b, dn); return true; } } @@ -2152,13 +2175,11 @@ int computeReconstructionWorkForBlocks( for (DatanodeStorageInfo target : targets) { targetList.append(' ').append(target.getDatanodeDescriptor()); } - blockLog.debug("BLOCK* ask {} to replicate {} to {}", rw.getSrcNodes(), - rw.getBlock(), targetList); + blockLog.debug("BLOCK* ask {} to replicate {} to {}", + rw.getSrcNodes(), rw.getBlock(), targetList); } } - - blockLog.debug( - "BLOCK* neededReconstruction = {} pendingReconstruction = {}", + blockLog.debug("BLOCK* neededReconstruction = {} pendingReconstruction = {}", neededReconstruction.size(), pendingReconstruction.size()); } @@ -2191,12 +2212,13 @@ BlockReconstructionWork scheduleReconstruction(BlockInfo block, NumberReplicas numReplicas = new NumberReplicas(); List liveBlockIndices = new ArrayList<>(); List liveBusyBlockIndices = new ArrayList<>(); + List excludeReconstructed = new ArrayList<>(); final DatanodeDescriptor[] srcNodes = chooseSourceDatanodes(block, containingNodes, liveReplicaNodes, numReplicas, - liveBlockIndices, liveBusyBlockIndices, priority); + liveBlockIndices, liveBusyBlockIndices, excludeReconstructed, priority); short requiredRedundancy = getExpectedLiveRedundancyNum(block, numReplicas); - if(srcNodes == null || srcNodes.length == 0) { + if (srcNodes == null || srcNodes.length == 0) { // block can not be reconstructed from any node LOG.debug("Block {} cannot be reconstructed from any node", block); NameNode.getNameNodeMetrics().incNumTimesReReplicationNotScheduled(); @@ -2220,8 +2242,8 @@ BlockReconstructionWork scheduleReconstruction(BlockInfo block, int pendingNum = pendingReconstruction.getNumReplicas(block); if (hasEnoughEffectiveReplicas(block, numReplicas, pendingNum)) { neededReconstruction.remove(block, priority); - blockLog.debug("BLOCK* Removing {} from neededReconstruction as" + - " it has enough replicas", block); + blockLog.debug("BLOCK* Removing {} from neededReconstruction as it has enough replicas", + block); NameNode.getNameNodeMetrics().incNumTimesReReplicationNotScheduled(); return null; } @@ -2261,9 +2283,13 @@ BlockReconstructionWork scheduleReconstruction(BlockInfo block, for (int i = 0; i < liveBusyBlockIndices.size(); i++) { busyIndices[i] = liveBusyBlockIndices.get(i); } + byte[] excludeReconstructedIndices = new byte[excludeReconstructed.size()]; + for (int i = 0; i < excludeReconstructed.size(); i++) { + excludeReconstructedIndices[i] = excludeReconstructed.get(i); + } return new ErasureCodingWork(getBlockPoolId(), block, bc, newSrcNodes, containingNodes, liveReplicaNodes, additionalReplRequired, - priority, newIndices, busyIndices); + priority, newIndices, busyIndices, excludeReconstructedIndices); } else { return new ReplicationWork(block, bc, srcNodes, containingNodes, liveReplicaNodes, additionalReplRequired, @@ -2317,8 +2343,8 @@ boolean validateReconstructionWork(BlockReconstructionWork rw) { if (hasEnoughEffectiveReplicas(block, numReplicas, pendingNum)) { neededReconstruction.remove(block, priority); rw.resetTargets(); - blockLog.debug("BLOCK* Removing {} from neededReconstruction as" + - " it has enough replicas", block); + blockLog.debug("BLOCK* Removing {} from neededReconstruction as it has enough replicas", + block); return false; } @@ -2349,8 +2375,8 @@ boolean validateReconstructionWork(BlockReconstructionWork rw) { // The reason we use 'pending' is so we can retry // reconstructions that fail after an appropriate amount of time. pendingReconstruction.increment(block, targets); - blockLog.debug("BLOCK* block {} is moved from neededReconstruction to " - + "pendingReconstruction", block); + blockLog.debug("BLOCK* block {} is moved from neededReconstruction to pendingReconstruction", + block); int numEffectiveReplicas = numReplicas.liveReplicas() + pendingNum; // remove from neededReconstruction @@ -2507,7 +2533,7 @@ DatanodeDescriptor[] chooseSourceDatanodes(BlockInfo block, List containingNodes, List nodesContainingLiveReplicas, NumberReplicas numReplicas, List liveBlockIndices, - List liveBusyBlockIndices, int priority) { + List liveBusyBlockIndices, List excludeReconstructed, int priority) { containingNodes.clear(); nodesContainingLiveReplicas.clear(); List srcNodes = new ArrayList<>(); @@ -2577,6 +2603,8 @@ DatanodeDescriptor[] chooseSourceDatanodes(BlockInfo block, if (isStriped && (state == StoredReplicaState.LIVE || state == StoredReplicaState.DECOMMISSIONING)) { liveBusyBlockIndices.add(blockIndex); + //HDFS-16566 ExcludeReconstructed won't be reconstructed. + excludeReconstructed.add(blockIndex); } continue; // already reached replication limit } @@ -2585,6 +2613,8 @@ DatanodeDescriptor[] chooseSourceDatanodes(BlockInfo block, if (isStriped && (state == StoredReplicaState.LIVE || state == StoredReplicaState.DECOMMISSIONING)) { liveBusyBlockIndices.add(blockIndex); + //HDFS-16566 ExcludeReconstructed won't be reconstructed. + excludeReconstructed.add(blockIndex); } continue; } @@ -2739,9 +2769,11 @@ public void removeBlocksAndUpdateSafemodeTotal(BlocksMapUpdateInfo blocks) { removeBlock(b); } if (trackBlockCounts) { - LOG.debug("Adjusting safe-mode totals for deletion." - + "decreasing safeBlocks by {}, totalBlocks by {}", - numRemovedSafe, numRemovedComplete); + if (LOG.isDebugEnabled()) { + LOG.debug("Adjusting safe-mode totals for deletion." + + "decreasing safeBlocks by {}, totalBlocks by {}", + numRemovedSafe, numRemovedComplete); + } bmSafeMode.adjustBlockTotals(-numRemovedSafe, -numRemovedComplete); } } @@ -2894,11 +2926,11 @@ public boolean processReport(final DatanodeID nodeID, namesystem.writeUnlock("processReport"); } - if(blockLog.isDebugEnabled()) { + if (blockLog.isDebugEnabled()) { for (Block b : invalidatedBlocks) { blockLog.debug("BLOCK* processReport 0x{} with lease ID 0x{}: {} on node {} size {} " + - "does not belong to any file.", strBlockReportId, fullBrLeaseId, b, - node, b.getNumBytes()); + "does not belong to any file.", strBlockReportId, fullBrLeaseId, b, + node, b.getNumBytes()); } } @@ -2930,9 +2962,10 @@ public void removeBRLeaseIfNeeded(final DatanodeID nodeID, node.setLastBlockReportTime(now()); node.setLastBlockReportMonotonic(Time.monotonicNow()); } - LOG.debug("Processing RPC with index {} out of total {} RPCs in " - + "processReport 0x{}", context.getCurRpc(), - context.getTotalRpcs(), Long.toHexString(context.getReportId())); + if (LOG.isDebugEnabled()) { + LOG.debug("Processing RPC with index {} out of total {} RPCs in processReport 0x{}", + context.getCurRpc(), context.getTotalRpcs(), Long.toHexString(context.getReportId())); + } } } finally { namesystem.writeUnlock("removeBRLeaseIfNeeded"); @@ -2957,14 +2990,16 @@ void rescanPostponedMisreplicatedBlocks() { BlockInfo bi = getStoredBlock(b); if (bi == null) { - LOG.debug("BLOCK* rescanPostponedMisreplicatedBlocks: " + - "Postponed mis-replicated block {} no longer found " + - "in block map.", b); + if (LOG.isDebugEnabled()) { + LOG.debug("BLOCK* rescanPostponedMisreplicatedBlocks: " + + "Postponed mis-replicated block {} no longer found " + + "in block map.", b); + } continue; } MisReplicationResult res = processMisReplicatedBlock(bi); - LOG.debug("BLOCK* rescanPostponedMisreplicatedBlocks: " + - "Re-scanned block {}, result is {}", b, res); + LOG.debug("BLOCK* rescanPostponedMisreplicatedBlocks: Re-scanned block {}, result is {}", + b, res); if (res == MisReplicationResult.POSTPONE) { rescannedMisreplicatedBlocks.add(b); } @@ -3056,9 +3091,11 @@ public void markBlockReplicasAsCorrupt(Block oldBlock, } } if (isCorrupt) { - blockLog.debug("BLOCK* markBlockReplicasAsCorrupt: mark block replica" + - " {} on {} as corrupt because the dn is not in the new committed " + - "storage list.", b, storage.getDatanodeDescriptor()); + if (blockLog.isDebugEnabled()) { + blockLog.debug("BLOCK* markBlockReplicasAsCorrupt: mark block replica" + + " {} on {} as corrupt because the dn is not in the new committed " + + "storage list.", b, storage.getDatanodeDescriptor()); + } markBlockAsCorrupt(b, storage, storage.getDatanodeDescriptor()); } } @@ -3090,6 +3127,7 @@ void processFirstBlockReport( iblk.getBlockName(), storageInfo.getDatanodeDescriptor(), iblk.getNumBytes(), reportedState); } + if (shouldPostponeBlocksFromFuture && isGenStampInFuture(iblk)) { queueReportedBlock(storageInfo, iblk, reportedState, QUEUE_REASON_FUTURE_GENSTAMP); @@ -3306,9 +3344,11 @@ private void queueReportedBlock(DatanodeStorageInfo storageInfo, Block block, ReplicaState reportedState, String reason) { assert shouldPostponeBlocksFromFuture; - LOG.debug("Queueing reported block {} in state {}" + - " from datanode {} for later processing because {}.", - block, reportedState, storageInfo.getDatanodeDescriptor(), reason); + if (LOG.isDebugEnabled()) { + LOG.debug("Queueing reported block {} in state {}" + + " from datanode {} for later processing because {}.", + block, reportedState, storageInfo.getDatanodeDescriptor(), reason); + } pendingDNMessages.enqueueReportedBlock(storageInfo, block, reportedState); } @@ -3575,9 +3615,8 @@ private Block addStoredBlock(final BlockInfo block, } if (storedBlock == null || storedBlock.isDeleted()) { // If this block does not belong to anyfile, then we are done. - blockLog.debug("BLOCK* addStoredBlock: {} on {} size {} but it does not" + - " belong to any file", block, node, block.getNumBytes()); - + blockLog.debug("BLOCK* addStoredBlock: {} on {} size {} but it does not belong to any file", + block, node, block.getNumBytes()); // we could add this block to invalidate set of this datanode. // it will happen in next block report otherwise. return block; @@ -3591,7 +3630,7 @@ private Block addStoredBlock(final BlockInfo block, curReplicaDelta = (node.isDecommissioned() || node.isDecommissionInProgress()) ? 0 : 1; if (logEveryBlock) { - blockLog.debug("BLOCK* addStoredBlock: {} is added to {} (size={})", + blockLog.info("BLOCK* addStoredBlock: {} is added to {} (size={})", node, storedBlock, storedBlock.getNumBytes()); } } else if (result == AddBlockResult.REPLACED) { @@ -3605,9 +3644,11 @@ private Block addStoredBlock(final BlockInfo block, corruptReplicas.removeFromCorruptReplicasMap(block, node, Reason.GENSTAMP_MISMATCH); curReplicaDelta = 0; - blockLog.debug("BLOCK* addStoredBlock: Redundant addStoredBlock request" - + " received for {} on node {} size {}", storedBlock, node, - storedBlock.getNumBytes()); + if (blockLog.isDebugEnabled()) { + blockLog.debug("BLOCK* addStoredBlock: Redundant addStoredBlock request" + + " received for {} on node {} size {}", storedBlock, node, + storedBlock.getNumBytes()); + } } // Now check for completion of blocks and safe block count @@ -3708,8 +3749,8 @@ private void invalidateCorruptReplicas(BlockInfo blk, Block reported, removedFromBlocksMap = false; } } catch (IOException e) { - blockLog.debug("invalidateCorruptReplicas error in deleting bad block" - + " {} on {}", blk, node, e); + blockLog.debug("invalidateCorruptReplicas error in deleting bad block {} on {}", + blk, node, e); removedFromBlocksMap = false; } } @@ -3891,8 +3932,8 @@ public int processMisReplicatedBlocks(List blocks) { BlockInfo blk = iter.next(); MisReplicationResult r = processMisReplicatedBlock(blk); processed++; - LOG.debug("BLOCK* processMisReplicatedBlocks: " + - "Re-scanned block {}, result is {}", blk, r); + LOG.debug("BLOCK* processMisReplicatedBlocks: Re-scanned block {}, result is {}", + blk, r); } } finally { namesystem.writeUnlock("processMisReplicatedBlocks"); @@ -4158,8 +4199,8 @@ private void processChosenExcessRedundancy( // final Block blockToInvalidate = getBlockOnStorage(storedBlock, chosen); addToInvalidates(blockToInvalidate, chosen.getDatanodeDescriptor()); - blockLog.debug("BLOCK* chooseExcessRedundancies: " - + "({}, {}) is added to invalidated blocks set", chosen, storedBlock); + blockLog.debug("BLOCK* chooseExcessRedundancies: ({}, {}) is added to invalidated blocks set", + chosen, storedBlock); } private void removeStoredBlock(DatanodeStorageInfo storageInfo, Block block, @@ -4181,8 +4222,8 @@ public void removeStoredBlock(BlockInfo storedBlock, DatanodeDescriptor node) { assert (namesystem.hasWriteLock()); { if (storedBlock == null || !blocksMap.removeNode(storedBlock, node)) { - blockLog.debug("BLOCK* removeStoredBlock: {} has already been" + - " removed from node {}", storedBlock, node); + blockLog.debug("BLOCK* removeStoredBlock: {} has already been removed from node {}", + storedBlock, node); return; } @@ -4194,8 +4235,10 @@ public void removeStoredBlock(BlockInfo storedBlock, DatanodeDescriptor node) { removed |= node.getCached().remove(cblock); removed |= node.getPendingUncached().remove(cblock); if (removed) { - blockLog.debug("BLOCK* removeStoredBlock: {} removed from caching " - + "related lists on node {}", storedBlock, node); + if (blockLog.isDebugEnabled()) { + blockLog.debug("BLOCK* removeStoredBlock: {} removed from caching " + + "related lists on node {}", storedBlock, node); + } } } @@ -4220,9 +4263,9 @@ private void removeStaleReplicas(List staleReplicas, for (ReplicaUnderConstruction r : staleReplicas) { removeStoredBlock(block, r.getExpectedStorageLocation().getDatanodeDescriptor()); - NameNode.blockStateChangeLog - .debug("BLOCK* Removing stale replica {}" + " of {}", r, - Block.toString(r)); + if (blockLog.isDebugEnabled()) { + blockLog.debug("BLOCK* Removing stale replica {} of {}", r, Block.toString(r)); + } } } /** @@ -4350,8 +4393,8 @@ private boolean processAndHandleReportedBlock( maxNumBlocksToLog, numBlocksLogged); } for (Block b : toInvalidate) { - blockLog.debug("BLOCK* addBlock: block {} on node {} size {} does not " + - "belong to any file", b, node, b.getNumBytes()); + blockLog.debug("BLOCK* addBlock: block {} on node {} size {} does not belong to any file", + b, node, b.getNumBytes()); addToInvalidates(b, node); } for (BlockToMarkCorrupt b : toCorrupt) { @@ -4432,9 +4475,11 @@ private void processIncrementalBlockReport(final DatanodeDescriptor node, blockLog.debug("BLOCK* block {}: {} is received from {}", rdbi.getStatus(), rdbi.getBlock(), node); } - blockLog.debug("*BLOCK* NameNode.processIncrementalBlockReport: from " - + "{} receiving: {}, received: {}, deleted: {}", node, receiving, - received, deleted); + if (blockLog.isDebugEnabled()) { + blockLog.debug("*BLOCK* NameNode.processIncrementalBlockReport: from " + + "{} receiving: {}, received: {}, deleted: {}", node, receiving, + received, deleted); + } } /** @@ -4809,8 +4854,10 @@ private int invalidateWorkForOneNode(DatanodeInfo dn) { } finally { namesystem.writeUnlock("invalidateWorkForOneNode"); } - blockLog.debug("BLOCK* {}: ask {} to delete {}", getClass().getSimpleName(), - dn, toInvalidate); + if (blockLog.isDebugEnabled()) { + blockLog.debug("BLOCK* {}: ask {} to delete {}", + getClass().getSimpleName(), dn, toInvalidate); + } return toInvalidate.size(); } @@ -5076,8 +5123,8 @@ public void run() { } } if (isSleep) { - LOG.debug("Clear markedDeleteQueue over {}" + - " millisecond to release the write lock", deleteBlockLockTimeMs); + LOG.debug("Clear markedDeleteQueue over {} millisecond to release the write lock", + deleteBlockLockTimeMs); } try { Thread.sleep(deleteBlockUnlockIntervalTimeMs); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java index 39a40f52ae5f7..1fef3db69d0fc 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java @@ -238,8 +238,8 @@ DatanodeStorageInfo[] chooseTarget(String src, return getPipeline(writer, results.toArray(new DatanodeStorageInfo[results.size()])); } catch (NotEnoughReplicasException nr) { - LOG.debug("Failed to choose with favored nodes (={}), disregard favored" - + " nodes hint and retry.", favoredNodes, nr); + LOG.debug("Failed to choose with favored nodes (={}), disregard favored nodes hint and retry", + favoredNodes, nr); // Fall back to regular block placement disregarding favored nodes hint return chooseTarget(src, numOfReplicas, writer, new ArrayList(numOfReplicas), false, @@ -715,17 +715,19 @@ protected DatanodeStorageInfo chooseLocalRack(Node localMachine, DatanodeDescriptor nextNode = resultStorage.getDatanodeDescriptor(); if (nextNode != localMachine) { if (LOG.isDebugEnabled()) { - LOG.debug("Failed to choose from local rack (location = " + localRack - + "), retry with the rack of the next replica (location = " - + nextNode.getNetworkLocation() + ")", e); + LOG.debug("Failed to choose from local rack (location = {}), retry with the rack " + + "of the next replica (location = {})", localRack, + nextNode.getNetworkLocation(), e); } return chooseFromNextRack(nextNode, excludedNodes, blocksize, maxNodesPerRack, results, avoidStaleNodes, storageTypes); } } - LOG.debug("Failed to choose from local rack (location = {}); the second" - + " replica is not found, retry choosing randomly", localRack, e); + if (LOG.isDebugEnabled()) { + LOG.debug("Failed to choose from local rack (location = {}); the second" + + " replica is not found, retry choosing randomly", localRack, e); + } //the second replica is not found, randomly choose one from the network return chooseRandom(NodeBase.ROOT, excludedNodes, blocksize, @@ -745,9 +747,10 @@ private DatanodeStorageInfo chooseFromNextRack(Node next, return chooseRandom(nextRack, excludedNodes, blocksize, maxNodesPerRack, results, avoidStaleNodes, storageTypes); } catch (NotEnoughReplicasException e) { - LOG.debug("Failed to choose from the next rack (location = {}), " - + "retry choosing randomly", nextRack, e); - // otherwise randomly choose one from the network + LOG.debug("Failed to choose from the next rack (location = {}), retry choosing randomly", + nextRack, e); + + // otherwise randomly choose one from the network return chooseRandom(NodeBase.ROOT, excludedNodes, blocksize, maxNodesPerRack, results, avoidStaleNodes, storageTypes); } @@ -775,10 +778,8 @@ protected void chooseRemoteRack(int numOfReplicas, excludedNodes, blocksize, maxReplicasPerRack, results, avoidStaleNodes, storageTypes); } catch (NotEnoughReplicasException e) { - if (LOG.isDebugEnabled()) { - LOG.debug("Failed to choose remote rack (location = ~" - + localMachine.getNetworkLocation() + "), fallback to local rack", e); - } + LOG.debug("Failed to choose remote rack (location = ~{}), fallback to local rack", + localMachine.getNetworkLocation(), e); chooseRandom(numOfReplicas-(results.size()-oldNumOfReplicas), localMachine.getNetworkLocation(), excludedNodes, blocksize, maxReplicasPerRack, results, avoidStaleNodes, storageTypes); @@ -1276,8 +1277,7 @@ public List chooseReplicasToDelete( firstOne = false; if (cur == null) { LOG.debug( - "No excess replica can be found. excessTypes: {}. " - + "moreThanOne: {}. exactlyOne: {}.", + "No excess replica can be found. excessTypes: {}. moreThanOne: {}. exactlyOne: {}.", excessTypes, moreThanOne, exactlyOne); break; } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeAdminDefaultMonitor.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeAdminDefaultMonitor.java index 3ff360466c98f..e642dfba35188 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeAdminDefaultMonitor.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeAdminDefaultMonitor.java @@ -201,6 +201,7 @@ private void check() { iterkey).iterator(); final List toRemove = new ArrayList<>(); final List unhealthyDns = new ArrayList<>(); + boolean isValidState = true; while (it.hasNext() && !exceededNumBlocksPerCheck() && namesystem .isRunning()) { @@ -265,6 +266,7 @@ private void check() { // to track maintenance expiration. dnAdmin.setInMaintenance(dn); } else { + isValidState = false; Preconditions.checkState(false, "Node %s is in an invalid state! " + "Invalid state: %s %s blocks are on this dn.", @@ -288,7 +290,11 @@ private void check() { // an invalid state. LOG.warn("DatanodeAdminMonitor caught exception when processing node " + "{}.", dn, e); - getPendingNodes().add(dn); + if(isValidState){ + getPendingNodes().add(dn); + } else { + LOG.warn("Ignoring the node {} which is in invalid state", dn); + } toRemove.add(dn); unhealthyDns.remove(dn); } finally { diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeAdminManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeAdminManager.java index 1c95c26a190c3..887cb1072d95f 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeAdminManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeAdminManager.java @@ -152,10 +152,12 @@ void activate(Configuration conf) { executor.scheduleWithFixedDelay(monitor, intervalSecs, intervalSecs, TimeUnit.SECONDS); - LOG.debug("Activating DatanodeAdminManager with interval {} seconds, " + - "{} max blocks per interval, " + - "{} max concurrently tracked nodes.", intervalSecs, - blocksPerInterval, maxConcurrentTrackedNodes); + if (LOG.isDebugEnabled()) { + LOG.debug("Activating DatanodeAdminManager with interval {} seconds, " + + "{} max blocks per interval, " + + "{} max concurrently tracked nodes.", intervalSecs, + blocksPerInterval, maxConcurrentTrackedNodes); + } } /** diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeDescriptor.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeDescriptor.java index a6ca697fa63f7..a2b7afedfdd31 100755 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeDescriptor.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeDescriptor.java @@ -683,10 +683,10 @@ public void addBlockToBeReplicated(Block block, */ void addBlockToBeErasureCoded(ExtendedBlock block, DatanodeDescriptor[] sources, DatanodeStorageInfo[] targets, - byte[] liveBlockIndices, ErasureCodingPolicy ecPolicy) { + byte[] liveBlockIndices, byte[] excludeReconstrutedIndices, ErasureCodingPolicy ecPolicy) { assert (block != null && sources != null && sources.length > 0); BlockECReconstructionInfo task = new BlockECReconstructionInfo(block, - sources, targets, liveBlockIndices, ecPolicy); + sources, targets, liveBlockIndices, excludeReconstrutedIndices, ecPolicy); erasurecodeBlocks.offer(task); BlockManager.LOG.debug("Adding block reconstruction task " + task + "to " + getName() + ", current queue size is " + erasurecodeBlocks.size()); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java index 237daed0960ff..a7a2e5488a0a3 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java @@ -842,9 +842,7 @@ private void removeDatanode(DatanodeDescriptor nodeInfo, decrementVersionCount(nodeInfo.getSoftwareVersion()); blockManager.getBlockReportLeaseManager().unregister(nodeInfo); - if (LOG.isDebugEnabled()) { - LOG.debug("remove datanode " + nodeInfo); - } + LOG.debug("remove datanode {}.", nodeInfo); blockManager.checkSafeMode(); } @@ -906,8 +904,8 @@ void addDatanode(final DatanodeDescriptor node) { resolveUpgradeDomain(node); if (LOG.isDebugEnabled()) { - LOG.debug(getClass().getSimpleName() + ".addDatanode: " - + "node " + node + " is added to datanodeMap."); + LOG.debug("{}.addDatanode: node {} is added to datanodeMap.", + getClass().getSimpleName(), node); } } @@ -918,9 +916,8 @@ private void wipeDatanode(final DatanodeID node) { host2DatanodeMap.remove(datanodeMap.remove(key)); } if (LOG.isDebugEnabled()) { - LOG.debug(getClass().getSimpleName() + ".wipeDatanode(" - + node + "): storage " + key - + " is removed from datanodeMap."); + LOG.debug("{}.wipeDatanode({}): storage {} is removed from datanodeMap.", + getClass().getSimpleName(), node, key); } } @@ -1189,10 +1186,7 @@ public void registerDatanode(DatanodeRegistration nodeReg) // The same datanode has been just restarted to serve the same data // storage. We do not need to remove old data blocks, the delta will // be calculated on the next block report from the datanode - if(NameNode.stateChangeLog.isDebugEnabled()) { - NameNode.stateChangeLog.debug("BLOCK* registerDatanode: " - + "node restarted."); - } + NameNode.stateChangeLog.debug("BLOCK* registerDatanode: node restarted."); } else { // nodeS is found /* The registering datanode is a replacement node for the existing @@ -1535,9 +1529,11 @@ void checkIfClusterIsNowMultiRack(DatanodeDescriptor node) { "now be replicated cross-rack"; LOG.info(message); } else { - message += "Not checking for mis-replicated blocks because this NN is " + - "not yet processing repl queues."; - LOG.debug(message); + if (LOG.isDebugEnabled()) { + message += "Not checking for mis-replicated blocks because this NN " + + "is not yet processing repl queues."; + LOG.debug(message); + } } hasClusterEverBeenMultiRack = true; if (blockManager.isPopulatingReplQueues()) { @@ -1659,11 +1655,9 @@ public List getDatanodeListForReport( } if (LOG.isDebugEnabled()) { - LOG.debug("getDatanodeListForReport with " + - "includedNodes = " + hostConfigManager.getIncludes() + - ", excludedNodes = " + hostConfigManager.getExcludes() + - ", foundNodes = " + foundNodes + - ", nodes = " + nodes); + LOG.debug("getDatanodeListForReport with includedNodes = {}, excludedNodes = {}" + + ", foundNodes = {}, nodes = {}.", hostConfigManager.getIncludes(), + hostConfigManager.getExcludes(), foundNodes, nodes); } return nodes; } @@ -1847,10 +1841,8 @@ public DatanodeCommand[] handleHeartbeat(DatanodeRegistration nodeReg, (double) (totalReplicateBlocks * maxTransfers) / totalBlocks); int numECTasks = (int) Math.ceil( (double) (totalECBlocks * maxTransfers) / totalBlocks); - if (LOG.isDebugEnabled()) { - LOG.debug("Pending replication tasks: " + numReplicationTasks - + " erasure-coded tasks: " + numECTasks); - } + LOG.debug("Pending replication tasks: {} erasure-coded tasks: {}.", + numReplicationTasks, numECTasks); // check pending replication tasks List pendingList = nodeinfo.getReplicationCommand( numReplicationTasks); @@ -1906,9 +1898,7 @@ public DatanodeCommand[] handleHeartbeat(DatanodeRegistration nodeReg, if (slowPeerTracker.isSlowPeerTrackerEnabled()) { final Map slowPeersMap = slowPeers.getSlowPeers(); if (!slowPeersMap.isEmpty()) { - if (LOG.isDebugEnabled()) { - LOG.debug("DataNode " + nodeReg + " reported slow peers: " + slowPeersMap); - } + LOG.debug("DataNode {} reported slow peers: {}.", nodeReg, slowPeersMap); for (Map.Entry slowNodeEntry : slowPeersMap.entrySet()) { slowPeerTracker.addReport(slowNodeEntry.getKey(), nodeReg.getIpcAddr(false), slowNodeEntry.getValue()); @@ -1918,10 +1908,7 @@ public DatanodeCommand[] handleHeartbeat(DatanodeRegistration nodeReg, if (slowDiskTracker != null) { if (!slowDisks.getSlowDisks().isEmpty()) { - if (LOG.isDebugEnabled()) { - LOG.debug("DataNode " + nodeReg + " reported slow disks: " + - slowDisks.getSlowDisks()); - } + LOG.debug("DataNode {} reported slow disks: {}.", nodeReg, slowDisks.getSlowDisks()); slowDiskTracker.addSlowDiskReport(nodeReg.getIpcAddr(false), slowDisks); } slowDiskTracker.checkAndUpdateReportIfNecessary(); @@ -1950,9 +1937,7 @@ public void handleLifeline(DatanodeRegistration nodeReg, StorageReport[] reports, long cacheCapacity, long cacheUsed, int xceiverCount, int failedVolumes, VolumeFailureSummary volumeFailureSummary) throws IOException { - if (LOG.isDebugEnabled()) { - LOG.debug("Received handleLifeline from nodeReg = " + nodeReg); - } + LOG.debug("Received handleLifeline from nodeReg = {}.", nodeReg); DatanodeDescriptor nodeinfo = getDatanode(nodeReg); if (nodeinfo == null || !nodeinfo.isRegistered()) { // This can happen if the lifeline message comes when DataNode is either diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/ErasureCodingWork.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/ErasureCodingWork.java index 6158677654b94..e5303a28d714e 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/ErasureCodingWork.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/ErasureCodingWork.java @@ -32,6 +32,7 @@ class ErasureCodingWork extends BlockReconstructionWork { private final byte[] liveBlockIndices; private final byte[] liveBusyBlockIndices; + private final byte[] excludeReconstructedIndices; private final String blockPoolId; public ErasureCodingWork(String blockPoolId, BlockInfo block, @@ -40,12 +41,14 @@ public ErasureCodingWork(String blockPoolId, BlockInfo block, List containingNodes, List liveReplicaStorages, int additionalReplRequired, int priority, - byte[] liveBlockIndices, byte[] liveBusyBlockIndices) { + byte[] liveBlockIndices, byte[] liveBusyBlockIndices, + byte[] excludeReconstrutedIndices) { super(block, bc, srcNodes, containingNodes, liveReplicaStorages, additionalReplRequired, priority); this.blockPoolId = blockPoolId; this.liveBlockIndices = liveBlockIndices; this.liveBusyBlockIndices = liveBusyBlockIndices; + this.excludeReconstructedIndices=excludeReconstrutedIndices; LOG.debug("Creating an ErasureCodingWork to {} reconstruct ", block); } @@ -147,7 +150,7 @@ void addTaskToDatanode(NumberReplicas numberReplicas) { } else { targets[0].getDatanodeDescriptor().addBlockToBeErasureCoded( new ExtendedBlock(blockPoolId, stripedBlk), getSrcNodes(), targets, - getLiveBlockIndices(), stripedBlk.getErasureCodingPolicy()); + liveBlockIndices, excludeReconstructedIndices, stripedBlk.getErasureCodingPolicy()); } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/HeartbeatManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/HeartbeatManager.java index b923ba3a65590..01e1b6392a09d 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/HeartbeatManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/HeartbeatManager.java @@ -71,6 +71,7 @@ class HeartbeatManager implements DatanodeStatistics { /** Heartbeat monitor thread. */ private final Daemon heartbeatThread = new Daemon(new Monitor()); private final StopWatch heartbeatStopWatch = new StopWatch(); + private final int numOfDeadDatanodesRemove; final Namesystem namesystem; final BlockManager blockManager; @@ -96,6 +97,9 @@ class HeartbeatManager implements DatanodeStatistics { enableLogStaleNodes = conf.getBoolean( DFSConfigKeys.DFS_NAMENODE_ENABLE_LOG_STALE_DATANODE_KEY, DFSConfigKeys.DFS_NAMENODE_ENABLE_LOG_STALE_DATANODE_DEFAULT); + this.numOfDeadDatanodesRemove = conf.getInt( + DFSConfigKeys.DFS_NAMENODE_REMOVE_DEAD_DATANODE_BATCHNUM_KEY, + DFSConfigKeys.DFS_NAMENODE_REMOVE_BAD_BATCH_NUM_DEFAULT); if (avoidStaleDataNodesForWrite && staleInterval < recheckInterval) { this.heartbeatRecheckInterval = staleInterval; @@ -404,7 +408,7 @@ private void dumpStaleNodes(List staleNodes) { /** * Check if there are any expired heartbeats, and if so, * whether any blocks have to be re-replicated. - * While removing dead datanodes, make sure that only one datanode is marked + * While removing dead datanodes, make sure that limited datanodes is marked * dead at a time within the synchronized section. Otherwise, a cascading * effect causes more datanodes to be declared dead. * Check if there are any failed storage and if so, @@ -436,12 +440,17 @@ void heartbeatCheck() { return; } boolean allAlive = false; + // Locate limited dead nodes. + List deadDatanodes = new ArrayList<>( + numOfDeadDatanodesRemove); + // Locate limited failed storages that isn't on a dead node. + List failedStorages = new ArrayList<>( + numOfDeadDatanodesRemove); + while (!allAlive) { - // locate the first dead node. - DatanodeDescriptor dead = null; - // locate the first failed storage that isn't on a dead node. - DatanodeStorageInfo failedStorage = null; + deadDatanodes.clear(); + failedStorages.clear(); // check the number of stale storages int numOfStaleStorages = 0; @@ -452,9 +461,10 @@ void heartbeatCheck() { if (shouldAbortHeartbeatCheck(0)) { return; } - if (dead == null && dm.isDatanodeDead(d)) { + if (deadDatanodes.size() < numOfDeadDatanodesRemove && + dm.isDatanodeDead(d)) { stats.incrExpiredHeartbeats(); - dead = d; + deadDatanodes.add(d); // remove the node from stale list to adjust the stale list size // before setting the stale count of the DatanodeManager removeNodeFromStaleList(d); @@ -476,10 +486,10 @@ void heartbeatCheck() { numOfStaleStorages++; } - if (failedStorage == null && + if (failedStorages.size() < numOfDeadDatanodesRemove && storageInfo.areBlocksOnFailedStorage() && - d != dead) { - failedStorage = storageInfo; + !deadDatanodes.contains(d)) { + failedStorages.add(storageInfo); } } } @@ -492,12 +502,12 @@ void heartbeatCheck() { // log nodes detected as stale since last heartBeat dumpStaleNodes(staleNodes); - allAlive = dead == null && failedStorage == null; + allAlive = deadDatanodes.isEmpty() && failedStorages.isEmpty(); if (!allAlive && namesystem.isInStartupSafeMode()) { return; } - if (dead != null) { + for (DatanodeDescriptor dead : deadDatanodes) { // acquire the fsnamesystem lock, and then remove the dead node. namesystem.writeLock(); try { @@ -506,7 +516,7 @@ void heartbeatCheck() { namesystem.writeUnlock("removeDeadDatanode"); } } - if (failedStorage != null) { + for (DatanodeStorageInfo failedStorage : failedStorages) { // acquire the fsnamesystem lock, and remove blocks on the storage. namesystem.writeLock(); try { diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/PendingReconstructionBlocks.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/PendingReconstructionBlocks.java index 3e56606197bb4..6c3b4c97bed37 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/PendingReconstructionBlocks.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/PendingReconstructionBlocks.java @@ -76,6 +76,14 @@ void start() { timerThread.start(); } + public void setTimeout(long timeoutPeriod) { + this.timeout = timeoutPeriod; + } + + public long getTimeout() { + return this.timeout; + } + /** * Add a block to the list of pending reconstructions * @param block The corresponding block diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/HostRestrictingAuthorizationFilter.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/HostRestrictingAuthorizationFilter.java index e9f1cf09e86d4..0308e55e4cf7a 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/HostRestrictingAuthorizationFilter.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/HostRestrictingAuthorizationFilter.java @@ -117,17 +117,12 @@ private boolean matchRule(String user, String remoteIp, String path) { String rulePath = rule.getPath(); LOG.trace("Evaluating rule, subnet: {}, path: {}", subnet != null ? subnet.getCidrSignature() : "*", rulePath); - try { - if ((subnet == null || subnet.isInRange(remoteIp)) - && FilenameUtils.directoryContains(rulePath, path)) { - LOG.debug("Found matching rule, subnet: {}, path: {}; returned true", - rule.getSubnet() != null ? subnet.getCidrSignature() : null, - rulePath); - return true; - } - } catch (IOException e) { - LOG.warn("Got IOException {}; returned false", e); - return false; + if ((subnet == null || subnet.isInRange(remoteIp)) + && FilenameUtils.directoryContains(rulePath, path)) { + LOG.debug("Found matching rule, subnet: {}, path: {}; returned true", + rule.getSubnet() != null ? subnet.getCidrSignature() : null, + rulePath); + return true; } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPOfferService.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPOfferService.java index a990e1915de6f..d660970c72542 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPOfferService.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPOfferService.java @@ -741,7 +741,6 @@ private boolean processCommandFromActive(DatanodeCommand cmd, // Exceptions caught here are not expected to be disk-related. throw e; } - dn.metrics.incrBlocksRemoved(toDelete.length); break; case DatanodeProtocol.DNA_CACHE: LOG.info("DatanodeCommand action: DNA_CACHE for " + diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java index 10438cd79e39c..58334bf5c0750 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java @@ -160,6 +160,7 @@ import org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.BlockPoolSlice; import org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsVolumeImpl; import org.apache.hadoop.hdfs.util.DataTransferThrottler; +import org.apache.hadoop.ipc.Server; import org.apache.hadoop.util.*; import org.apache.hadoop.hdfs.client.BlockReportOptions; import org.apache.hadoop.hdfs.client.HdfsClientConfigKeys; @@ -889,8 +890,11 @@ private String reconfDfsUsageParameters(String property, String newVal) String result = null; try { LOG.info("Reconfiguring {} to {}", property, newVal); + if (data == null) { + LOG.debug("FsDatasetSpi has not been initialized."); + throw new IOException("FsDatasetSpi has not been initialized"); + } if (property.equals(FS_DU_INTERVAL_KEY)) { - Preconditions.checkNotNull(data, "FsDatasetSpi has not been initialized."); long interval = (newVal == null ? FS_DU_INTERVAL_DEFAULT : Long.parseLong(newVal)); result = Long.toString(interval); @@ -902,7 +906,6 @@ private String reconfDfsUsageParameters(String property, String newVal) } } } else if (property.equals(FS_GETSPACEUSED_JITTER_KEY)) { - Preconditions.checkNotNull(data, "FsDatasetSpi has not been initialized."); long jitter = (newVal == null ? FS_GETSPACEUSED_JITTER_DEFAULT : Long.parseLong(newVal)); result = Long.toString(jitter); @@ -914,7 +917,6 @@ private String reconfDfsUsageParameters(String property, String newVal) } } } else if (property.equals(FS_GETSPACEUSED_CLASSNAME)) { - Preconditions.checkNotNull(data, "FsDatasetSpi has not been initialized."); Class klass; if (newVal == null) { if (Shell.WINDOWS) { @@ -1174,7 +1176,7 @@ private void refreshVolumes(String newVolumes) throws IOException { .newFixedThreadPool(changedVolumes.newLocations.size()); List> exceptions = Lists.newArrayList(); - Preconditions.checkNotNull(data, "Storage not yet initialized"); + checkStorageState("refreshVolumes"); for (final StorageLocation location : changedVolumes.newLocations) { exceptions.add(service.submit(new Callable() { @Override @@ -1274,7 +1276,7 @@ private synchronized void removeVolumes( clearFailure, Joiner.on(",").join(storageLocations))); IOException ioe = null; - Preconditions.checkNotNull(data, "Storage not yet initialized"); + checkStorageState("removeVolumes"); // Remove volumes and block infos from FsDataset. data.removeVolumes(storageLocations, clearFailure); @@ -1438,7 +1440,7 @@ private void checkSuperuserPrivilege() throws IOException, AccessControlExceptio return; } // Try to get the ugi in the RPC call. - UserGroupInformation callerUgi = ipcServer.getRemoteUser(); + UserGroupInformation callerUgi = Server.getRemoteUser(); if (callerUgi == null) { // This is not from RPC. callerUgi = UserGroupInformation.getCurrentUser(); @@ -2301,7 +2303,7 @@ public BlockLocalPathInfo getBlockLocalPathInfo(ExtendedBlock block, Token token) throws IOException { checkBlockLocalPathAccess(); checkBlockToken(block, token, BlockTokenIdentifier.AccessMode.READ); - Preconditions.checkNotNull(data, "Storage not yet initialized"); + checkStorageState("getBlockLocalPathInfo"); BlockLocalPathInfo info = data.getBlockLocalPathInfo(block); if (info != null) { LOG.trace("getBlockLocalPathInfo successful " + @@ -2351,7 +2353,7 @@ FileInputStream[] requestShortCircuitFdsForRead(final ExtendedBlock blk, FileInputStream fis[] = new FileInputStream[2]; try { - Preconditions.checkNotNull(data, "Storage not yet initialized"); + checkStorageState("requestShortCircuitFdsForRead"); fis[0] = (FileInputStream)data.getBlockInputStream(blk, 0); fis[1] = DatanodeUtil.getMetaDataInputStream(blk, data); } catch (ClassCastException e) { @@ -3382,7 +3384,7 @@ public static void main(String args[]) { @Override // InterDatanodeProtocol public ReplicaRecoveryInfo initReplicaRecovery(RecoveringBlock rBlock) throws IOException { - Preconditions.checkNotNull(data, "Storage not yet initialized"); + checkStorageState("initReplicaRecovery"); return data.initReplicaRecovery(rBlock); } @@ -3393,7 +3395,7 @@ public ReplicaRecoveryInfo initReplicaRecovery(RecoveringBlock rBlock) public String updateReplicaUnderRecovery(final ExtendedBlock oldBlock, final long recoveryId, final long newBlockId, final long newLength) throws IOException { - Preconditions.checkNotNull(data, "Storage not yet initialized"); + checkStorageState("updateReplicaUnderRecovery"); final Replica r = data.updateReplicaUnderRecovery(oldBlock, recoveryId, newBlockId, newLength); // Notify the namenode of the updated block info. This is important @@ -3628,7 +3630,10 @@ public String getBPServiceActorInfo() { */ @Override // DataNodeMXBean public String getVolumeInfo() { - Preconditions.checkNotNull(data, "Storage not yet initialized"); + if (data == null) { + LOG.debug("Storage not yet initialized."); + return ""; + } return JSON.toString(data.getVolumeInfoMap()); } @@ -3642,7 +3647,7 @@ public String getDiskBalancerStatus() { try { return getDiskBalancer().queryWorkStatus().toJsonString(); } catch (IOException ex) { - LOG.debug("Reading diskbalancer Status failed. ex:{}", ex); + LOG.debug("Reading diskbalancer Status failed.", ex); return ""; } } @@ -3676,10 +3681,23 @@ public void deleteBlockPool(String blockPoolId, boolean force) "The block pool is still running. First do a refreshNamenodes to " + "shutdown the block pool service"); } - Preconditions.checkNotNull(data, "Storage not yet initialized"); + checkStorageState("deleteBlockPool"); data.deleteBlockPool(blockPoolId, force); } + /** + * Check if storage has been initialized. + * @param methodName caller name + * @throws IOException throw IOException if not yet initialized. + */ + private void checkStorageState(String methodName) throws IOException { + if (data == null) { + String message = "Storage not yet initialized for " + methodName; + LOG.debug(message); + throw new IOException(message); + } + } + @Override // ClientDatanodeProtocol public synchronized void shutdownDatanode(boolean forUpgrade) throws IOException { checkSuperuserPrivilege(); @@ -4128,7 +4146,7 @@ public String getSlowDisks() { @Override public List getVolumeReport() throws IOException { checkSuperuserPrivilege(); - Preconditions.checkNotNull(data, "Storage not yet initialized"); + checkStorageState("getVolumeReport"); Map volumeInfoMap = data.getVolumeInfoMap(); if (volumeInfoMap == null) { LOG.warn("DataNode volume info not available."); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/ErasureCodingWorker.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/ErasureCodingWorker.java index 0f145d6cdd7b2..74c4cf1bd5f34 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/ErasureCodingWorker.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/ErasureCodingWorker.java @@ -127,7 +127,7 @@ public void processErasureCodingTasks( reconInfo.getExtendedBlock(), reconInfo.getErasureCodingPolicy(), reconInfo.getLiveBlockIndices(), reconInfo.getSourceDnInfos(), reconInfo.getTargetDnInfos(), reconInfo.getTargetStorageTypes(), - reconInfo.getTargetStorageIDs()); + reconInfo.getTargetStorageIDs(), reconInfo.getExcludeReconstructedIndices()); // It may throw IllegalArgumentException from task#stripedReader // constructor. final StripedBlockReconstructor task = diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedReconstructionInfo.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedReconstructionInfo.java index c166f5ec03125..caf8dfa950446 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedReconstructionInfo.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedReconstructionInfo.java @@ -41,26 +41,28 @@ public class StripedReconstructionInfo { private final DatanodeInfo[] targets; private final StorageType[] targetStorageTypes; private final String[] targetStorageIds; + private final byte[] excludeReconstructedIndices; public StripedReconstructionInfo(ExtendedBlock blockGroup, ErasureCodingPolicy ecPolicy, byte[] liveIndices, DatanodeInfo[] sources, byte[] targetIndices) { this(blockGroup, ecPolicy, liveIndices, sources, targetIndices, null, - null, null); + null, null, new byte[0]); } StripedReconstructionInfo(ExtendedBlock blockGroup, ErasureCodingPolicy ecPolicy, byte[] liveIndices, DatanodeInfo[] sources, DatanodeInfo[] targets, StorageType[] targetStorageTypes, - String[] targetStorageIds) { + String[] targetStorageIds, byte[] excludeReconstructedIndices) { this(blockGroup, ecPolicy, liveIndices, sources, null, targets, - targetStorageTypes, targetStorageIds); + targetStorageTypes, targetStorageIds, excludeReconstructedIndices); } private StripedReconstructionInfo(ExtendedBlock blockGroup, ErasureCodingPolicy ecPolicy, byte[] liveIndices, DatanodeInfo[] sources, byte[] targetIndices, DatanodeInfo[] targets, - StorageType[] targetStorageTypes, String[] targetStorageIds) { + StorageType[] targetStorageTypes, String[] targetStorageIds, + byte[] excludeReconstructedIndices) { this.blockGroup = blockGroup; this.ecPolicy = ecPolicy; @@ -70,6 +72,7 @@ private StripedReconstructionInfo(ExtendedBlock blockGroup, this.targets = targets; this.targetStorageTypes = targetStorageTypes; this.targetStorageIds = targetStorageIds; + this.excludeReconstructedIndices = excludeReconstructedIndices; } ExtendedBlock getBlockGroup() { @@ -104,5 +107,9 @@ String[] getTargetStorageIds() { return targetStorageIds; } + byte[] getExcludeReconstructedIndices() { + return excludeReconstructedIndices; + } + } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedReconstructor.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedReconstructor.java index 2caa872d99b54..466c7c4254c8b 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedReconstructor.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedReconstructor.java @@ -120,6 +120,7 @@ abstract class StripedReconstructor { private final CachingStrategy cachingStrategy; private long maxTargetLength = 0L; private final BitSet liveBitSet; + private final BitSet excludeBitSet; // metrics private AtomicLong bytesRead = new AtomicLong(0); @@ -137,6 +138,12 @@ abstract class StripedReconstructor { for (int i = 0; i < stripedReconInfo.getLiveIndices().length; i++) { liveBitSet.set(stripedReconInfo.getLiveIndices()[i]); } + excludeBitSet = new BitSet( + ecPolicy.getNumDataUnits() + ecPolicy.getNumParityUnits()); + for (int i = 0; i < stripedReconInfo.getExcludeReconstructedIndices().length; i++) { + excludeBitSet.set(stripedReconInfo.getExcludeReconstructedIndices()[i]); + } + blockGroup = stripedReconInfo.getBlockGroup(); stripedReader = new StripedReader(this, datanode, conf, stripedReconInfo); cachingStrategy = CachingStrategy.newDefaultStrategy(); @@ -261,6 +268,10 @@ BitSet getLiveBitSet() { return liveBitSet; } + BitSet getExcludeBitSet(){ + return excludeBitSet; + } + long getMaxTargetLength() { return maxTargetLength; } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedWriter.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedWriter.java index 590ecb46eed52..ca06dfc120531 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedWriter.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedWriter.java @@ -123,13 +123,14 @@ void init() throws IOException { private void initTargetIndices() { BitSet bitset = reconstructor.getLiveBitSet(); + BitSet excludebitset=reconstructor.getExcludeBitSet(); int m = 0; hasValidTargets = false; for (int i = 0; i < dataBlkNum + parityBlkNum; i++) { if (!bitset.get(i)) { if (reconstructor.getBlockLen(i) > 0) { - if (m < targets.length) { + if (m < targets.length && !excludebitset.get(i)) { targetIndices[m++] = (short)i; hasValidTargets = true; } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java index 633eeab03e9cc..adb90ab7ba8ec 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java @@ -2350,6 +2350,7 @@ private void invalidate(String bpid, Block[] invalidBlks, boolean async) removing = volumeMap.remove(bpid, invalidBlks[i]); addDeletingBlock(bpid, removing.getBlockId()); LOG.debug("Block file {} is to be deleted", removing.getBlockURI()); + datanode.getMetrics().incrBlocksRemoved(1); if (removing instanceof ReplicaInPipeline) { ((ReplicaInPipeline) removing).releaseAllBytesReserved(); } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/HostRestrictingAuthorizationFilterHandler.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/HostRestrictingAuthorizationFilterHandler.java index 8c5d538d23384..c76d93469196b 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/HostRestrictingAuthorizationFilterHandler.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/HostRestrictingAuthorizationFilterHandler.java @@ -43,8 +43,8 @@ import java.util.List; import java.util.Map; -import static io.netty.handler.codec.http.HttpHeaders.Names.CONNECTION; -import static io.netty.handler.codec.http.HttpHeaders.Values.CLOSE; +import static io.netty.handler.codec.http.HttpHeaderNames.CONNECTION; +import static io.netty.handler.codec.http.HttpHeaderValues.CLOSE; import static io.netty.handler.codec.http.HttpResponseStatus.INTERNAL_SERVER_ERROR; import static io.netty.handler.codec.http.HttpVersion.HTTP_1_1; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/RestCsrfPreventionFilterHandler.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/RestCsrfPreventionFilterHandler.java index a2c2d370439ac..4e3c2d1ae7bda 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/RestCsrfPreventionFilterHandler.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/RestCsrfPreventionFilterHandler.java @@ -17,8 +17,8 @@ */ package org.apache.hadoop.hdfs.server.datanode.web; -import static io.netty.handler.codec.http.HttpHeaders.Names.CONNECTION; -import static io.netty.handler.codec.http.HttpHeaders.Values.CLOSE; +import static io.netty.handler.codec.http.HttpHeaderNames.CONNECTION; +import static io.netty.handler.codec.http.HttpHeaderValues.CLOSE; import static io.netty.handler.codec.http.HttpResponseStatus.INTERNAL_SERVER_ERROR; import static io.netty.handler.codec.http.HttpVersion.HTTP_1_1; import static org.apache.hadoop.hdfs.client.HdfsClientConfigKeys.DFS_WEBHDFS_REST_CSRF_ENABLED_DEFAULT; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/SimpleHttpProxyHandler.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/SimpleHttpProxyHandler.java index 51aeab024b81d..ef8d90a4c4e6c 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/SimpleHttpProxyHandler.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/SimpleHttpProxyHandler.java @@ -34,12 +34,12 @@ import io.netty.handler.codec.http.HttpRequest; import io.netty.handler.codec.http.HttpRequestEncoder; import io.netty.handler.codec.http.HttpResponseEncoder; +import io.netty.handler.codec.http.HttpHeaderValues; import org.slf4j.Logger; import java.net.InetSocketAddress; -import static io.netty.handler.codec.http.HttpHeaders.Names.CONNECTION; -import static io.netty.handler.codec.http.HttpHeaders.Values; +import static io.netty.handler.codec.http.HttpHeaderNames.CONNECTION; import static io.netty.handler.codec.http.HttpResponseStatus.INTERNAL_SERVER_ERROR; import static io.netty.handler.codec.http.HttpVersion.HTTP_1_1; @@ -119,12 +119,12 @@ public void operationComplete(ChannelFuture future) throws Exception { ctx.channel().pipeline().remove(HttpResponseEncoder.class); HttpRequest newReq = new DefaultFullHttpRequest(HTTP_1_1, req.method(), req.uri()); newReq.headers().add(req.headers()); - newReq.headers().set(CONNECTION, Values.CLOSE); + newReq.headers().set(CONNECTION, HttpHeaderValues.CLOSE); future.channel().writeAndFlush(newReq); } else { DefaultHttpResponse resp = new DefaultHttpResponse(HTTP_1_1, INTERNAL_SERVER_ERROR); - resp.headers().set(CONNECTION, Values.CLOSE); + resp.headers().set(CONNECTION, HttpHeaderValues.CLOSE); LOG.info("Proxy " + uri + " failed. Cause: ", future.cause()); ctx.writeAndFlush(resp).addListener(ChannelFutureListener.CLOSE); client.close(); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/webhdfs/ExceptionHandler.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/webhdfs/ExceptionHandler.java index 02ec25c13c874..6fe0851bb3f70 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/webhdfs/ExceptionHandler.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/webhdfs/ExceptionHandler.java @@ -33,8 +33,8 @@ import java.io.FileNotFoundException; import java.io.IOException; -import static io.netty.handler.codec.http.HttpHeaders.Names.CONTENT_LENGTH; -import static io.netty.handler.codec.http.HttpHeaders.Names.CONTENT_TYPE; +import static io.netty.handler.codec.http.HttpHeaderNames.CONTENT_LENGTH; +import static io.netty.handler.codec.http.HttpHeaderNames.CONTENT_TYPE; import static io.netty.handler.codec.http.HttpResponseStatus.BAD_REQUEST; import static io.netty.handler.codec.http.HttpResponseStatus.FORBIDDEN; import static io.netty.handler.codec.http.HttpResponseStatus.INTERNAL_SERVER_ERROR; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/webhdfs/HdfsWriter.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/webhdfs/HdfsWriter.java index 3d928ff3eed64..f01479a9b92e6 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/webhdfs/HdfsWriter.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/webhdfs/HdfsWriter.java @@ -30,8 +30,8 @@ import java.io.IOException; import java.io.OutputStream; -import static io.netty.handler.codec.http.HttpHeaders.Names.CONNECTION; -import static io.netty.handler.codec.http.HttpHeaders.Values.CLOSE; +import static io.netty.handler.codec.http.HttpHeaderNames.CONNECTION; +import static io.netty.handler.codec.http.HttpHeaderValues.CLOSE; class HdfsWriter extends SimpleChannelInboundHandler { private final DFSClient client; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/webhdfs/WebHdfsHandler.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/webhdfs/WebHdfsHandler.java index 7567162840893..8de736ac0e237 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/webhdfs/WebHdfsHandler.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/webhdfs/WebHdfsHandler.java @@ -62,17 +62,17 @@ import java.security.PrivilegedExceptionAction; import java.util.EnumSet; -import static io.netty.handler.codec.http.HttpHeaders.Names.ACCEPT; -import static io.netty.handler.codec.http.HttpHeaders.Names.ACCESS_CONTROL_ALLOW_HEADERS; -import static io.netty.handler.codec.http.HttpHeaders.Names.ACCESS_CONTROL_ALLOW_METHODS; -import static io.netty.handler.codec.http.HttpHeaders.Names.ACCESS_CONTROL_ALLOW_ORIGIN; -import static io.netty.handler.codec.http.HttpHeaders.Names.ACCESS_CONTROL_MAX_AGE; -import static io.netty.handler.codec.http.HttpHeaders.Names.CONNECTION; -import static io.netty.handler.codec.http.HttpHeaders.Names.CONTENT_LENGTH; -import static io.netty.handler.codec.http.HttpHeaders.Names.CONTENT_TYPE; -import static io.netty.handler.codec.http.HttpHeaders.Names.LOCATION; -import static io.netty.handler.codec.http.HttpHeaders.Values.CLOSE; -import static io.netty.handler.codec.http.HttpHeaders.Values.KEEP_ALIVE; +import static io.netty.handler.codec.http.HttpHeaderNames.ACCEPT; +import static io.netty.handler.codec.http.HttpHeaderNames.ACCESS_CONTROL_ALLOW_HEADERS; +import static io.netty.handler.codec.http.HttpHeaderNames.ACCESS_CONTROL_ALLOW_METHODS; +import static io.netty.handler.codec.http.HttpHeaderNames.ACCESS_CONTROL_ALLOW_ORIGIN; +import static io.netty.handler.codec.http.HttpHeaderNames.ACCESS_CONTROL_MAX_AGE; +import static io.netty.handler.codec.http.HttpHeaderNames.CONNECTION; +import static io.netty.handler.codec.http.HttpHeaderNames.CONTENT_LENGTH; +import static io.netty.handler.codec.http.HttpHeaderNames.CONTENT_TYPE; +import static io.netty.handler.codec.http.HttpHeaderNames.LOCATION; +import static io.netty.handler.codec.http.HttpHeaderValues.CLOSE; +import static io.netty.handler.codec.http.HttpHeaderValues.KEEP_ALIVE; import static io.netty.handler.codec.http.HttpMethod.GET; import static io.netty.handler.codec.http.HttpMethod.OPTIONS; import static io.netty.handler.codec.http.HttpMethod.POST; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/CacheManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/CacheManager.java index 03dab29a54a5d..e71b057595952 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/CacheManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/CacheManager.java @@ -976,9 +976,8 @@ private void setCachedLocations(LocatedBlock block) { public final void processCacheReport(final DatanodeID datanodeID, final List blockIds) throws IOException { if (!enabled) { - LOG.debug("Ignoring cache report from {} because {} = false. " + - "number of blocks: {}", datanodeID, - DFS_NAMENODE_CACHING_ENABLED_KEY, blockIds.size()); + LOG.debug("Ignoring cache report from {} because {} = false. number of blocks: {}", + datanodeID, DFS_NAMENODE_CACHING_ENABLED_KEY, blockIds.size()); return; } namesystem.writeLock(); @@ -1003,9 +1002,8 @@ public final void processCacheReport(final DatanodeID datanodeID, if (metrics != null) { metrics.addCacheBlockReport((int) (endTime - startTime)); } - LOG.debug("Processed cache report from {}, blocks: {}, " + - "processing time: {} msecs", datanodeID, blockIds.size(), - (endTime - startTime)); + LOG.debug("Processed cache report from {}, blocks: {}, processing time: {} msecs", + datanodeID, blockIds.size(), (endTime - startTime)); } private void processCacheReportImpl(final DatanodeDescriptor datanode, diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java index 850b2fc570884..5bb6872e5889d 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java @@ -17,6 +17,7 @@ */ package org.apache.hadoop.hdfs.server.namenode; +import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_IP_PROXY_USERS; import static org.apache.hadoop.util.ExitUtil.terminate; import static org.apache.hadoop.util.Time.monotonicNow; @@ -30,6 +31,7 @@ import java.util.List; import java.util.concurrent.atomic.LongAdder; +import org.apache.commons.lang3.tuple.Pair; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.conf.Configuration; @@ -107,7 +109,6 @@ import org.apache.hadoop.hdfs.server.protocol.NamespaceInfo; import org.apache.hadoop.hdfs.server.protocol.RemoteEditLogManifest; import org.apache.hadoop.io.IOUtils; -import org.apache.hadoop.ipc.Server; import org.apache.hadoop.security.token.delegation.DelegationKey; import org.apache.hadoop.util.Lists; @@ -195,6 +196,9 @@ private enum State { protected final OpInstanceCache cache = new OpInstanceCache(); + // Users who can override the client ip + private final String[] ipProxyUsers; + /** * The edit directories that are shared between primary and secondary. */ @@ -246,6 +250,7 @@ static FSEditLog newInstance(Configuration conf, NNStorage storage, * @param editsDirs List of journals to use */ FSEditLog(Configuration conf, NNStorage storage, List editsDirs) { + ipProxyUsers = conf.getStrings(DFS_NAMENODE_IP_PROXY_USERS); isSyncRunning = false; this.conf = conf; this.storage = storage; @@ -799,8 +804,10 @@ private void printStatistics(boolean force) { /** Record the RPC IDs if necessary */ private void logRpcIds(FSEditLogOp op, boolean toLogRpcIds) { if (toLogRpcIds) { - op.setRpcClientId(Server.getClientId()); - op.setRpcCallId(Server.getCallId()); + Pair clientIdAndCallId = + NameNode.getClientIdAndCallId(this.ipProxyUsers); + op.setRpcClientId(clientIdAndCallId.getLeft()); + op.setRpcCallId(clientIdAndCallId.getRight()); } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogOp.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogOp.java index 8bddc6741a1aa..c8e0cd224e1af 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogOp.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogOp.java @@ -823,7 +823,7 @@ static class AddOp extends AddCloseOp { } static AddOp getInstance(OpInstanceCache cache) { - return (AddOp) cache.get(OP_ADD); + return cache.get(OP_ADD); } @Override @@ -851,7 +851,7 @@ static class CloseOp extends AddCloseOp { } static CloseOp getInstance(OpInstanceCache cache) { - return (CloseOp)cache.get(OP_CLOSE); + return cache.get(OP_CLOSE); } @Override @@ -969,7 +969,7 @@ static class AddBlockOp extends FSEditLogOp { } static AddBlockOp getInstance(OpInstanceCache cache) { - return (AddBlockOp) cache.get(OP_ADD_BLOCK); + return cache.get(OP_ADD_BLOCK); } @Override @@ -1081,7 +1081,7 @@ static class UpdateBlocksOp extends FSEditLogOp implements BlockListUpdatingOp { } static UpdateBlocksOp getInstance(OpInstanceCache cache) { - return (UpdateBlocksOp)cache.get(OP_UPDATE_BLOCKS); + return cache.get(OP_UPDATE_BLOCKS); } @Override @@ -1175,7 +1175,7 @@ static class SetReplicationOp extends FSEditLogOp { } static SetReplicationOp getInstance(OpInstanceCache cache) { - return (SetReplicationOp)cache.get(OP_SET_REPLICATION); + return cache.get(OP_SET_REPLICATION); } @Override @@ -1254,7 +1254,7 @@ static class ConcatDeleteOp extends FSEditLogOp { } static ConcatDeleteOp getInstance(OpInstanceCache cache) { - return (ConcatDeleteOp)cache.get(OP_CONCAT_DELETE); + return cache.get(OP_CONCAT_DELETE); } @Override @@ -1412,7 +1412,7 @@ static class RenameOldOp extends FSEditLogOp { } static RenameOldOp getInstance(OpInstanceCache cache) { - return (RenameOldOp)cache.get(OP_RENAME_OLD); + return cache.get(OP_RENAME_OLD); } @Override @@ -1524,7 +1524,7 @@ static class DeleteOp extends FSEditLogOp { } static DeleteOp getInstance(OpInstanceCache cache) { - return (DeleteOp)cache.get(OP_DELETE); + return cache.get(OP_DELETE); } @Override @@ -1625,7 +1625,7 @@ static class MkdirOp extends FSEditLogOp { } static MkdirOp getInstance(OpInstanceCache cache) { - return (MkdirOp)cache.get(OP_MKDIR); + return cache.get(OP_MKDIR); } @Override @@ -1798,7 +1798,7 @@ static class SetGenstampV1Op extends FSEditLogOp { } static SetGenstampV1Op getInstance(OpInstanceCache cache) { - return (SetGenstampV1Op)cache.get(OP_SET_GENSTAMP_V1); + return cache.get(OP_SET_GENSTAMP_V1); } @Override @@ -1864,7 +1864,7 @@ static class SetGenstampV2Op extends FSEditLogOp { } static SetGenstampV2Op getInstance(OpInstanceCache cache) { - return (SetGenstampV2Op)cache.get(OP_SET_GENSTAMP_V2); + return cache.get(OP_SET_GENSTAMP_V2); } @Override @@ -1922,7 +1922,7 @@ static class AllocateBlockIdOp extends FSEditLogOp { } static AllocateBlockIdOp getInstance(OpInstanceCache cache) { - return (AllocateBlockIdOp)cache.get(OP_ALLOCATE_BLOCK_ID); + return cache.get(OP_ALLOCATE_BLOCK_ID); } @Override @@ -1981,7 +1981,7 @@ static class SetPermissionsOp extends FSEditLogOp { } static SetPermissionsOp getInstance(OpInstanceCache cache) { - return (SetPermissionsOp)cache.get(OP_SET_PERMISSIONS); + return cache.get(OP_SET_PERMISSIONS); } @Override @@ -2054,7 +2054,7 @@ static class SetOwnerOp extends FSEditLogOp { } static SetOwnerOp getInstance(OpInstanceCache cache) { - return (SetOwnerOp)cache.get(OP_SET_OWNER); + return cache.get(OP_SET_OWNER); } @Override @@ -2141,7 +2141,7 @@ static class SetNSQuotaOp extends FSEditLogOp { } static SetNSQuotaOp getInstance(OpInstanceCache cache) { - return (SetNSQuotaOp)cache.get(OP_SET_NS_QUOTA); + return cache.get(OP_SET_NS_QUOTA); } @Override @@ -2199,7 +2199,7 @@ static class ClearNSQuotaOp extends FSEditLogOp { } static ClearNSQuotaOp getInstance(OpInstanceCache cache) { - return (ClearNSQuotaOp)cache.get(OP_CLEAR_NS_QUOTA); + return cache.get(OP_CLEAR_NS_QUOTA); } @Override @@ -2253,7 +2253,7 @@ static class SetQuotaOp extends FSEditLogOp { } static SetQuotaOp getInstance(OpInstanceCache cache) { - return (SetQuotaOp)cache.get(OP_SET_QUOTA); + return cache.get(OP_SET_QUOTA); } @Override @@ -2338,7 +2338,7 @@ static class SetQuotaByStorageTypeOp extends FSEditLogOp { } static SetQuotaByStorageTypeOp getInstance(OpInstanceCache cache) { - return (SetQuotaByStorageTypeOp)cache.get(OP_SET_QUOTA_BY_STORAGETYPE); + return cache.get(OP_SET_QUOTA_BY_STORAGETYPE); } @Override @@ -2421,7 +2421,7 @@ static class TimesOp extends FSEditLogOp { } static TimesOp getInstance(OpInstanceCache cache) { - return (TimesOp)cache.get(OP_TIMES); + return cache.get(OP_TIMES); } @Override @@ -2530,7 +2530,7 @@ static class SymlinkOp extends FSEditLogOp { } static SymlinkOp getInstance(OpInstanceCache cache) { - return (SymlinkOp)cache.get(OP_SYMLINK); + return cache.get(OP_SYMLINK); } @Override @@ -2689,7 +2689,7 @@ static class RenameOp extends FSEditLogOp { } static RenameOp getInstance(OpInstanceCache cache) { - return (RenameOp)cache.get(OP_RENAME); + return cache.get(OP_RENAME); } @Override @@ -2854,7 +2854,7 @@ static class TruncateOp extends FSEditLogOp { } static TruncateOp getInstance(OpInstanceCache cache) { - return (TruncateOp)cache.get(OP_TRUNCATE); + return cache.get(OP_TRUNCATE); } @Override @@ -2988,7 +2988,7 @@ static class ReassignLeaseOp extends FSEditLogOp { } static ReassignLeaseOp getInstance(OpInstanceCache cache) { - return (ReassignLeaseOp)cache.get(OP_REASSIGN_LEASE); + return cache.get(OP_REASSIGN_LEASE); } @Override @@ -3070,7 +3070,7 @@ static class GetDelegationTokenOp extends FSEditLogOp { } static GetDelegationTokenOp getInstance(OpInstanceCache cache) { - return (GetDelegationTokenOp)cache.get(OP_GET_DELEGATION_TOKEN); + return cache.get(OP_GET_DELEGATION_TOKEN); } @Override @@ -3149,7 +3149,7 @@ static class RenewDelegationTokenOp extends FSEditLogOp { } static RenewDelegationTokenOp getInstance(OpInstanceCache cache) { - return (RenewDelegationTokenOp)cache.get(OP_RENEW_DELEGATION_TOKEN); + return cache.get(OP_RENEW_DELEGATION_TOKEN); } @Override @@ -3227,7 +3227,7 @@ static class CancelDelegationTokenOp extends FSEditLogOp { } static CancelDelegationTokenOp getInstance(OpInstanceCache cache) { - return (CancelDelegationTokenOp)cache.get(OP_CANCEL_DELEGATION_TOKEN); + return cache.get(OP_CANCEL_DELEGATION_TOKEN); } @Override @@ -3286,7 +3286,7 @@ static class UpdateMasterKeyOp extends FSEditLogOp { } static UpdateMasterKeyOp getInstance(OpInstanceCache cache) { - return (UpdateMasterKeyOp)cache.get(OP_UPDATE_MASTER_KEY); + return cache.get(OP_UPDATE_MASTER_KEY); } @Override @@ -3345,7 +3345,7 @@ private LogSegmentOp(FSEditLogOpCodes code) { static LogSegmentOp getInstance(OpInstanceCache cache, FSEditLogOpCodes code) { - return (LogSegmentOp)cache.get(code); + return cache.get(code); } @Override @@ -3404,7 +3404,7 @@ static class InvalidOp extends FSEditLogOp { } static InvalidOp getInstance(OpInstanceCache cache) { - return (InvalidOp)cache.get(OP_INVALID); + return cache.get(OP_INVALID); } @Override @@ -3457,7 +3457,7 @@ public CreateSnapshotOp() { } static CreateSnapshotOp getInstance(OpInstanceCache cache) { - return (CreateSnapshotOp)cache.get(OP_CREATE_SNAPSHOT); + return cache.get(OP_CREATE_SNAPSHOT); } @Override @@ -3562,7 +3562,7 @@ static class DeleteSnapshotOp extends FSEditLogOp { } static DeleteSnapshotOp getInstance(OpInstanceCache cache) { - return (DeleteSnapshotOp)cache.get(OP_DELETE_SNAPSHOT); + return cache.get(OP_DELETE_SNAPSHOT); } @Override @@ -3669,7 +3669,7 @@ static class RenameSnapshotOp extends FSEditLogOp { } static RenameSnapshotOp getInstance(OpInstanceCache cache) { - return (RenameSnapshotOp) cache.get(OP_RENAME_SNAPSHOT); + return cache.get(OP_RENAME_SNAPSHOT); } @Override @@ -3788,7 +3788,7 @@ public AllowSnapshotOp(String snapRoot) { } static AllowSnapshotOp getInstance(OpInstanceCache cache) { - return (AllowSnapshotOp) cache.get(OP_ALLOW_SNAPSHOT); + return cache.get(OP_ALLOW_SNAPSHOT); } @Override @@ -3847,7 +3847,7 @@ public DisallowSnapshotOp(String snapRoot) { } static DisallowSnapshotOp getInstance(OpInstanceCache cache) { - return (DisallowSnapshotOp) cache.get(OP_DISALLOW_SNAPSHOT); + return cache.get(OP_DISALLOW_SNAPSHOT); } void resetSubFields() { @@ -3901,7 +3901,7 @@ public AddCacheDirectiveInfoOp() { } static AddCacheDirectiveInfoOp getInstance(OpInstanceCache cache) { - return (AddCacheDirectiveInfoOp) cache.get(OP_ADD_CACHE_DIRECTIVE); + return cache.get(OP_ADD_CACHE_DIRECTIVE); } @Override @@ -3971,7 +3971,7 @@ public ModifyCacheDirectiveInfoOp() { } static ModifyCacheDirectiveInfoOp getInstance(OpInstanceCache cache) { - return (ModifyCacheDirectiveInfoOp) cache.get(OP_MODIFY_CACHE_DIRECTIVE); + return cache.get(OP_MODIFY_CACHE_DIRECTIVE); } @Override @@ -4047,7 +4047,7 @@ public RemoveCacheDirectiveInfoOp() { } static RemoveCacheDirectiveInfoOp getInstance(OpInstanceCache cache) { - return (RemoveCacheDirectiveInfoOp) cache.get(OP_REMOVE_CACHE_DIRECTIVE); + return cache.get(OP_REMOVE_CACHE_DIRECTIVE); } @Override @@ -4104,7 +4104,7 @@ public AddCachePoolOp() { } static AddCachePoolOp getInstance(OpInstanceCache cache) { - return (AddCachePoolOp) cache.get(OP_ADD_CACHE_POOL); + return cache.get(OP_ADD_CACHE_POOL); } @Override @@ -4170,7 +4170,7 @@ public ModifyCachePoolOp() { } static ModifyCachePoolOp getInstance(OpInstanceCache cache) { - return (ModifyCachePoolOp) cache.get(OP_MODIFY_CACHE_POOL); + return cache.get(OP_MODIFY_CACHE_POOL); } @Override @@ -4243,7 +4243,7 @@ public RemoveCachePoolOp() { } static RemoveCachePoolOp getInstance(OpInstanceCache cache) { - return (RemoveCachePoolOp) cache.get(OP_REMOVE_CACHE_POOL); + return cache.get(OP_REMOVE_CACHE_POOL); } @Override @@ -4300,7 +4300,7 @@ static class RemoveXAttrOp extends FSEditLogOp { } static RemoveXAttrOp getInstance(OpInstanceCache cache) { - return (RemoveXAttrOp) cache.get(OP_REMOVE_XATTR); + return cache.get(OP_REMOVE_XATTR); } @Override @@ -4353,7 +4353,7 @@ static class SetXAttrOp extends FSEditLogOp { } static SetXAttrOp getInstance(OpInstanceCache cache) { - return (SetXAttrOp) cache.get(OP_SET_XATTR); + return cache.get(OP_SET_XATTR); } @Override @@ -4406,7 +4406,7 @@ static class SetAclOp extends FSEditLogOp { } static SetAclOp getInstance(OpInstanceCache cache) { - return (SetAclOp) cache.get(OP_SET_ACL); + return cache.get(OP_SET_ACL); } @Override @@ -4507,8 +4507,7 @@ static class AddErasureCodingPolicyOp extends FSEditLogOp { } static AddErasureCodingPolicyOp getInstance(OpInstanceCache cache) { - return (AddErasureCodingPolicyOp) cache - .get(OP_ADD_ERASURE_CODING_POLICY); + return cache.get(OP_ADD_ERASURE_CODING_POLICY); } @Override @@ -4620,8 +4619,7 @@ static class EnableErasureCodingPolicyOp extends FSEditLogOp { } static EnableErasureCodingPolicyOp getInstance(OpInstanceCache cache) { - return (EnableErasureCodingPolicyOp) cache - .get(OP_ENABLE_ERASURE_CODING_POLICY); + return cache.get(OP_ENABLE_ERASURE_CODING_POLICY); } @Override @@ -4689,8 +4687,7 @@ static class DisableErasureCodingPolicyOp extends FSEditLogOp { } static DisableErasureCodingPolicyOp getInstance(OpInstanceCache cache) { - return (DisableErasureCodingPolicyOp) cache - .get(OP_DISABLE_ERASURE_CODING_POLICY); + return cache.get(OP_DISABLE_ERASURE_CODING_POLICY); } @Override @@ -4756,8 +4753,7 @@ static class RemoveErasureCodingPolicyOp extends FSEditLogOp { } static RemoveErasureCodingPolicyOp getInstance(OpInstanceCache cache) { - return (RemoveErasureCodingPolicyOp) cache - .get(OP_REMOVE_ERASURE_CODING_POLICY); + return cache.get(OP_REMOVE_ERASURE_CODING_POLICY); } @Override @@ -4879,7 +4875,7 @@ static class SetStoragePolicyOp extends FSEditLogOp { } static SetStoragePolicyOp getInstance(OpInstanceCache cache) { - return (SetStoragePolicyOp) cache.get(OP_SET_STORAGE_POLICY); + return cache.get(OP_SET_STORAGE_POLICY); } @Override @@ -4946,7 +4942,7 @@ static class RollingUpgradeStartOp extends RollingUpgradeOp { } static RollingUpgradeStartOp getInstance(OpInstanceCache cache) { - return (RollingUpgradeStartOp) cache.get(OP_ROLLING_UPGRADE_START); + return cache.get(OP_ROLLING_UPGRADE_START); } } @@ -4956,7 +4952,7 @@ static class RollingUpgradeFinalizeOp extends RollingUpgradeOp { } static RollingUpgradeFinalizeOp getInstance(OpInstanceCache cache) { - return (RollingUpgradeFinalizeOp) cache.get(OP_ROLLING_UPGRADE_FINALIZE); + return cache.get(OP_ROLLING_UPGRADE_FINALIZE); } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java index 7302596dc6028..c149d6021bbb8 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java @@ -649,7 +649,7 @@ void openEditLogForWrite(int layoutVersion) throws IOException { */ void reloadFromImageFile(File file, FSNamesystem target) throws IOException { target.clear(); - LOG.debug("Reloading namespace from " + file); + LOG.debug("Reloading namespace from {}.", file); loadFSImage(file, target, null, false); } @@ -728,7 +728,7 @@ LayoutVersion.Feature.TXID_BASED_LAYOUT, getLayoutVersion())) { } for (EditLogInputStream l : editStreams) { - LOG.debug("Planning to load edit log stream: " + l); + LOG.debug("Planning to load edit log stream: {}.", l); } if (!editStreams.iterator().hasNext()) { LOG.info("No edit log streams selected."); @@ -892,8 +892,10 @@ public long loadEdits(Iterable editStreams, FSNamesystem target, long maxTxnsToRead, StartupOption startOpt, MetaRecoveryContext recovery) throws IOException { - LOG.debug("About to load edits:\n " + Joiner.on("\n ").join(editStreams)); - + if (LOG.isDebugEnabled()) { + LOG.debug("About to load edits:\n {}.", Joiner.on("\n ").join(editStreams)); + } + long prevLastAppliedTxId = lastAppliedTxId; long remainingReadTxns = maxTxnsToRead; try { @@ -1348,10 +1350,10 @@ private void renameImageFileInDir(StorageDirectory sd, NameNodeFile fromNnf, final File fromFile = NNStorage.getStorageFile(sd, fromNnf, txid); final File toFile = NNStorage.getStorageFile(sd, toNnf, txid); // renameTo fails on Windows if the destination file already exists. - if(LOG.isDebugEnabled()) { - LOG.debug("renaming " + fromFile.getAbsolutePath() - + " to " + toFile.getAbsolutePath()); + if (LOG.isDebugEnabled()) { + LOG.debug("renaming {} to {}", fromFile.getAbsoluteFile(), toFile.getAbsolutePath()); } + if (!fromFile.renameTo(toFile)) { if (!toFile.delete() || !fromFile.renameTo(toFile)) { throw new IOException("renaming " + fromFile.getAbsolutePath() + " to " + diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java index 13894b4fecf8c..99f2089fe8d52 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java @@ -103,6 +103,7 @@ import org.apache.commons.text.CaseUtils; import org.apache.hadoop.hdfs.protocol.ECTopologyVerifierResult; import org.apache.hadoop.hdfs.protocol.HdfsConstants; +import org.apache.hadoop.hdfs.protocol.HdfsLocatedFileStatus; import org.apache.hadoop.hdfs.protocol.SnapshotStatus; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_STORAGE_POLICY_ENABLED_KEY; import static org.apache.hadoop.hdfs.server.namenode.FSDirStatAndListingOp.*; @@ -416,7 +417,7 @@ void logAuditEvent(boolean succeeded, String cmd, String src) private void logAuditEvent(boolean succeeded, String cmd, String src, String dst, FileStatus stat) throws IOException { if (isAuditEnabled() && isExternalInvocation()) { - logAuditEvent(succeeded, Server.getRemoteUser(), Server.getRemoteIp(), + logAuditEvent(succeeded, NameNode.getRemoteUser(), Server.getRemoteIp(), cmd, src, dst, stat); } } @@ -1407,8 +1408,8 @@ void startActiveServices() throws IOException { } if (LOG.isDebugEnabled()) { - LOG.debug("NameNode metadata after re-processing " + - "replication and invalidation queues during failover:\n" + + LOG.debug("NameNode metadata after re-processing {}" + + "replication and invalidation queues during failover:\n", metaSaveAsString()); } @@ -2202,14 +2203,8 @@ LocatedBlocks getBlockLocations(String clientMachine, String srcArg, } } } - } else if (haEnabled && haContext != null && - haContext.getState().getServiceState() == OBSERVER) { - for (LocatedBlock b : res.blocks.getLocatedBlocks()) { - if (b.getLocations() == null || b.getLocations().length == 0) { - throw new ObserverRetryOnActiveException("Zero blocklocations " - + "for " + srcArg); - } - } + } else if (isObserver()) { + checkBlockLocationsWhenObserver(res.blocks, srcArg); } } finally { readUnlock(operationName, getLockReportInfoSupplier(srcArg)); @@ -2675,8 +2670,8 @@ CryptoProtocolVersion chooseProtocolVersion( for (CryptoProtocolVersion c : supportedVersions) { if (c.equals(CryptoProtocolVersion.UNKNOWN)) { - LOG.debug("Ignoring unknown CryptoProtocolVersion provided by " + - "client: {}", c.getUnknownValue()); + LOG.debug("Ignoring unknown CryptoProtocolVersion provided by client: {}", + c.getUnknownValue()); continue; } if (c.equals(required)) { @@ -2987,8 +2982,7 @@ LastBlockWithStatus appendFile(String srcArg, String holder, requireEffectiveLayoutVersionForFeature(Feature.APPEND_NEW_BLOCK); } - NameNode.stateChangeLog.debug( - "DIR* NameSystem.appendFile: src={}, holder={}, clientMachine={}", + NameNode.stateChangeLog.debug("DIR* NameSystem.appendFile: src={}, holder={}, clientMachine={}", srcArg, holder, clientMachine); try { boolean skipSync = false; @@ -3045,8 +3039,8 @@ LocatedBlock getAdditionalBlock( DatanodeInfo[] excludedNodes, String[] favoredNodes, EnumSet flags) throws IOException { final String operationName = "getAdditionalBlock"; - NameNode.stateChangeLog.debug("BLOCK* getAdditionalBlock: {} inodeId {}" + - " for {}", src, fileId, clientName); + NameNode.stateChangeLog.debug("BLOCK* getAdditionalBlock: {} inodeId {} for {}", + src, fileId, clientName); LocatedBlock[] onRetryBlock = new LocatedBlock[1]; FSDirWriteFileOp.ValidateAddBlockResult r; @@ -3148,8 +3142,7 @@ LocatedBlock getAdditionalDatanode(String src, long fileId, */ void abandonBlock(ExtendedBlock b, long fileId, String src, String holder) throws IOException { - NameNode.stateChangeLog.debug( - "BLOCK* NameSystem.abandonBlock: {} of file {}", b, src); + NameNode.stateChangeLog.debug("BLOCK* NameSystem.abandonBlock: {} of file {}", b, src); checkOperation(OperationCategory.WRITE); final FSPermissionChecker pc = getPermissionChecker(); FSPermissionChecker.setOperationType(null); @@ -3158,8 +3151,8 @@ void abandonBlock(ExtendedBlock b, long fileId, String src, String holder) checkOperation(OperationCategory.WRITE); checkNameNodeSafeMode("Cannot abandon block " + b + " for file" + src); FSDirWriteFileOp.abandonBlock(dir, pc, b, fileId, src, holder); - NameNode.stateChangeLog.debug("BLOCK* NameSystem.abandonBlock: {} is " + - "removed from pendingCreates", b); + NameNode.stateChangeLog.debug( + "BLOCK* NameSystem.abandonBlock: {} is removed from pendingCreates", b); } finally { writeUnlock("abandonBlock"); } @@ -3472,6 +3465,10 @@ HdfsFileStatus getFileInfo(final String src, boolean resolveLink, logAuditEvent(false, operationName, src); throw e; } + if (needLocation && isObserver() && stat instanceof HdfsLocatedFileStatus) { + LocatedBlocks lbs = ((HdfsLocatedFileStatus) stat).getLocatedBlocks(); + checkBlockLocationsWhenObserver(lbs, src); + } logAuditEvent(true, operationName, src); return stat; } @@ -4016,9 +4013,10 @@ void commitBlockSynchronization(ExtendedBlock oldBlock, if ((!iFile.isUnderConstruction() || storedBlock.isComplete()) && iFile.getLastBlock().isComplete()) { if (LOG.isDebugEnabled()) { - LOG.debug("Unexpected block (={}) since the file (={}) is not " - + "under construction", oldBlock, iFile.getLocalName()); + LOG.debug("Unexpected block (={}) since the file (={}) is not under construction", + oldBlock, iFile.getLocalName()); } + return; } @@ -4176,6 +4174,14 @@ DirectoryListing getListing(String src, byte[] startAfter, logAuditEvent(false, operationName, src); throw e; } + if (needLocation && isObserver()) { + for (HdfsFileStatus fs : dl.getPartialListing()) { + if (fs instanceof HdfsLocatedFileStatus) { + LocatedBlocks lbs = ((HdfsLocatedFileStatus) fs).getLocatedBlocks(); + checkBlockLocationsWhenObserver(lbs, fs.toString()); + } + } + } logAuditEvent(true, operationName, src); return dl; } @@ -4478,8 +4484,8 @@ private void closeFile(String path, INodeFile file) { assert hasWriteLock(); // file is closed getEditLog().logCloseFile(path, file); - NameNode.stateChangeLog.debug("closeFile: {} with {} blocks is persisted" + - " to the file system", path, file.getBlocks().length); + NameNode.stateChangeLog.debug("closeFile: {} with {} blocks is persisted to the file system", + path, file.getBlocks().length); } /** @@ -6107,9 +6113,7 @@ Collection listCorruptFileBlocks(String path, if (cookieTab[0] == null) { cookieTab[0] = String.valueOf(getIntCookie(cookieTab[0])); } - if (LOG.isDebugEnabled()) { - LOG.debug("there are no corrupt file blocks."); - } + LOG.debug("there are no corrupt file blocks."); return corruptFiles; } @@ -9023,4 +9027,17 @@ public void checkErasureCodingSupported(String operationName) throw new UnsupportedActionException(operationName + " not supported."); } } + + private boolean isObserver() { + return haEnabled && haContext != null && haContext.getState().getServiceState() == OBSERVER; + } + + private void checkBlockLocationsWhenObserver(LocatedBlocks blocks, String src) + throws ObserverRetryOnActiveException { + for (LocatedBlock b : blocks.getLocatedBlocks()) { + if (b.getLocations() == null || b.getLocations().length == 0) { + throw new ObserverRetryOnActiveException("Zero blocklocations for " + src); + } + } + } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java index 0610048ccacb4..3d3b65d8e217b 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java @@ -17,7 +17,10 @@ */ package org.apache.hadoop.hdfs.server.namenode; +import org.apache.commons.lang3.ArrayUtils; +import org.apache.commons.lang3.tuple.Pair; import org.apache.hadoop.classification.VisibleForTesting; +import org.apache.hadoop.ipc.CallerContext; import org.apache.hadoop.thirdparty.com.google.common.base.Joiner; import org.apache.hadoop.util.Preconditions; @@ -200,6 +203,8 @@ import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_BLOCKPLACEMENTPOLICY_EXCLUDE_SLOW_NODES_ENABLED_DEFAULT; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_MAX_SLOWPEER_COLLECT_NODES_KEY; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_MAX_SLOWPEER_COLLECT_NODES_DEFAULT; +import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_RECONSTRUCTION_PENDING_TIMEOUT_SEC_KEY; +import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_RECONSTRUCTION_PENDING_TIMEOUT_SEC_DEFAULT; import static org.apache.hadoop.util.ExitUtil.terminate; import static org.apache.hadoop.util.ToolRunner.confirmPrompt; @@ -347,7 +352,8 @@ public enum OperationCategory { DFS_NAMENODE_MAX_SLOWPEER_COLLECT_NODES_KEY, DFS_BLOCK_INVALIDATE_LIMIT_KEY, DFS_DATANODE_PEER_STATS_ENABLED_KEY, - DFS_DATANODE_MAX_NODES_TO_REPORT_KEY)); + DFS_DATANODE_MAX_NODES_TO_REPORT_KEY, + DFS_NAMENODE_RECONSTRUCTION_PENDING_TIMEOUT_SEC_KEY)); private static final String USAGE = "Usage: hdfs namenode [" + StartupOption.BACKUP.getName() + "] | \n\t[" @@ -494,6 +500,97 @@ public static NameNodeMetrics getNameNodeMetrics() { return metrics; } + /** + * Try to obtain the actual client info according to the current user. + * @param ipProxyUsers Users who can override client infos + */ + private static String clientInfoFromContext( + final String[] ipProxyUsers) { + if (ipProxyUsers != null) { + UserGroupInformation user = + UserGroupInformation.getRealUserOrSelf(Server.getRemoteUser()); + if (user != null && + ArrayUtils.contains(ipProxyUsers, user.getShortUserName())) { + CallerContext context = CallerContext.getCurrent(); + if (context != null && context.isContextValid()) { + return context.getContext(); + } + } + } + return null; + } + + /** + * Try to obtain the value corresponding to the key by parsing the content. + * @param content the full content to be parsed. + * @param key trying to obtain the value of the key. + * @return the value corresponding to the key. + */ + @VisibleForTesting + public static String parseSpecialValue(String content, String key) { + int posn = content.indexOf(key); + if (posn != -1) { + posn += key.length(); + int end = content.indexOf(",", posn); + return end == -1 ? content.substring(posn) + : content.substring(posn, end); + } + return null; + } + + /** + * Try to obtain the actual client's machine according to the current user. + * @param ipProxyUsers Users who can override client infos. + * @return The actual client's machine. + */ + public static String getClientMachine(final String[] ipProxyUsers) { + String clientMachine = null; + String cc = clientInfoFromContext(ipProxyUsers); + if (cc != null) { + // if the rpc has a caller context of "clientIp:1.2.3.4,CLI", + // return "1.2.3.4" as the client machine. + String key = CallerContext.CLIENT_IP_STR + + CallerContext.Builder.KEY_VALUE_SEPARATOR; + clientMachine = parseSpecialValue(cc, key); + } + + if (clientMachine == null) { + clientMachine = Server.getRemoteAddress(); + if (clientMachine == null) { //not a RPC client + clientMachine = ""; + } + } + return clientMachine; + } + + /** + * Try to obtain the actual client's id and call id + * according to the current user. + * @param ipProxyUsers Users who can override client infos + * @return The actual client's id and call id. + */ + public static Pair getClientIdAndCallId( + final String[] ipProxyUsers) { + byte[] clientId = Server.getClientId(); + int callId = Server.getCallId(); + String cc = clientInfoFromContext(ipProxyUsers); + if (cc != null) { + String clientIdKey = CallerContext.CLIENT_ID_STR + + CallerContext.Builder.KEY_VALUE_SEPARATOR; + String clientIdStr = parseSpecialValue(cc, clientIdKey); + if (clientIdStr != null) { + clientId = StringUtils.hexStringToByte(clientIdStr); + } + String callIdKey = CallerContext.CLIENT_CALL_ID_STR + + CallerContext.Builder.KEY_VALUE_SEPARATOR; + String callIdStr = parseSpecialValue(cc, callIdKey); + if (callIdStr != null) { + callId = Integer.parseInt(callIdStr); + } + } + return Pair.of(clientId, callId); + } + /** * Returns object used for reporting namenode startup progress. * @@ -2207,7 +2304,8 @@ protected String reconfigurePropertyImpl(String property, String newVal) } else if (property.equals(DFS_NAMENODE_REPLICATION_MAX_STREAMS_KEY) || property.equals(DFS_NAMENODE_REPLICATION_STREAMS_HARD_LIMIT_KEY) || property.equals( - DFS_NAMENODE_REPLICATION_WORK_MULTIPLIER_PER_ITERATION)) { + DFS_NAMENODE_REPLICATION_WORK_MULTIPLIER_PER_ITERATION) + || property.equals(DFS_NAMENODE_RECONSTRUCTION_PENDING_TIMEOUT_SEC_KEY)) { return reconfReplicationParameters(newVal, property); } else if (property.equals(DFS_BLOCK_REPLICATOR_CLASSNAME_KEY) || property .equals(DFS_BLOCK_PLACEMENT_EC_CLASSNAME_KEY)) { @@ -2253,6 +2351,14 @@ private String reconfReplicationParameters(final String newVal, DFS_NAMENODE_REPLICATION_WORK_MULTIPLIER_PER_ITERATION_DEFAULT, newVal)); newSetting = bm.getBlocksReplWorkMultiplier(); + } else if ( + property.equals( + DFS_NAMENODE_RECONSTRUCTION_PENDING_TIMEOUT_SEC_KEY)) { + bm.setReconstructionPendingTimeout( + adjustNewVal( + DFS_NAMENODE_RECONSTRUCTION_PENDING_TIMEOUT_SEC_DEFAULT, + newVal)); + newSetting = bm.getReconstructionPendingTimeout(); } else { throw new IllegalArgumentException("Unexpected property " + property + " in reconfReplicationParameters"); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java index c86211b058bdd..b19bfc13acf65 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java @@ -46,8 +46,7 @@ import java.util.Map; import java.util.Set; -import org.apache.commons.lang3.ArrayUtils; -import org.apache.hadoop.ipc.CallerContext; +import org.apache.commons.lang3.tuple.Pair; import org.apache.hadoop.HadoopIllegalArgumentException; import org.apache.hadoop.classification.InterfaceAudience; @@ -271,7 +270,7 @@ public class NameNodeRpcServer implements NamenodeProtocols { private final String defaultECPolicyName; - // Users who can override the client ip + // Users who can override the client info private final String[] ipProxyUsers; public NameNodeRpcServer(Configuration conf, NameNode nn) @@ -711,8 +710,7 @@ public NamenodeCommand startCheckpoint(NamenodeRegistration registration) if(!nn.isRole(NamenodeRole.NAMENODE)) throw new IOException("Only an ACTIVE node can invoke startCheckpoint."); - CacheEntryWithPayload cacheEntry = RetryCache.waitForCompletion(retryCache, - null); + CacheEntryWithPayload cacheEntry = getCacheEntryWithPayload(null); if (cacheEntry != null && cacheEntry.isSuccess()) { return (NamenodeCommand) cacheEntry.getPayload(); } @@ -725,13 +723,33 @@ public NamenodeCommand startCheckpoint(NamenodeRegistration registration) return ret; } + /** + * Return the current CacheEntry. + */ + private CacheEntry getCacheEntry() { + Pair clientInfo = + NameNode.getClientIdAndCallId(this.ipProxyUsers); + return RetryCache.waitForCompletion( + retryCache, clientInfo.getLeft(), clientInfo.getRight()); + } + + /** + * Return the current CacheEntryWithPayload. + */ + private CacheEntryWithPayload getCacheEntryWithPayload(Object payload) { + Pair clientInfo = + NameNode.getClientIdAndCallId(this.ipProxyUsers); + return RetryCache.waitForCompletion(retryCache, payload, + clientInfo.getLeft(), clientInfo.getRight()); + } + @Override // NamenodeProtocol public void endCheckpoint(NamenodeRegistration registration, CheckpointSignature sig) throws IOException { String operationName = "endCheckpoint"; checkNNStartup(); namesystem.checkSuperuserPrivilege(operationName); - CacheEntry cacheEntry = RetryCache.waitForCompletion(retryCache); + CacheEntry cacheEntry = getCacheEntry(); if (cacheEntry != null && cacheEntry.isSuccess()) { return; // Return previous response } @@ -792,16 +810,14 @@ public HdfsFileStatus create(String src, FsPermission masked, throws IOException { checkNNStartup(); String clientMachine = getClientMachine(); - if (stateChangeLog.isDebugEnabled()) { - stateChangeLog.debug("*DIR* NameNode.create: file " - +src+" for "+clientName+" at "+clientMachine); - } + stateChangeLog.debug("*DIR* NameNode.create: file {} for {} at {}.", + src, clientName, clientMachine); if (!checkPathLength(src)) { throw new IOException("create: Pathname too long. Limit " + MAX_PATH_LENGTH + " characters, " + MAX_PATH_DEPTH + " levels."); } namesystem.checkOperation(OperationCategory.WRITE); - CacheEntryWithPayload cacheEntry = RetryCache.waitForCompletion(retryCache, null); + CacheEntryWithPayload cacheEntry = getCacheEntryWithPayload(null); if (cacheEntry != null && cacheEntry.isSuccess()) { return (HdfsFileStatus) cacheEntry.getPayload(); } @@ -827,13 +843,10 @@ public LastBlockWithStatus append(String src, String clientName, EnumSetWritable flag) throws IOException { checkNNStartup(); String clientMachine = getClientMachine(); - if (stateChangeLog.isDebugEnabled()) { - stateChangeLog.debug("*DIR* NameNode.append: file " - +src+" for "+clientName+" at "+clientMachine); - } + stateChangeLog.debug("*DIR* NameNode.append: file {} for {} at {}.", + src, clientName, clientMachine); namesystem.checkOperation(OperationCategory.WRITE); - CacheEntryWithPayload cacheEntry = RetryCache.waitForCompletion(retryCache, - null); + CacheEntryWithPayload cacheEntry = getCacheEntryWithPayload(null); if (cacheEntry != null && cacheEntry.isSuccess()) { return (LastBlockWithStatus) cacheEntry.getPayload(); } @@ -877,8 +890,8 @@ public void unsetStoragePolicy(String src) public void setStoragePolicy(String src, String policyName) throws IOException { checkNNStartup(); - stateChangeLog.debug("*DIR* NameNode.setStoragePolicy for path: {}, " + - "policyName: {}", src, policyName); + stateChangeLog.debug("*DIR* NameNode.setStoragePolicy for path: {}, policyName: {}", + src, policyName); namesystem.setStoragePolicy(src, policyName); } @@ -932,13 +945,9 @@ public LocatedBlock getAdditionalDatanode(final String src, ) throws IOException { checkNNStartup(); if (LOG.isDebugEnabled()) { - LOG.debug("getAdditionalDatanode: src=" + src - + ", fileId=" + fileId - + ", blk=" + blk - + ", existings=" + Arrays.asList(existings) - + ", excludes=" + Arrays.asList(excludes) - + ", numAdditionalNodes=" + numAdditionalNodes - + ", clientName=" + clientName); + LOG.debug("getAdditionalDatanode: src={}, fileId={}, blk={}, existings={}, excludes={}" + + ", numAdditionalNodes={}, clientName={}", src, fileId, blk, Arrays.asList(existings), + Arrays.asList(excludes), numAdditionalNodes, clientName); } metrics.incrGetAdditionalDatanodeOps(); @@ -999,7 +1008,7 @@ public void updatePipeline(String clientName, ExtendedBlock oldBlock, throws IOException { checkNNStartup(); namesystem.checkOperation(OperationCategory.WRITE); - CacheEntry cacheEntry = RetryCache.waitForCompletion(retryCache); + CacheEntry cacheEntry = getCacheEntry(); if (cacheEntry != null && cacheEntry.isSuccess()) { return; // Return previous response } @@ -1036,15 +1045,13 @@ public long getPreferredBlockSize(String filename) @Override // ClientProtocol public boolean rename(String src, String dst) throws IOException { checkNNStartup(); - if(stateChangeLog.isDebugEnabled()) { - stateChangeLog.debug("*DIR* NameNode.rename: " + src + " to " + dst); - } + stateChangeLog.debug("*DIR* NameNode.rename: {} to {}.", src, dst); if (!checkPathLength(dst)) { throw new IOException("rename: Pathname too long. Limit " + MAX_PATH_LENGTH + " characters, " + MAX_PATH_DEPTH + " levels."); } namesystem.checkOperation(OperationCategory.WRITE); - CacheEntry cacheEntry = RetryCache.waitForCompletion(retryCache); + CacheEntry cacheEntry = getCacheEntry(); if (cacheEntry != null && cacheEntry.isSuccess()) { return true; // Return previous response } @@ -1064,10 +1071,12 @@ public boolean rename(String src, String dst) throws IOException { @Override // ClientProtocol public void concat(String trg, String[] src) throws IOException { checkNNStartup(); - stateChangeLog.debug("*DIR* NameNode.concat: src path {} to" + - " target path {}", Arrays.toString(src), trg); + if (stateChangeLog.isDebugEnabled()) { + stateChangeLog.debug("*DIR* NameNode.concat: src path {} to target path {}", + Arrays.toString(src), trg); + } namesystem.checkOperation(OperationCategory.WRITE); - CacheEntry cacheEntry = RetryCache.waitForCompletion(retryCache); + CacheEntry cacheEntry = getCacheEntry(); if (cacheEntry != null && cacheEntry.isSuccess()) { return; // Return previous response } @@ -1085,15 +1094,13 @@ public void concat(String trg, String[] src) throws IOException { public void rename2(String src, String dst, Options.Rename... options) throws IOException { checkNNStartup(); - if(stateChangeLog.isDebugEnabled()) { - stateChangeLog.debug("*DIR* NameNode.rename: " + src + " to " + dst); - } + stateChangeLog.debug("*DIR* NameNode.rename: {} to {}.", src, dst); if (!checkPathLength(dst)) { throw new IOException("rename: Pathname too long. Limit " + MAX_PATH_LENGTH + " characters, " + MAX_PATH_DEPTH + " levels."); } namesystem.checkOperation(OperationCategory.WRITE); - CacheEntry cacheEntry = RetryCache.waitForCompletion(retryCache); + CacheEntry cacheEntry = getCacheEntry(); if (cacheEntry != null && cacheEntry.isSuccess()) { return; // Return previous response } @@ -1111,8 +1118,7 @@ public void rename2(String src, String dst, Options.Rename... options) public boolean truncate(String src, long newLength, String clientName) throws IOException { checkNNStartup(); - stateChangeLog - .debug("*DIR* NameNode.truncate: " + src + " to " + newLength); + stateChangeLog.debug("*DIR* NameNode.truncate: {} to {}", src, newLength); String clientMachine = getClientMachine(); try { return namesystem.truncate( @@ -1125,12 +1131,9 @@ public boolean truncate(String src, long newLength, String clientName) @Override // ClientProtocol public boolean delete(String src, boolean recursive) throws IOException { checkNNStartup(); - if (stateChangeLog.isDebugEnabled()) { - stateChangeLog.debug("*DIR* Namenode.delete: src=" + src - + ", recursive=" + recursive); - } + stateChangeLog.debug("*DIR* Namenode.delete: src={}, recursive={}.", src, recursive); namesystem.checkOperation(OperationCategory.WRITE); - CacheEntry cacheEntry = RetryCache.waitForCompletion(retryCache); + CacheEntry cacheEntry = getCacheEntry(); if (cacheEntry != null && cacheEntry.isSuccess()) { return true; // Return previous response } @@ -1161,9 +1164,7 @@ private boolean checkPathLength(String src) { public boolean mkdirs(String src, FsPermission masked, boolean createParent) throws IOException { checkNNStartup(); - if(stateChangeLog.isDebugEnabled()) { - stateChangeLog.debug("*DIR* NameNode.mkdirs: " + src); - } + stateChangeLog.debug("*DIR* NameNode.mkdirs: {}.", src); if (!checkPathLength(src)) { throw new IOException("mkdirs: Pathname too long. Limit " + MAX_PATH_LENGTH + " characters, " + MAX_PATH_DEPTH + " levels."); @@ -1174,7 +1175,14 @@ public boolean mkdirs(String src, FsPermission masked, boolean createParent) } @Override // ClientProtocol - public void renewLease(String clientName) throws IOException { + public void renewLease(String clientName, List namespaces) + throws IOException { + if (namespaces != null && namespaces.size() > 0) { + LOG.warn("namespaces({}) should be null or empty " + + "on NameNode side, please check it.", namespaces); + throw new IOException("namespaces(" + namespaces + + ") should be null or empty"); + } checkNNStartup(); namesystem.renewLease(clientName); } @@ -1308,7 +1316,7 @@ public boolean restoreFailedStorage(String arg) throws IOException { @Override // ClientProtocol public boolean saveNamespace(long timeWindow, long txGap) throws IOException { checkNNStartup(); - CacheEntry cacheEntry = RetryCache.waitForCompletion(retryCache); + CacheEntry cacheEntry = getCacheEntry(); if (cacheEntry != null && cacheEntry.isSuccess()) { return true; // Return previous response } @@ -1496,7 +1504,7 @@ public QuotaUsage getQuotaUsage(String path) throws IOException { public void satisfyStoragePolicy(String src) throws IOException { checkNNStartup(); namesystem.checkOperation(OperationCategory.WRITE); - CacheEntry cacheEntry = RetryCache.waitForCompletion(retryCache); + CacheEntry cacheEntry = getCacheEntry(); if (cacheEntry != null && cacheEntry.isSuccess()) { return; // Return previous response } @@ -1543,7 +1551,7 @@ public void createSymlink(String target, String link, FsPermission dirPerms, boolean createParent) throws IOException { checkNNStartup(); namesystem.checkOperation(OperationCategory.WRITE); - CacheEntry cacheEntry = RetryCache.waitForCompletion(retryCache); + CacheEntry cacheEntry = getCacheEntry(); if (cacheEntry != null && cacheEntry.isSuccess()) { return; // Return previous response } @@ -1625,10 +1633,8 @@ public DatanodeCommand blockReport(final DatanodeRegistration nodeReg, final BlockReportContext context) throws IOException { checkNNStartup(); verifyRequest(nodeReg); - if(blockStateChangeLog.isDebugEnabled()) { - blockStateChangeLog.debug("*BLOCK* NameNode.blockReport: " - + "from " + nodeReg + ", reports.length=" + reports.length); - } + blockStateChangeLog.debug("*BLOCK* NameNode.blockReport: from {}, reports.length={}.", + nodeReg, reports.length); final BlockManager bm = namesystem.getBlockManager(); boolean noStaleStorages = false; try { @@ -1671,10 +1677,8 @@ public DatanodeCommand cacheReport(DatanodeRegistration nodeReg, String poolId, List blockIds) throws IOException { checkNNStartup(); verifyRequest(nodeReg); - if (blockStateChangeLog.isDebugEnabled()) { - blockStateChangeLog.debug("*BLOCK* NameNode.cacheReport: " - + "from " + nodeReg + " " + blockIds.size() + " blocks"); - } + blockStateChangeLog.debug("*BLOCK* NameNode.cacheReport: from {} {} blocks", + nodeReg, blockIds.size()); namesystem.getCacheManager().processCacheReport(nodeReg, blockIds); return null; } @@ -1686,11 +1690,8 @@ public void blockReceivedAndDeleted(final DatanodeRegistration nodeReg, checkNNStartup(); verifyRequest(nodeReg); metrics.incrBlockReceivedAndDeletedOps(); - if(blockStateChangeLog.isDebugEnabled()) { - blockStateChangeLog.debug("*BLOCK* NameNode.blockReceivedAndDeleted: " - +"from "+nodeReg+" "+receivedAndDeletedBlocks.length - +" blocks."); - } + blockStateChangeLog.debug("*BLOCK* NameNode.blockReceivedAndDeleted: from {} {} blocks.", + nodeReg, receivedAndDeletedBlocks.length); final BlockManager bm = namesystem.getBlockManager(); for (final StorageReceivedDeletedBlocks r : receivedAndDeletedBlocks) { bm.enqueueBlockOp(new Runnable() { @@ -1818,9 +1819,7 @@ public Collection refresh(String identifier, String[] args) { @Override // GetUserMappingsProtocol public String[] getGroupsForUser(String user) throws IOException { - if (LOG.isDebugEnabled()) { - LOG.debug("Getting groups for user " + user); - } + LOG.debug("Getting groups for user {}", user); return UserGroupInformation.createRemoteUser(user).getGroupNames(); } @@ -1913,34 +1912,11 @@ private void verifySoftwareVersion(DatanodeRegistration dnReg) } } + /** + * Get the actual client's machine. + */ private String getClientMachine() { - if (ipProxyUsers != null) { - // Get the real user (or effective if it isn't a proxy user) - UserGroupInformation user = - UserGroupInformation.getRealUserOrSelf(Server.getRemoteUser()); - if (user != null && - ArrayUtils.contains(ipProxyUsers, user.getShortUserName())) { - CallerContext context = CallerContext.getCurrent(); - if (context != null && context.isContextValid()) { - String cc = context.getContext(); - // if the rpc has a caller context of "clientIp:1.2.3.4,CLI", - // return "1.2.3.4" as the client machine. - String key = CallerContext.CLIENT_IP_STR + - CallerContext.Builder.KEY_VALUE_SEPARATOR; - int posn = cc.indexOf(key); - if (posn != -1) { - posn += key.length(); - int end = cc.indexOf(",", posn); - return end == -1 ? cc.substring(posn) : cc.substring(posn, end); - } - } - } - } - String clientMachine = Server.getRemoteAddress(); - if (clientMachine == null) { //not a RPC client - clientMachine = ""; - } - return clientMachine; + return NameNode.getClientMachine(this.ipProxyUsers); } @Override @@ -1960,8 +1936,7 @@ public String createSnapshot(String snapshotRoot, String snapshotName) + MAX_PATH_LENGTH + " characters, " + MAX_PATH_DEPTH + " levels."); } namesystem.checkOperation(OperationCategory.WRITE); - CacheEntryWithPayload cacheEntry = RetryCache.waitForCompletion(retryCache, - null); + CacheEntryWithPayload cacheEntry = getCacheEntryWithPayload(null); if (cacheEntry != null && cacheEntry.isSuccess()) { return (String) cacheEntry.getPayload(); } @@ -1988,7 +1963,7 @@ public void deleteSnapshot(String snapshotRoot, String snapshotName) } namesystem.checkOperation(OperationCategory.WRITE); metrics.incrDeleteSnapshotOps(); - CacheEntry cacheEntry = RetryCache.waitForCompletion(retryCache); + CacheEntry cacheEntry = getCacheEntry(); if (cacheEntry != null && cacheEntry.isSuccess()) { return; // Return previous response } @@ -2022,15 +1997,15 @@ public void disallowSnapshot(String snapshot) throws IOException { public void renameSnapshot(String snapshotRoot, String snapshotOldName, String snapshotNewName) throws IOException { checkNNStartup(); - LOG.debug("*DIR* NameNode.renameSnapshot: Snapshot Path {}, " + - "snapshotOldName {}, snapshotNewName {}", snapshotRoot, - snapshotOldName, snapshotNewName); + LOG.debug( + "*DIR* NameNode.renameSnapshot: Snapshot Path {},snapshotOldName {}, snapshotNewName {}", + snapshotRoot, snapshotOldName, snapshotNewName); if (snapshotNewName == null || snapshotNewName.isEmpty()) { throw new IOException("The new snapshot name is null or empty."); } namesystem.checkOperation(OperationCategory.WRITE); metrics.incrRenameSnapshotOps(); - CacheEntry cacheEntry = RetryCache.waitForCompletion(retryCache); + CacheEntry cacheEntry = getCacheEntry(); if (cacheEntry != null && cacheEntry.isSuccess()) { return; // Return previous response } @@ -2091,8 +2066,7 @@ public long addCacheDirective( CacheDirectiveInfo path, EnumSet flags) throws IOException { checkNNStartup(); namesystem.checkOperation(OperationCategory.WRITE); - CacheEntryWithPayload cacheEntry = RetryCache.waitForCompletion - (retryCache, null); + CacheEntryWithPayload cacheEntry = getCacheEntryWithPayload(null); if (cacheEntry != null && cacheEntry.isSuccess()) { return (Long) cacheEntry.getPayload(); } @@ -2113,7 +2087,7 @@ public void modifyCacheDirective( CacheDirectiveInfo directive, EnumSet flags) throws IOException { checkNNStartup(); namesystem.checkOperation(OperationCategory.WRITE); - CacheEntry cacheEntry = RetryCache.waitForCompletion(retryCache); + CacheEntry cacheEntry = getCacheEntry(); if (cacheEntry != null && cacheEntry.isSuccess()) { return; } @@ -2131,7 +2105,7 @@ public void modifyCacheDirective( public void removeCacheDirective(long id) throws IOException { checkNNStartup(); namesystem.checkOperation(OperationCategory.WRITE); - CacheEntry cacheEntry = RetryCache.waitForCompletion(retryCache); + CacheEntry cacheEntry = getCacheEntry(); if (cacheEntry != null && cacheEntry.isSuccess()) { return; } @@ -2158,7 +2132,7 @@ public BatchedEntries listCacheDirectives(long prevId, public void addCachePool(CachePoolInfo info) throws IOException { checkNNStartup(); namesystem.checkOperation(OperationCategory.WRITE); - CacheEntry cacheEntry = RetryCache.waitForCompletion(retryCache); + CacheEntry cacheEntry = getCacheEntry(); if (cacheEntry != null && cacheEntry.isSuccess()) { return; // Return previous response } @@ -2175,7 +2149,7 @@ public void addCachePool(CachePoolInfo info) throws IOException { public void modifyCachePool(CachePoolInfo info) throws IOException { checkNNStartup(); namesystem.checkOperation(OperationCategory.WRITE); - CacheEntry cacheEntry = RetryCache.waitForCompletion(retryCache); + CacheEntry cacheEntry = getCacheEntry(); if (cacheEntry != null && cacheEntry.isSuccess()) { return; // Return previous response } @@ -2192,7 +2166,7 @@ public void modifyCachePool(CachePoolInfo info) throws IOException { public void removeCachePool(String cachePoolName) throws IOException { checkNNStartup(); namesystem.checkOperation(OperationCategory.WRITE); - CacheEntry cacheEntry = RetryCache.waitForCompletion(retryCache); + CacheEntry cacheEntry = getCacheEntry(); if (cacheEntry != null && cacheEntry.isSuccess()) { return; } @@ -2255,7 +2229,7 @@ public void createEncryptionZone(String src, String keyName) throws IOException { checkNNStartup(); namesystem.checkOperation(OperationCategory.WRITE); - final CacheEntry cacheEntry = RetryCache.waitForCompletion(retryCache); + final CacheEntry cacheEntry = getCacheEntry(); if (cacheEntry != null && cacheEntry.isSuccess()) { return; } @@ -2287,7 +2261,7 @@ public void reencryptEncryptionZone(final String zone, final ReencryptAction action) throws IOException { checkNNStartup(); namesystem.checkOperation(OperationCategory.WRITE); - final CacheEntry cacheEntry = RetryCache.waitForCompletion(retryCache); + final CacheEntry cacheEntry = getCacheEntry(); if (cacheEntry != null && cacheEntry.isSuccess()) { return; } @@ -2311,7 +2285,7 @@ public BatchedEntries listReencryptionStatus( public void setErasureCodingPolicy(String src, String ecPolicyName) throws IOException { checkNNStartup(); - final CacheEntry cacheEntry = RetryCache.waitForCompletion(retryCache); + final CacheEntry cacheEntry = getCacheEntry(); if (cacheEntry != null && cacheEntry.isSuccess()) { return; } @@ -2319,8 +2293,7 @@ public void setErasureCodingPolicy(String src, String ecPolicyName) try { if (ecPolicyName == null) { ecPolicyName = defaultECPolicyName; - LOG.debug("No policy name is specified, " + - "set the default policy name instead"); + LOG.debug("No policy name is specified, set the default policy name instead"); } LOG.debug("Set erasure coding policy {} on {}", ecPolicyName, src); namesystem.setErasureCodingPolicy(src, ecPolicyName, cacheEntry != null); @@ -2335,7 +2308,7 @@ public void setXAttr(String src, XAttr xAttr, EnumSet flag) throws IOException { checkNNStartup(); namesystem.checkOperation(OperationCategory.WRITE); - CacheEntry cacheEntry = RetryCache.waitForCompletion(retryCache); + CacheEntry cacheEntry = getCacheEntry(); if (cacheEntry != null && cacheEntry.isSuccess()) { return; // Return previous response } @@ -2365,7 +2338,7 @@ public List listXAttrs(String src) throws IOException { public void removeXAttr(String src, XAttr xAttr) throws IOException { checkNNStartup(); namesystem.checkOperation(OperationCategory.WRITE); - CacheEntry cacheEntry = RetryCache.waitForCompletion(retryCache); + CacheEntry cacheEntry = getCacheEntry(); if (cacheEntry != null && cacheEntry.isSuccess()) { return; // Return previous response } @@ -2548,7 +2521,7 @@ public ErasureCodingPolicy getErasureCodingPolicy(String src) throws IOException @Override // ClientProtocol public void unsetErasureCodingPolicy(String src) throws IOException { checkNNStartup(); - final CacheEntry cacheEntry = RetryCache.waitForCompletion(retryCache); + final CacheEntry cacheEntry = getCacheEntry(); if (cacheEntry != null && cacheEntry.isSuccess()) { return; } @@ -2574,8 +2547,7 @@ public AddErasureCodingPolicyResponse[] addErasureCodingPolicies( String operationName = "addErasureCodingPolicies"; checkNNStartup(); namesystem.checkSuperuserPrivilege(operationName); - final CacheEntryWithPayload cacheEntry = - RetryCache.waitForCompletion(retryCache, null); + final CacheEntryWithPayload cacheEntry = getCacheEntryWithPayload(null); if (cacheEntry != null && cacheEntry.isSuccess()) { return (AddErasureCodingPolicyResponse[]) cacheEntry.getPayload(); } @@ -2598,7 +2570,7 @@ public void removeErasureCodingPolicy(String ecPolicyName) String operationName = "removeErasureCodingPolicy"; checkNNStartup(); namesystem.checkSuperuserPrivilege(operationName); - final CacheEntry cacheEntry = RetryCache.waitForCompletion(retryCache); + final CacheEntry cacheEntry = getCacheEntry(); if (cacheEntry != null && cacheEntry.isSuccess()) { return; } @@ -2617,7 +2589,7 @@ public void enableErasureCodingPolicy(String ecPolicyName) String operationName = "enableErasureCodingPolicy"; checkNNStartup(); namesystem.checkSuperuserPrivilege(operationName); - final CacheEntry cacheEntry = RetryCache.waitForCompletion(retryCache); + final CacheEntry cacheEntry = getCacheEntry(); if (cacheEntry != null && cacheEntry.isSuccess()) { return; } @@ -2636,7 +2608,7 @@ public void disableErasureCodingPolicy(String ecPolicyName) String operationName = "disableErasureCodingPolicy"; checkNNStartup(); namesystem.checkSuperuserPrivilege(operationName); - final CacheEntry cacheEntry = RetryCache.waitForCompletion(retryCache); + final CacheEntry cacheEntry = getCacheEntry(); if (cacheEntry != null && cacheEntry.isSuccess()) { return; } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/EditLogTailer.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/EditLogTailer.java index 25596dce9f51d..f72ec7c9177df 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/EditLogTailer.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/EditLogTailer.java @@ -262,9 +262,7 @@ public EditLogTailer(FSNamesystem namesystem, Configuration conf) { nnCount = nns.size(); // setup the iterator to endlessly loop the nns this.nnLookup = Iterators.cycle(nns); - - LOG.debug("logRollPeriodMs=" + logRollPeriodMs + - " sleepTime=" + sleepTimeMs); + LOG.debug("logRollPeriodMs={} sleepTime={}.", logRollPeriodMs, sleepTimeMs); } public void start() { @@ -328,38 +326,39 @@ public Void run() throws Exception { @VisibleForTesting public long doTailEdits() throws IOException, InterruptedException { + Collection streams; + FSImage image = namesystem.getFSImage(); + + long lastTxnId = image.getLastAppliedTxId(); + LOG.debug("lastTxnId: {}", lastTxnId); + long startTime = timer.monotonicNow(); + try { + streams = editLog.selectInputStreams(lastTxnId + 1, 0, + null, inProgressOk, true); + } catch (IOException ioe) { + // This is acceptable. If we try to tail edits in the middle of an edits + // log roll, i.e. the last one has been finalized but the new inprogress + // edits file hasn't been started yet. + LOG.warn("Edits tailer failed to find any streams. Will try again " + + "later.", ioe); + return 0; + } finally { + NameNode.getNameNodeMetrics().addEditLogFetchTime( + timer.monotonicNow() - startTime); + } // Write lock needs to be interruptible here because the // transitionToActive RPC takes the write lock before calling // tailer.stop() -- so if we're not interruptible, it will // deadlock. namesystem.writeLockInterruptibly(); try { - FSImage image = namesystem.getFSImage(); - - long lastTxnId = image.getLastAppliedTxId(); - - if (LOG.isDebugEnabled()) { - LOG.debug("lastTxnId: " + lastTxnId); - } - Collection streams; - long startTime = timer.monotonicNow(); - try { - streams = editLog.selectInputStreams(lastTxnId + 1, 0, - null, inProgressOk, true); - } catch (IOException ioe) { - // This is acceptable. If we try to tail edits in the middle of an edits - // log roll, i.e. the last one has been finalized but the new inprogress - // edits file hasn't been started yet. - LOG.warn("Edits tailer failed to find any streams. Will try again " + - "later.", ioe); + long currentLastTxnId = image.getLastAppliedTxId(); + if (lastTxnId != currentLastTxnId) { + LOG.warn("The currentLastTxnId({}) is different from preLastTxtId({})", + currentLastTxnId, lastTxnId); return 0; - } finally { - NameNode.getNameNodeMetrics().addEditLogFetchTime( - timer.monotonicNow() - startTime); - } - if (LOG.isDebugEnabled()) { - LOG.debug("edit streams to load from: " + streams.size()); } + LOG.debug("edit streams to load from: {}.", streams.size()); // Once we have streams to load, errors encountered are legitimate cause // for concern, so we don't catch them here. Simple errors reading from @@ -372,10 +371,7 @@ public long doTailEdits() throws IOException, InterruptedException { editsLoaded = elie.getNumEditsLoaded(); throw elie; } finally { - if (editsLoaded > 0 || LOG.isDebugEnabled()) { - LOG.debug(String.format("Loaded %d edits starting from txid %d ", - editsLoaded, lastTxnId)); - } + LOG.debug("Loaded {} edits starting from txid {}.", editsLoaded, lastTxnId); NameNode.getNameNodeMetrics().addNumEditLogLoaded(editsLoaded); } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/BlockECReconstructionCommand.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/BlockECReconstructionCommand.java index b2495c8d6d9d8..3b1e2d608465a 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/BlockECReconstructionCommand.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/BlockECReconstructionCommand.java @@ -78,21 +78,23 @@ public static class BlockECReconstructionInfo { private String[] targetStorageIDs; private StorageType[] targetStorageTypes; private final byte[] liveBlockIndices; + private final byte[] excludeReconstructedIndices; private final ErasureCodingPolicy ecPolicy; public BlockECReconstructionInfo(ExtendedBlock block, DatanodeInfo[] sources, DatanodeStorageInfo[] targetDnStorageInfo, - byte[] liveBlockIndices, ErasureCodingPolicy ecPolicy) { + byte[] liveBlockIndices, byte[] excludeReconstructedIndices, ErasureCodingPolicy ecPolicy) { this(block, sources, DatanodeStorageInfo .toDatanodeInfos(targetDnStorageInfo), DatanodeStorageInfo .toStorageIDs(targetDnStorageInfo), DatanodeStorageInfo - .toStorageTypes(targetDnStorageInfo), liveBlockIndices, ecPolicy); + .toStorageTypes(targetDnStorageInfo), liveBlockIndices, + excludeReconstructedIndices, ecPolicy); } public BlockECReconstructionInfo(ExtendedBlock block, DatanodeInfo[] sources, DatanodeInfo[] targets, String[] targetStorageIDs, StorageType[] targetStorageTypes, - byte[] liveBlockIndices, ErasureCodingPolicy ecPolicy) { + byte[] liveBlockIndices, byte[] excludeReconstructedIndices, ErasureCodingPolicy ecPolicy) { this.block = block; this.sources = sources; this.targets = targets; @@ -100,6 +102,7 @@ public BlockECReconstructionInfo(ExtendedBlock block, this.targetStorageTypes = targetStorageTypes; this.liveBlockIndices = liveBlockIndices == null ? new byte[]{} : liveBlockIndices; + this.excludeReconstructedIndices = excludeReconstructedIndices; this.ecPolicy = ecPolicy; } @@ -127,6 +130,10 @@ public byte[] getLiveBlockIndices() { return liveBlockIndices; } + public byte[] getExcludeReconstructedIndices() { + return excludeReconstructedIndices; + } + public ErasureCodingPolicy getErasureCodingPolicy() { return ecPolicy; } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/FSImageHandler.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/FSImageHandler.java index bfa35aaf1e8be..963406771b18a 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/FSImageHandler.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/FSImageHandler.java @@ -40,10 +40,10 @@ import java.util.List; import java.util.Map; -import static io.netty.handler.codec.http.HttpHeaders.Names.CONNECTION; -import static io.netty.handler.codec.http.HttpHeaders.Names.CONTENT_LENGTH; -import static io.netty.handler.codec.http.HttpHeaders.Names.CONTENT_TYPE; -import static io.netty.handler.codec.http.HttpHeaders.Values.CLOSE; +import static io.netty.handler.codec.http.HttpHeaderNames.CONNECTION; +import static io.netty.handler.codec.http.HttpHeaderNames.CONTENT_LENGTH; +import static io.netty.handler.codec.http.HttpHeaderNames.CONTENT_TYPE; +import static io.netty.handler.codec.http.HttpHeaderValues.CLOSE; import static io.netty.handler.codec.http.HttpResponseStatus.BAD_REQUEST; import static io.netty.handler.codec.http.HttpResponseStatus.FORBIDDEN; import static io.netty.handler.codec.http.HttpResponseStatus.INTERNAL_SERVER_ERROR; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/OfflineImageViewerPB.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/OfflineImageViewerPB.java index 05e687ab97e43..2233a3c3d243c 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/OfflineImageViewerPB.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/OfflineImageViewerPB.java @@ -80,6 +80,7 @@ public class OfflineImageViewerPB { + " delimiter. The default delimiter is \\t, though this may be\n" + " changed via the -delimiter argument.\n" + " -sp print storage policy, used by delimiter only.\n" + + " -ec print erasure coding policy, used by delimiter only.\n" + " * DetectCorruption: Detect potential corruption of the image by\n" + " selectively loading parts of it and actively searching for\n" + " inconsistencies. Outputs a summary of the found corruptions\n" @@ -132,6 +133,7 @@ private static Options buildOptions() { options.addOption("addr", true, ""); options.addOption("delimiter", true, ""); options.addOption("sp", false, ""); + options.addOption("ec", false, ""); options.addOption("t", "temp", true, ""); options.addOption("m", "multiThread", true, ""); @@ -228,9 +230,11 @@ public static int run(String[] args) throws Exception { break; case "DELIMITED": boolean printStoragePolicy = cmd.hasOption("sp"); + boolean printECPolicy = cmd.hasOption("ec"); try (PBImageDelimitedTextWriter writer = new PBImageDelimitedTextWriter(out, delimiter, - tempPath, printStoragePolicy, threads, outputFile)) { + tempPath, printStoragePolicy, printECPolicy, threads, + outputFile, conf)) { writer.visit(inputFile); } break; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/PBImageDelimitedTextWriter.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/PBImageDelimitedTextWriter.java index 3e080ec8e65cd..39fd7658ef54d 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/PBImageDelimitedTextWriter.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/PBImageDelimitedTextWriter.java @@ -17,9 +17,12 @@ */ package org.apache.hadoop.hdfs.tools.offlineImageViewer; +import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.permission.PermissionStatus; +import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy; import org.apache.hadoop.hdfs.protocol.HdfsConstants; +import org.apache.hadoop.hdfs.server.namenode.ErasureCodingPolicyManager; import org.apache.hadoop.hdfs.server.namenode.FsImageProto.INodeSection.INode; import org.apache.hadoop.hdfs.server.namenode.FsImageProto.INodeSection.INodeDirectory; import org.apache.hadoop.hdfs.server.namenode.FsImageProto.INodeSection.INodeFile; @@ -46,6 +49,8 @@ public class PBImageDelimitedTextWriter extends PBImageTextWriter { private static final String DATE_FORMAT = "yyyy-MM-dd HH:mm"; private boolean printStoragePolicy; + private boolean printECPolicy; + private ErasureCodingPolicyManager ecManager; static class OutputEntryBuilder { private final SimpleDateFormat dateFormatter = @@ -62,6 +67,7 @@ static class OutputEntryBuilder { private long nsQuota = 0; private long dsQuota = 0; private int storagePolicy = 0; + private String ecPolicy = "-"; private String dirPermission = "-"; private PermissionStatus permissionStatus; @@ -83,6 +89,13 @@ static class OutputEntryBuilder { aclPermission = "+"; } storagePolicy = file.getStoragePolicyID(); + if (writer.printECPolicy && file.hasErasureCodingPolicyID()) { + ErasureCodingPolicy policy = writer.ecManager. + getByID((byte) file.getErasureCodingPolicyID()); + if (policy != null) { + ecPolicy = policy.getName(); + } + } break; case DIRECTORY: INodeDirectory dir = inode.getDirectory(); @@ -95,6 +108,12 @@ static class OutputEntryBuilder { aclPermission = "+"; } storagePolicy = writer.getStoragePolicy(dir.getXAttrs()); + if (writer.printECPolicy) { + String name= writer.getErasureCodingPolicyName(dir.getXAttrs()); + if (name != null) { + ecPolicy = name; + } + } break; case SYMLINK: INodeSymlink s = inode.getSymlink(); @@ -134,6 +153,9 @@ public String build() { if (writer.printStoragePolicy) { writer.append(buffer, storagePolicy); } + if (writer.printECPolicy) { + writer.append(buffer, ecPolicy); + } return buffer.substring(1); } } @@ -146,14 +168,21 @@ public String build() { PBImageDelimitedTextWriter(PrintStream out, String delimiter, String tempPath, boolean printStoragePolicy) throws IOException { - this(out, delimiter, tempPath, printStoragePolicy, 1, "-"); + this(out, delimiter, tempPath, printStoragePolicy, false, 1, "-", null); } PBImageDelimitedTextWriter(PrintStream out, String delimiter, - String tempPath, boolean printStoragePolicy, int threads, - String parallelOut) throws IOException { + String tempPath, boolean printStoragePolicy, + boolean printECPolicy, int threads, + String parallelOut, Configuration conf) + throws IOException { super(out, delimiter, tempPath, threads, parallelOut); this.printStoragePolicy = printStoragePolicy; + if (printECPolicy && conf != null) { + this.printECPolicy = true; + ecManager = ErasureCodingPolicyManager.getInstance(); + ecManager.init(conf); + } } @Override @@ -187,6 +216,9 @@ public String getHeader() { if (printStoragePolicy) { append(buffer, "StoragePolicyId"); } + if (printECPolicy) { + append(buffer, "ErasureCodingPolicy"); + } return buffer.toString(); } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/PBImageTextWriter.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/PBImageTextWriter.java index 2dab44a036ac6..5773d7fecf0b5 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/PBImageTextWriter.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/PBImageTextWriter.java @@ -27,6 +27,8 @@ import java.io.PrintStream; import java.io.RandomAccessFile; import java.io.UnsupportedEncodingException; +import java.io.ByteArrayInputStream; +import java.io.DataInputStream; import java.nio.ByteBuffer; import java.nio.channels.FileChannel; import java.util.ArrayList; @@ -63,6 +65,7 @@ import org.apache.hadoop.hdfs.server.namenode.INodeId; import org.apache.hadoop.hdfs.server.namenode.SerialNumberManager; import org.apache.hadoop.io.IOUtils; +import org.apache.hadoop.io.WritableUtils; import org.apache.hadoop.util.LimitInputStream; import org.apache.hadoop.util.Lists; import org.apache.hadoop.util.Time; @@ -77,6 +80,8 @@ import org.apache.hadoop.util.Preconditions; import org.apache.hadoop.thirdparty.com.google.common.collect.ImmutableList; +import static org.apache.hadoop.hdfs.server.common.HdfsServerConstants.XATTR_ERASURECODING_POLICY; + /** * This class reads the protobuf-based fsimage and generates text output * for each inode to {@link PBImageTextWriter#out}. The sub-class can override @@ -1029,4 +1034,23 @@ public static void mergeFiles(String[] srcPaths, String resultPath) } } } + + public String getErasureCodingPolicyName + (INodeSection.XAttrFeatureProto xattrFeatureProto) { + List xattrs = + FSImageFormatPBINode.Loader.loadXAttrs(xattrFeatureProto, stringTable); + for (XAttr xattr : xattrs) { + if (XATTR_ERASURECODING_POLICY.contains(xattr.getName())){ + try{ + ByteArrayInputStream bIn = new ByteArrayInputStream(xattr.getValue()); + DataInputStream dIn = new DataInputStream(bIn); + return WritableUtils.readString(dIn); + } catch (IOException ioException){ + return null; + } + } + } + return null; + } + } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml index 33ffd07c8de2b..da19904cfbd01 100755 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml @@ -5365,6 +5365,14 @@ + + dfs.namenode.remove.dead.datanode.batchnum + 10 + + Maximum number of datanodes removed by HeartbeatManager per scan. + + + dfs.namenode.snapshot.capture.openfiles false @@ -6446,4 +6454,12 @@ frequently than this time, the client will give up waiting. + + dfs.client.output.stream.uniq.default.key + DEFAULT + + The default prefix key to construct the uniqKey for one DFSOutputStream. + If the namespace is DEFAULT, it's best to change this conf to other value. + + diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/dfshealth.html b/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/dfshealth.html index 8c577001b2a68..4bf2e2bd7f5fe 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/dfshealth.html +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/dfshealth.html @@ -489,6 +489,7 @@ diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/dfshealth.js b/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/dfshealth.js index 86502ddc132d9..42fb059f09ed1 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/dfshealth.js +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/dfshealth.js @@ -580,45 +580,3 @@ load_page(); }); })(); - -function open_hostip_list(x0, x1) { - close_hostip_list(); - var ips = new Array(); - for (var i = 0; i < liveNodes.length; i++) { - var dn = liveNodes[i]; - var index = (dn.usedSpace / dn.capacity) * 100.0; - if (index == 0) { - index = 1; - } - //More than 100% do not care,so not record in 95%-100% bar - if (index > x0 && index <= x1) { - ips.push(dn.infoAddr.split(":")[0]); - } - } - var ipsText = ''; - for (var i = 0; i < ips.length; i++) { - ipsText += ips[i] + '\n'; - } - var histogram_div = document.getElementById('datanode-usage-histogram'); - histogram_div.setAttribute('style', 'position: relative'); - var ips_div = document.createElement("textarea"); - ips_div.setAttribute('id', 'datanode_ips'); - ips_div.setAttribute('rows', '8'); - ips_div.setAttribute('cols', '14'); - ips_div.setAttribute('style', 'position: absolute;top: 0px;right: -38px;'); - ips_div.setAttribute('readonly', 'readonly'); - histogram_div.appendChild(ips_div); - - var close_div = document.createElement("div"); - histogram_div.appendChild(close_div); - close_div.setAttribute('id', 'close_ips'); - close_div.setAttribute('style', 'position: absolute;top: 0px;right: -62px;width:20px;height;20px'); - close_div.setAttribute('onclick', 'close_hostip_list()'); - close_div.innerHTML = "X"; - ips_div.innerHTML = ipsText; -} - -function close_hostip_list() { - $("#datanode_ips").remove(); - $("#close_ips").remove(); -} diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/histogram-hostip.js b/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/histogram-hostip.js new file mode 100644 index 0000000000000..eff8e506aa845 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/histogram-hostip.js @@ -0,0 +1,58 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +function open_hostip_list(x0, x1) { + close_hostip_list(); + var ips = new Array(); + for (var i = 0; i < liveNodes.length; i++) { + var dn = liveNodes[i]; + var index = (dn.usedSpace / dn.capacity) * 100.0; + if (index == 0) { + index = 1; + } + //More than 100% do not care,so not record in 95%-100% bar + if (index > x0 && index <= x1) { + ips.push(dn.infoAddr.split(":")[0]); + } + } + var ipsText = ''; + for (var i = 0; i < ips.length; i++) { + ipsText += ips[i] + '\n'; + } + var histogram_div = document.getElementById('datanode-usage-histogram'); + histogram_div.setAttribute('style', 'position: relative'); + var ips_div = document.createElement("textarea"); + ips_div.setAttribute('id', 'datanode_ips'); + ips_div.setAttribute('rows', '8'); + ips_div.setAttribute('cols', '14'); + ips_div.setAttribute('style', 'position: absolute;top: 0px;right: -38px;'); + ips_div.setAttribute('readonly', 'readonly'); + histogram_div.appendChild(ips_div); + + var close_div = document.createElement("div"); + histogram_div.appendChild(close_div); + close_div.setAttribute('id', 'close_ips'); + close_div.setAttribute('style', 'position: absolute;top: 0px;right: -62px;width:20px;height;20px'); + close_div.setAttribute('onclick', 'close_hostip_list()'); + close_div.innerHTML = "X"; + ips_div.innerHTML = ipsText; +} + +function close_hostip_list() { + $("#datanode_ips").remove(); + $("#close_ips").remove(); +} diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/jquery.dataTables.min.js b/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/jquery.dataTables.min.js index 85dd817e13257..07af1c39938cb 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/jquery.dataTables.min.js +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/jquery.dataTables.min.js @@ -1,160 +1,166 @@ -/*! DataTables 1.10.7 - * ©2008-2015 SpryMedia Ltd - datatables.net/license - */ -(function(Ea,Q,k){var P=function(h){function W(a){var b,c,e={};h.each(a,function(d){if((b=d.match(/^([^A-Z]+?)([A-Z])/))&&-1!=="a aa ai ao as b fn i m o s ".indexOf(b[1]+" "))c=d.replace(b[0],b[2].toLowerCase()),e[c]=d,"o"===b[1]&&W(a[d])});a._hungarianMap=e}function H(a,b,c){a._hungarianMap||W(a);var e;h.each(b,function(d){e=a._hungarianMap[d];if(e!==k&&(c||b[e]===k))"o"===e.charAt(0)?(b[e]||(b[e]={}),h.extend(!0,b[e],b[d]),H(a[e],b[e],c)):b[e]=b[d]})}function P(a){var b=m.defaults.oLanguage,c=a.sZeroRecords; -!a.sEmptyTable&&(c&&"No data available in table"===b.sEmptyTable)&&E(a,a,"sZeroRecords","sEmptyTable");!a.sLoadingRecords&&(c&&"Loading..."===b.sLoadingRecords)&&E(a,a,"sZeroRecords","sLoadingRecords");a.sInfoThousands&&(a.sThousands=a.sInfoThousands);(a=a.sDecimal)&&db(a)}function eb(a){A(a,"ordering","bSort");A(a,"orderMulti","bSortMulti");A(a,"orderClasses","bSortClasses");A(a,"orderCellsTop","bSortCellsTop");A(a,"order","aaSorting");A(a,"orderFixed","aaSortingFixed");A(a,"paging","bPaginate"); -A(a,"pagingType","sPaginationType");A(a,"pageLength","iDisplayLength");A(a,"searching","bFilter");if(a=a.aoSearchCols)for(var b=0,c=a.length;b").css({position:"absolute",top:0,left:0,height:1,width:1,overflow:"hidden"}).append(h("
    ").css({position:"absolute", -top:1,left:1,width:100,overflow:"scroll"}).append(h('
    ').css({width:"100%",height:10}))).appendTo("body"),c=b.find(".test");a.bScrollOversize=100===c[0].offsetWidth;a.bScrollbarLeft=1!==Math.round(c.offset().left);b.remove()}function hb(a,b,c,e,d,f){var g,j=!1;c!==k&&(g=c,j=!0);for(;e!==d;)a.hasOwnProperty(e)&&(g=j?b(g,a[e],e,a):a[e],j=!0,e+=f);return g}function Fa(a,b){var c=m.defaults.column,e=a.aoColumns.length,c=h.extend({},m.models.oColumn,c,{nTh:b?b:Q.createElement("th"),sTitle:c.sTitle? -c.sTitle:b?b.innerHTML:"",aDataSort:c.aDataSort?c.aDataSort:[e],mData:c.mData?c.mData:e,idx:e});a.aoColumns.push(c);c=a.aoPreSearchCols;c[e]=h.extend({},m.models.oSearch,c[e]);ka(a,e,h(b).data())}function ka(a,b,c){var b=a.aoColumns[b],e=a.oClasses,d=h(b.nTh);if(!b.sWidthOrig){b.sWidthOrig=d.attr("width")||null;var f=(d.attr("style")||"").match(/width:\s*(\d+[pxem%]+)/);f&&(b.sWidthOrig=f[1])}c!==k&&null!==c&&(fb(c),H(m.defaults.column,c),c.mDataProp!==k&&!c.mData&&(c.mData=c.mDataProp),c.sType&& -(b._sManualType=c.sType),c.className&&!c.sClass&&(c.sClass=c.className),h.extend(b,c),E(b,c,"sWidth","sWidthOrig"),c.iDataSort!==k&&(b.aDataSort=[c.iDataSort]),E(b,c,"aDataSort"));var g=b.mData,j=R(g),i=b.mRender?R(b.mRender):null,c=function(a){return"string"===typeof a&&-1!==a.indexOf("@")};b._bAttrSrc=h.isPlainObject(g)&&(c(g.sort)||c(g.type)||c(g.filter));b.fnGetData=function(a,b,c){var e=j(a,b,k,c);return i&&b?i(e,b,a,c):e};b.fnSetData=function(a,b,c){return S(g)(a,b,c)};"number"!==typeof g&& -(a._rowReadObject=!0);a.oFeatures.bSort||(b.bSortable=!1,d.addClass(e.sSortableNone));a=-1!==h.inArray("asc",b.asSorting);c=-1!==h.inArray("desc",b.asSorting);!b.bSortable||!a&&!c?(b.sSortingClass=e.sSortableNone,b.sSortingClassJUI=""):a&&!c?(b.sSortingClass=e.sSortableAsc,b.sSortingClassJUI=e.sSortJUIAscAllowed):!a&&c?(b.sSortingClass=e.sSortableDesc,b.sSortingClassJUI=e.sSortJUIDescAllowed):(b.sSortingClass=e.sSortable,b.sSortingClassJUI=e.sSortJUI)}function X(a){if(!1!==a.oFeatures.bAutoWidth){var b= -a.aoColumns;Ga(a);for(var c=0,e=b.length;cq[f])e(l.length+q[f],o);else if("string"===typeof q[f]){j=0;for(i=l.length;jb&&a[d]--; -1!=e&&c===k&&a.splice(e,1)}function ca(a,b,c,e){var d=a.aoData[b],f,g=function(c,f){for(;c.childNodes.length;)c.removeChild(c.firstChild);c.innerHTML=x(a,b,f,"display")};if("dom"===c||(!c||"auto"===c)&&"dom"===d.src)d._aData=na(a,d,e,e===k?k:d._aData).data;else{var j=d.anCells;if(j)if(e!==k)g(j[e],e);else{c=0;for(f=j.length;c").appendTo(g));b=0;for(c=l.length;btr").attr("role","row");h(g).find(">tr>th, >tr>td").addClass(o.sHeaderTH); -h(j).find(">tr>th, >tr>td").addClass(o.sFooterTH);if(null!==j){a=a.aoFooter[0];b=0;for(c=a.length;b=a.fnRecordsDisplay()?0:g,a.iInitDisplayStart=-1);var g=a._iDisplayStart,o=a.fnDisplayEnd();if(a.bDeferLoading)a.bDeferLoading=!1,a.iDraw++,C(a,!1);else if(j){if(!a.bDestroying&&!kb(a))return}else a.iDraw++;if(0!==i.length){f=j?a.aoData.length:o;for(j=j?0:g;j",{"class":d?e[0]:""}).append(h("",{valign:"top",colSpan:aa(a),"class":a.oClasses.sRowEmpty}).html(c))[0];w(a,"aoHeaderCallback","header",[h(a.nTHead).children("tr")[0],La(a),g,o,i]);w(a,"aoFooterCallback","footer",[h(a.nTFoot).children("tr")[0],La(a),g,o,i]);e=h(a.nTBody);e.children().detach();e.append(h(b));w(a,"aoDrawCallback","draw",[a]);a.bSorted=!1;a.bFiltered=!1;a.bDrawing= -!1}}function N(a,b){var c=a.oFeatures,e=c.bFilter;c.bSort&&lb(a);e?fa(a,a.oPreviousSearch):a.aiDisplay=a.aiDisplayMaster.slice();!0!==b&&(a._iDisplayStart=0);a._drawHold=b;M(a);a._drawHold=!1}function mb(a){var b=a.oClasses,c=h(a.nTable),c=h("
    ").insertBefore(c),e=a.oFeatures,d=h("
    ",{id:a.sTableId+"_wrapper","class":b.sWrapper+(a.nTFoot?"":" "+b.sNoFooter)});a.nHolding=c[0];a.nTableWrapper=d[0];a.nTableReinsertBefore=a.nTable.nextSibling;for(var f=a.sDom.split(""),g,j,i,o,l,q,n=0;n")[0];o=f[n+1];if("'"==o||'"'==o){l="";for(q=2;f[n+q]!=o;)l+=f[n+q],q++;"H"==l?l=b.sJUIHeader:"F"==l&&(l=b.sJUIFooter);-1!=l.indexOf(".")?(o=l.split("."),i.id=o[0].substr(1,o[0].length-1),i.className=o[1]):"#"==l.charAt(0)?i.id=l.substr(1,l.length-1):i.className=l;n+=q}d.append(i);d=h(i)}else if(">"==j)d=d.parent();else if("l"==j&&e.bPaginate&&e.bLengthChange)g=nb(a);else if("f"==j&&e.bFilter)g=ob(a);else if("r"==j&&e.bProcessing)g=pb(a);else if("t"==j)g=qb(a);else if("i"== -j&&e.bInfo)g=rb(a);else if("p"==j&&e.bPaginate)g=sb(a);else if(0!==m.ext.feature.length){i=m.ext.feature;q=0;for(o=i.length;q',j=e.sSearch,j=j.match(/_INPUT_/)?j.replace("_INPUT_",g):j+g,b=h("
    ",{id:!f.f?c+"_filter":null,"class":b.sFilter}).append(h("
    ").addClass(b.sLength);a.aanFeatures.l||(i[0].id=c+"_length");i.children().append(a.oLanguage.sLengthMenu.replace("_MENU_",d[0].outerHTML));h("select",i).val(a._iDisplayLength).bind("change.DT",function(){Ra(a,h(this).val());M(a)});h(a.nTable).bind("length.dt.DT",function(b,c,f){a===c&&h("select",i).val(f)});return i[0]}function sb(a){var b=a.sPaginationType,c=m.ext.pager[b],e="function"===typeof c,d=function(a){M(a)},b=h("
    ").addClass(a.oClasses.sPaging+b)[0], -f=a.aanFeatures;e||c.fnInit(a,b,d);f.p||(b.id=a.sTableId+"_paginate",a.aoDrawCallback.push({fn:function(a){if(e){var b=a._iDisplayStart,i=a._iDisplayLength,h=a.fnRecordsDisplay(),l=-1===i,b=l?0:Math.ceil(b/i),i=l?1:Math.ceil(h/i),h=c(b,i),q,l=0;for(q=f.p.length;lf&&(e=0)): -"first"==b?e=0:"previous"==b?(e=0<=d?e-d:0,0>e&&(e=0)):"next"==b?e+d",{id:!a.aanFeatures.r?a.sTableId+"_processing":null,"class":a.oClasses.sProcessing}).html(a.oLanguage.sProcessing).insertBefore(a.nTable)[0]}function C(a,b){a.oFeatures.bProcessing&&h(a.aanFeatures.r).css("display",b?"block":"none");w(a, -null,"processing",[a,b])}function qb(a){var b=h(a.nTable);b.attr("role","grid");var c=a.oScroll;if(""===c.sX&&""===c.sY)return a.nTable;var e=c.sX,d=c.sY,f=a.oClasses,g=b.children("caption"),j=g.length?g[0]._captionSide:null,i=h(b[0].cloneNode(!1)),o=h(b[0].cloneNode(!1)),l=b.children("tfoot");c.sX&&"100%"===b.attr("width")&&b.removeAttr("width");l.length||(l=null);c=h("
    ",{"class":f.sScrollWrapper}).append(h("
    ",{"class":f.sScrollHead}).css({overflow:"hidden",position:"relative",border:0, -width:e?!e?null:s(e):"100%"}).append(h("
    ",{"class":f.sScrollHeadInner}).css({"box-sizing":"content-box",width:c.sXInner||"100%"}).append(i.removeAttr("id").css("margin-left",0).append("top"===j?g:null).append(b.children("thead"))))).append(h("
    ",{"class":f.sScrollBody}).css({overflow:"auto",height:!d?null:s(d),width:!e?null:s(e)}).append(b));l&&c.append(h("
    ",{"class":f.sScrollFoot}).css({overflow:"hidden",border:0,width:e?!e?null:s(e):"100%"}).append(h("
    ",{"class":f.sScrollFootInner}).append(o.removeAttr("id").css("margin-left", -0).append("bottom"===j?g:null).append(b.children("tfoot")))));var b=c.children(),q=b[0],f=b[1],n=l?b[2]:null;if(e)h(f).on("scroll.DT",function(){var a=this.scrollLeft;q.scrollLeft=a;l&&(n.scrollLeft=a)});a.nScrollHead=q;a.nScrollBody=f;a.nScrollFoot=n;a.aoDrawCallback.push({fn:Y,sName:"scrolling"});return c[0]}function Y(a){var b=a.oScroll,c=b.sX,e=b.sXInner,d=b.sY,f=b.iBarWidth,g=h(a.nScrollHead),j=g[0].style,i=g.children("div"),o=i[0].style,l=i.children("table"),i=a.nScrollBody,q=h(i),n=i.style, -k=h(a.nScrollFoot).children("div"),p=k.children("table"),m=h(a.nTHead),r=h(a.nTable),t=r[0],O=t.style,L=a.nTFoot?h(a.nTFoot):null,ha=a.oBrowser,w=ha.bScrollOversize,v,u,y,x,z,A=[],B=[],C=[],D,E=function(a){a=a.style;a.paddingTop="0";a.paddingBottom="0";a.borderTopWidth="0";a.borderBottomWidth="0";a.height=0};r.children("thead, tfoot").remove();z=m.clone().prependTo(r);v=m.find("tr");y=z.find("tr");z.find("th, td").removeAttr("tabindex");L&&(x=L.clone().prependTo(r),u=L.find("tr"),x=x.find("tr")); -c||(n.width="100%",g[0].style.width="100%");h.each(qa(a,z),function(b,c){D=la(a,b);c.style.width=a.aoColumns[D].sWidth});L&&G(function(a){a.style.width=""},x);b.bCollapse&&""!==d&&(n.height=q[0].offsetHeight+m[0].offsetHeight+"px");g=r.outerWidth();if(""===c){if(O.width="100%",w&&(r.find("tbody").height()>i.offsetHeight||"scroll"==q.css("overflow-y")))O.width=s(r.outerWidth()-f)}else""!==e?O.width=s(e):g==q.width()&&q.height()g-f&&(O.width=s(g))):O.width= -s(g);g=r.outerWidth();G(E,y);G(function(a){C.push(a.innerHTML);A.push(s(h(a).css("width")))},y);G(function(a,b){a.style.width=A[b]},v);h(y).height(0);L&&(G(E,x),G(function(a){B.push(s(h(a).css("width")))},x),G(function(a,b){a.style.width=B[b]},u),h(x).height(0));G(function(a,b){a.innerHTML='
    '+C[b]+"
    ";a.style.width=A[b]},y);L&&G(function(a,b){a.innerHTML="";a.style.width=B[b]},x);if(r.outerWidth()i.offsetHeight|| -"scroll"==q.css("overflow-y")?g+f:g;if(w&&(i.scrollHeight>i.offsetHeight||"scroll"==q.css("overflow-y")))O.width=s(u-f);(""===c||""!==e)&&I(a,1,"Possible column misalignment",6)}else u="100%";n.width=s(u);j.width=s(u);L&&(a.nScrollFoot.style.width=s(u));!d&&w&&(n.height=s(t.offsetHeight+f));d&&b.bCollapse&&(n.height=s(d),b=c&&t.offsetWidth>i.offsetWidth?f:0,t.offsetHeighti.clientHeight|| -"scroll"==q.css("overflow-y");ha="padding"+(ha.bScrollbarLeft?"Left":"Right");o[ha]=l?f+"px":"0px";L&&(p[0].style.width=s(b),k[0].style.width=s(b),k[0].style[ha]=l?f+"px":"0px");q.scroll();if((a.bSorted||a.bFiltered)&&!a._drawHold)i.scrollTop=0}function G(a,b,c){for(var e=0,d=0,f=b.length,g,j;d").appendTo(j.find("tbody"));j.find("tfoot th, tfoot td").css("width", -"");i=qa(a,j.find("thead")[0]);for(n=0;n").css("width",s(a)).appendTo(b||Q.body),e=c[0].offsetWidth;c.remove();return e}function Fb(a,b){var c=a.oScroll;if(c.sX||c.sY)c=!c.sX?c.iBarWidth:0,b.style.width=s(h(b).outerWidth()-c)}function Eb(a,b){var c=Gb(a,b);if(0>c)return null;var e=a.aoData[c];return!e.nTr?h("").html(x(a,c,b,"display"))[0]:e.anCells[b]}function Gb(a,b){for(var c,e=-1,d=-1,f=0,g=a.aoData.length;fe&&(e=c.length,d=f);return d}function s(a){return null===a?"0px":"number"==typeof a?0>a?"0px":a+"px":a.match(/\d$/)?a+"px":a}function Hb(){var a=m.__scrollbarWidth;if(a===k){var b=h("

    ").css({position:"absolute",top:0,left:0,width:"100%",height:150,padding:0,overflow:"scroll",visibility:"hidden"}).appendTo("body"),a=b[0].offsetWidth-b[0].clientWidth;m.__scrollbarWidth=a;b.remove()}return a}function U(a){var b,c,e=[],d=a.aoColumns,f,g,j,i;b=a.aaSortingFixed;c=h.isPlainObject(b);var o=[]; -f=function(a){a.length&&!h.isArray(a[0])?o.push(a):o.push.apply(o,a)};h.isArray(b)&&f(b);c&&b.pre&&f(b.pre);f(a.aaSorting);c&&b.post&&f(b.post);for(a=0;ad?1:0,0!==c)return"asc"===j.dir?c:-c;c=e[a];d=e[b];return cd?1:0}):i.sort(function(a,b){var c,g,j,i,k=h.length,m=f[a]._aSortData,r=f[b]._aSortData;for(j=0;jg?1:0})}a.bSorted=!0}function Jb(a){for(var b,c,e=a.aoColumns,d=U(a),a=a.oLanguage.oAria,f=0,g=e.length;f/g,"");var i=c.nTh;i.removeAttribute("aria-sort");c.bSortable&&(0d?d+1:3));d=0;for(f=e.length;dd?d+1:3))}a.aLastSort=e}function Ib(a,b){var c=a.aoColumns[b],e=m.ext.order[c.sSortDataType],d;e&&(d=e.call(a.oInstance,a,b,$(a,b)));for(var f,g=m.ext.type.order[c.sType+"-pre"],j=0,i=a.aoData.length;j=e.length?[0,c[1]]:c)}));d.search!==k&&h.extend(a.oPreviousSearch,Ab(d.search));b=0;for(c=d.columns.length;b=c&&(b=c-e);b-=b%e;if(-1===e||0>b)b=0;a._iDisplayStart=b}function Pa(a,b){var c=a.renderer,e=m.ext.renderer[b];return h.isPlainObject(c)&&c[b]?e[c[b]]||e._:"string"===typeof c?e[c]||e._:e._}function B(a){return a.oFeatures.bServerSide? -"ssp":a.ajax||a.sAjaxSource?"ajax":"dom"}function Wa(a,b){var c=[],c=Mb.numbers_length,e=Math.floor(c/2);b<=c?c=V(0,b):a<=e?(c=V(0,c-2),c.push("ellipsis"),c.push(b-1)):(a>=b-1-e?c=V(b-(c-2),b):(c=V(a-e+2,a+e-1),c.push("ellipsis"),c.push(b-1)),c.splice(0,0,"ellipsis"),c.splice(0,0,0));c.DT_el="span";return c}function db(a){h.each({num:function(b){return Aa(b,a)},"num-fmt":function(b){return Aa(b,a,Xa)},"html-num":function(b){return Aa(b,a,Ba)},"html-num-fmt":function(b){return Aa(b,a,Ba,Xa)}},function(b, -c){u.type.order[b+a+"-pre"]=c;b.match(/^html\-/)&&(u.type.search[b+a]=u.type.search.html)})}function Nb(a){return function(){var b=[za(this[m.ext.iApiIndex])].concat(Array.prototype.slice.call(arguments));return m.ext.internal[a].apply(this,b)}}var m,u,t,r,v,Ya={},Ob=/[\r\n]/g,Ba=/<.*?>/g,ac=/^[\w\+\-]/,bc=/[\w\+\-]$/,Yb=RegExp("(\\/|\\.|\\*|\\+|\\?|\\||\\(|\\)|\\[|\\]|\\{|\\}|\\\\|\\$|\\^|\\-)","g"),Xa=/[',$\u00a3\u20ac\u00a5%\u2009\u202F\u20BD\u20a9\u20BArfk]/gi,J=function(a){return!a||!0===a|| -"-"===a?!0:!1},Pb=function(a){var b=parseInt(a,10);return!isNaN(b)&&isFinite(a)?b:null},Qb=function(a,b){Ya[b]||(Ya[b]=RegExp(va(b),"g"));return"string"===typeof a&&"."!==b?a.replace(/\./g,"").replace(Ya[b],"."):a},Za=function(a,b,c){var e="string"===typeof a;if(J(a))return!0;b&&e&&(a=Qb(a,b));c&&e&&(a=a.replace(Xa,""));return!isNaN(parseFloat(a))&&isFinite(a)},Rb=function(a,b,c){return J(a)?!0:!(J(a)||"string"===typeof a)?null:Za(a.replace(Ba,""),b,c)?!0:null},D=function(a,b,c){var e=[],d=0,f=a.length; -if(c!==k)for(;d")[0],Zb=wa.textContent!==k,$b=/<.*?>/g;m=function(a){this.$=function(a,b){return this.api(!0).$(a,b)};this._=function(a,b){return this.api(!0).rows(a,b).data()};this.api=function(a){return a?new t(za(this[u.iApiIndex])):new t(this)};this.fnAddData=function(a,b){var c=this.api(!0),e=h.isArray(a)&&(h.isArray(a[0])||h.isPlainObject(a[0]))?c.rows.add(a):c.row.add(a);(b=== -k||b)&&c.draw();return e.flatten().toArray()};this.fnAdjustColumnSizing=function(a){var b=this.api(!0).columns.adjust(),c=b.settings()[0],e=c.oScroll;a===k||a?b.draw(!1):(""!==e.sX||""!==e.sY)&&Y(c)};this.fnClearTable=function(a){var b=this.api(!0).clear();(a===k||a)&&b.draw()};this.fnClose=function(a){this.api(!0).row(a).child.hide()};this.fnDeleteRow=function(a,b,c){var e=this.api(!0),a=e.rows(a),d=a.settings()[0],h=d.aoData[a[0][0]];a.remove();b&&b.call(this,d,h);(c===k||c)&&e.draw();return h}; -this.fnDestroy=function(a){this.api(!0).destroy(a)};this.fnDraw=function(a){this.api(!0).draw(a)};this.fnFilter=function(a,b,c,e,d,h){d=this.api(!0);null===b||b===k?d.search(a,c,e,h):d.column(b).search(a,c,e,h);d.draw()};this.fnGetData=function(a,b){var c=this.api(!0);if(a!==k){var e=a.nodeName?a.nodeName.toLowerCase():"";return b!==k||"td"==e||"th"==e?c.cell(a,b).data():c.row(a).data()||null}return c.data().toArray()};this.fnGetNodes=function(a){var b=this.api(!0);return a!==k?b.row(a).node():b.rows().nodes().flatten().toArray()}; -this.fnGetPosition=function(a){var b=this.api(!0),c=a.nodeName.toUpperCase();return"TR"==c?b.row(a).index():"TD"==c||"TH"==c?(a=b.cell(a).index(),[a.row,a.columnVisible,a.column]):null};this.fnIsOpen=function(a){return this.api(!0).row(a).child.isShown()};this.fnOpen=function(a,b,c){return this.api(!0).row(a).child(b,c).show().child()[0]};this.fnPageChange=function(a,b){var c=this.api(!0).page(a);(b===k||b)&&c.draw(!1)};this.fnSetColumnVis=function(a,b,c){a=this.api(!0).column(a).visible(b);(c=== -k||c)&&a.columns.adjust().draw()};this.fnSettings=function(){return za(this[u.iApiIndex])};this.fnSort=function(a){this.api(!0).order(a).draw()};this.fnSortListener=function(a,b,c){this.api(!0).order.listener(a,b,c)};this.fnUpdate=function(a,b,c,e,d){var h=this.api(!0);c===k||null===c?h.row(b).data(a):h.cell(b,c).data(a);(d===k||d)&&h.columns.adjust();(e===k||e)&&h.draw();return 0};this.fnVersionCheck=u.fnVersionCheck;var b=this,c=a===k,e=this.length;c&&(a={});this.oApi=this.internal=u.internal;for(var d in m.ext.internal)d&& -(this[d]=Nb(d));this.each(function(){var d={},d=1t<"F"ip>'),p.renderer)? -h.isPlainObject(p.renderer)&&!p.renderer.header&&(p.renderer.header="jqueryui"):p.renderer="jqueryui":h.extend(i,m.ext.classes,d.oClasses);q.addClass(i.sTable);if(""!==p.oScroll.sX||""!==p.oScroll.sY)p.oScroll.iBarWidth=Hb();!0===p.oScroll.sX&&(p.oScroll.sX="100%");p.iInitDisplayStart===k&&(p.iInitDisplayStart=d.iDisplayStart,p._iDisplayStart=d.iDisplayStart);null!==d.iDeferLoading&&(p.bDeferLoading=!0,g=h.isArray(d.iDeferLoading),p._iRecordsDisplay=g?d.iDeferLoading[0]:d.iDeferLoading,p._iRecordsTotal= -g?d.iDeferLoading[1]:d.iDeferLoading);var t=p.oLanguage;h.extend(!0,t,d.oLanguage);""!==t.sUrl&&(h.ajax({dataType:"json",url:t.sUrl,success:function(a){P(a);H(l.oLanguage,a);h.extend(true,t,a);ga(p)},error:function(){ga(p)}}),o=!0);null===d.asStripeClasses&&(p.asStripeClasses=[i.sStripeOdd,i.sStripeEven]);var g=p.asStripeClasses,s=q.children("tbody").find("tr").eq(0);-1!==h.inArray(!0,h.map(g,function(a){return s.hasClass(a)}))&&(h("tbody tr",this).removeClass(g.join(" ")),p.asDestroyStripes=g.slice()); -n=[];g=this.getElementsByTagName("thead");0!==g.length&&(da(p.aoHeader,g[0]),n=qa(p));if(null===d.aoColumns){r=[];g=0;for(j=n.length;g").appendTo(this));p.nTHead=j[0];j=q.children("tbody");0===j.length&&(j=h("").appendTo(this));p.nTBody=j[0];j=q.children("tfoot");if(0===j.length&&0").appendTo(this);0===j.length||0===j.children().length?q.addClass(i.sNoFooter): -0a?new t(b[a],this[a]):null},filter:function(a){var b=[];if(y.filter)b=y.filter.call(this,a,this);else for(var c=0,e=this.length;c").addClass(b), -h("td",c).addClass(b).html(a)[0].colSpan=aa(e),d.push(c[0]))};f(a,b);c._details&&c._details.remove();c._details=h(d);c._detailsShow&&c._details.insertAfter(c.nTr)}return this});r(["row().child.show()","row().child().show()"],function(){Vb(this,!0);return this});r(["row().child.hide()","row().child().hide()"],function(){Vb(this,!1);return this});r(["row().child.remove()","row().child().remove()"],function(){cb(this);return this});r("row().child.isShown()",function(){var a=this.context;return a.length&& -this.length?a[0].aoData[this[0]]._detailsShow||!1:!1});var dc=/^(.+):(name|visIdx|visible)$/,Wb=function(a,b,c,e,d){for(var c=[],e=0,f=d.length;e=0?b:g.length+b];if(typeof a==="function"){var d=Ca(c, -f);return h.map(g,function(b,f){return a(f,Wb(c,f,0,0,d),i[f])?f:null})}var k=typeof a==="string"?a.match(dc):"";if(k)switch(k[2]){case "visIdx":case "visible":b=parseInt(k[1],10);if(b<0){var m=h.map(g,function(a,b){return a.bVisible?b:null});return[m[m.length+b]]}return[la(c,b)];case "name":return h.map(j,function(a,b){return a===k[1]?b:null})}else return h(i).filter(a).map(function(){return h.inArray(this,i)}).toArray()},c,f)},1);c.selector.cols=a;c.selector.opts=b;return c});v("columns().header()", -"column().header()",function(){return this.iterator("column",function(a,b){return a.aoColumns[b].nTh},1)});v("columns().footer()","column().footer()",function(){return this.iterator("column",function(a,b){return a.aoColumns[b].nTf},1)});v("columns().data()","column().data()",function(){return this.iterator("column-rows",Wb,1)});v("columns().dataSrc()","column().dataSrc()",function(){return this.iterator("column",function(a,b){return a.aoColumns[b].mData},1)});v("columns().cache()","column().cache()", -function(a){return this.iterator("column-rows",function(b,c,e,d,f){return ia(b.aoData,f,"search"===a?"_aFilterData":"_aSortData",c)},1)});v("columns().nodes()","column().nodes()",function(){return this.iterator("column-rows",function(a,b,c,e,d){return ia(a.aoData,d,"anCells",b)},1)});v("columns().visible()","column().visible()",function(a,b){return this.iterator("column",function(c,e){if(a===k)return c.aoColumns[e].bVisible;var d=c.aoColumns,f=d[e],g=c.aoData,j,i,m;if(a!==k&&f.bVisible!==a){if(a){var l= -h.inArray(!0,D(d,"bVisible"),e+1);j=0;for(i=g.length;je;return!0};m.isDataTable=m.fnIsDataTable=function(a){var b=h(a).get(0),c=!1;h.each(m.settings, -function(a,d){var f=d.nScrollHead?h("table",d.nScrollHead)[0]:null,g=d.nScrollFoot?h("table",d.nScrollFoot)[0]:null;if(d.nTable===b||f===b||g===b)c=!0});return c};m.tables=m.fnTables=function(a){return h.map(m.settings,function(b){if(!a||a&&h(b.nTable).is(":visible"))return b.nTable})};m.util={throttle:ua,escapeRegex:va};m.camelToHungarian=H;r("$()",function(a,b){var c=this.rows(b).nodes(),c=h(c);return h([].concat(c.filter(a).toArray(),c.find(a).toArray()))});h.each(["on","one","off"],function(a, -b){r(b+"()",function(){var a=Array.prototype.slice.call(arguments);a[0].match(/\.dt\b/)||(a[0]+=".dt");var e=h(this.tables().nodes());e[b].apply(e,a);return this})});r("clear()",function(){return this.iterator("table",function(a){oa(a)})});r("settings()",function(){return new t(this.context,this.context)});r("init()",function(){var a=this.context;return a.length?a[0].oInit:null});r("data()",function(){return this.iterator("table",function(a){return D(a.aoData,"_aData")}).flatten()});r("destroy()", -function(a){a=a||!1;return this.iterator("table",function(b){var c=b.nTableWrapper.parentNode,e=b.oClasses,d=b.nTable,f=b.nTBody,g=b.nTHead,j=b.nTFoot,i=h(d),f=h(f),k=h(b.nTableWrapper),l=h.map(b.aoData,function(a){return a.nTr}),q;b.bDestroying=!0;w(b,"aoDestroyCallback","destroy",[b]);a||(new t(b)).columns().visible(!0);k.unbind(".DT").find(":not(tbody *)").unbind(".DT");h(Ea).unbind(".DT-"+b.sInstance);d!=g.parentNode&&(i.children("thead").detach(),i.append(g));j&&d!=j.parentNode&&(i.children("tfoot").detach(), -i.append(j));i.detach();k.detach();b.aaSorting=[];b.aaSortingFixed=[];xa(b);h(l).removeClass(b.asStripeClasses.join(" "));h("th, td",g).removeClass(e.sSortable+" "+e.sSortableAsc+" "+e.sSortableDesc+" "+e.sSortableNone);b.bJUI&&(h("th span."+e.sSortIcon+", td span."+e.sSortIcon,g).detach(),h("th, td",g).each(function(){var a=h("div."+e.sSortJUIWrapper,this);h(this).append(a.contents());a.detach()}));!a&&c&&c.insertBefore(d,b.nTableReinsertBefore);f.children().detach();f.append(l);i.css("width",b.sDestroyWidth).removeClass(e.sTable); -(q=b.asDestroyStripes.length)&&f.children().each(function(a){h(this).addClass(b.asDestroyStripes[a%q])});c=h.inArray(b,m.settings);-1!==c&&m.settings.splice(c,1)})});h.each(["column","row","cell"],function(a,b){r(b+"s().every()",function(a){return this.iterator(b,function(e,d,f){a.call((new t(e))[b](d,f))})})});r("i18n()",function(a,b,c){var e=this.context[0],a=R(a)(e.oLanguage);a===k&&(a=b);c!==k&&h.isPlainObject(a)&&(a=a[c]!==k?a[c]:a._);return a.replace("%d",c)});m.version="1.10.7";m.settings= -[];m.models={};m.models.oSearch={bCaseInsensitive:!0,sSearch:"",bRegex:!1,bSmart:!0};m.models.oRow={nTr:null,anCells:null,_aData:[],_aSortData:null,_aFilterData:null,_sFilterRow:null,_sRowStripe:"",src:null};m.models.oColumn={idx:null,aDataSort:null,asSorting:null,bSearchable:null,bSortable:null,bVisible:null,_sManualType:null,_bAttrSrc:!1,fnCreatedCell:null,fnGetData:null,fnSetData:null,mData:null,mRender:null,nTh:null,nTf:null,sClass:null,sContentPadding:null,sDefaultContent:null,sName:null,sSortDataType:"std", -sSortingClass:null,sSortingClassJUI:null,sTitle:null,sType:null,sWidth:null,sWidthOrig:null};m.defaults={aaData:null,aaSorting:[[0,"asc"]],aaSortingFixed:[],ajax:null,aLengthMenu:[10,25,50,100],aoColumns:null,aoColumnDefs:null,aoSearchCols:[],asStripeClasses:null,bAutoWidth:!0,bDeferRender:!1,bDestroy:!1,bFilter:!0,bInfo:!0,bJQueryUI:!1,bLengthChange:!0,bPaginate:!0,bProcessing:!1,bRetrieve:!1,bScrollCollapse:!1,bServerSide:!1,bSort:!0,bSortMulti:!0,bSortCellsTop:!1,bSortClasses:!0,bStateSave:!1, -fnCreatedRow:null,fnDrawCallback:null,fnFooterCallback:null,fnFormatNumber:function(a){return a.toString().replace(/\B(?=(\d{3})+(?!\d))/g,this.oLanguage.sThousands)},fnHeaderCallback:null,fnInfoCallback:null,fnInitComplete:null,fnPreDrawCallback:null,fnRowCallback:null,fnServerData:null,fnServerParams:null,fnStateLoadCallback:function(a){try{return JSON.parse((-1===a.iStateDuration?sessionStorage:localStorage).getItem("DataTables_"+a.sInstance+"_"+location.pathname))}catch(b){}},fnStateLoadParams:null, -fnStateLoaded:null,fnStateSaveCallback:function(a,b){try{(-1===a.iStateDuration?sessionStorage:localStorage).setItem("DataTables_"+a.sInstance+"_"+location.pathname,JSON.stringify(b))}catch(c){}},fnStateSaveParams:null,iStateDuration:7200,iDeferLoading:null,iDisplayLength:10,iDisplayStart:0,iTabIndex:0,oClasses:{},oLanguage:{oAria:{sSortAscending:": activate to sort column ascending",sSortDescending:": activate to sort column descending"},oPaginate:{sFirst:"First",sLast:"Last",sNext:"Next",sPrevious:"Previous"}, -sEmptyTable:"No data available in table",sInfo:"Showing _START_ to _END_ of _TOTAL_ entries",sInfoEmpty:"Showing 0 to 0 of 0 entries",sInfoFiltered:"(filtered from _MAX_ total entries)",sInfoPostFix:"",sDecimal:"",sThousands:",",sLengthMenu:"Show _MENU_ entries",sLoadingRecords:"Loading...",sProcessing:"Processing...",sSearch:"Search:",sSearchPlaceholder:"",sUrl:"",sZeroRecords:"No matching records found"},oSearch:h.extend({},m.models.oSearch),sAjaxDataProp:"data",sAjaxSource:null,sDom:"lfrtip",searchDelay:null, -sPaginationType:"simple_numbers",sScrollX:"",sScrollXInner:"",sScrollY:"",sServerMethod:"GET",renderer:null};W(m.defaults);m.defaults.column={aDataSort:null,iDataSort:-1,asSorting:["asc","desc"],bSearchable:!0,bSortable:!0,bVisible:!0,fnCreatedCell:null,mData:null,mRender:null,sCellType:"td",sClass:"",sContentPadding:"",sDefaultContent:null,sName:"",sSortDataType:"std",sTitle:null,sType:null,sWidth:null};W(m.defaults.column);m.models.oSettings={oFeatures:{bAutoWidth:null,bDeferRender:null,bFilter:null, -bInfo:null,bLengthChange:null,bPaginate:null,bProcessing:null,bServerSide:null,bSort:null,bSortMulti:null,bSortClasses:null,bStateSave:null},oScroll:{bCollapse:null,iBarWidth:0,sX:null,sXInner:null,sY:null},oLanguage:{fnInfoCallback:null},oBrowser:{bScrollOversize:!1,bScrollbarLeft:!1},ajax:null,aanFeatures:[],aoData:[],aiDisplay:[],aiDisplayMaster:[],aoColumns:[],aoHeader:[],aoFooter:[],oPreviousSearch:{},aoPreSearchCols:[],aaSorting:null,aaSortingFixed:[],asStripeClasses:null,asDestroyStripes:[], -sDestroyWidth:0,aoRowCallback:[],aoHeaderCallback:[],aoFooterCallback:[],aoDrawCallback:[],aoRowCreatedCallback:[],aoPreDrawCallback:[],aoInitComplete:[],aoStateSaveParams:[],aoStateLoadParams:[],aoStateLoaded:[],sTableId:"",nTable:null,nTHead:null,nTFoot:null,nTBody:null,nTableWrapper:null,bDeferLoading:!1,bInitialised:!1,aoOpenRows:[],sDom:null,searchDelay:null,sPaginationType:"two_button",iStateDuration:0,aoStateSave:[],aoStateLoad:[],oSavedState:null,oLoadedState:null,sAjaxSource:null,sAjaxDataProp:null, -bAjaxDataGet:!0,jqXHR:null,json:k,oAjaxData:k,fnServerData:null,aoServerParams:[],sServerMethod:null,fnFormatNumber:null,aLengthMenu:null,iDraw:0,bDrawing:!1,iDrawError:-1,_iDisplayLength:10,_iDisplayStart:0,_iRecordsTotal:0,_iRecordsDisplay:0,bJUI:null,oClasses:{},bFiltered:!1,bSorted:!1,bSortCellsTop:null,oInit:null,aoDestroyCallback:[],fnRecordsTotal:function(){return"ssp"==B(this)?1*this._iRecordsTotal:this.aiDisplayMaster.length},fnRecordsDisplay:function(){return"ssp"==B(this)?1*this._iRecordsDisplay: -this.aiDisplay.length},fnDisplayEnd:function(){var a=this._iDisplayLength,b=this._iDisplayStart,c=b+a,e=this.aiDisplay.length,d=this.oFeatures,f=d.bPaginate;return d.bServerSide?!1===f||-1===a?b+e:Math.min(b+a,this._iRecordsDisplay):!f||c>e||-1===a?e:c},oInstance:null,sInstance:null,iTabIndex:0,nScrollHead:null,nScrollFoot:null,aLastSort:[],oPlugins:{}};m.ext=u={buttons:{},classes:{},errMode:"alert",feature:[],search:[],selector:{cell:[],column:[],row:[]},internal:{},legacy:{ajax:null},pager:{},renderer:{pageButton:{}, -header:{}},order:{},type:{detect:[],search:{},order:{}},_unique:0,fnVersionCheck:m.fnVersionCheck,iApiIndex:0,oJUIClasses:{},sVersion:m.version};h.extend(u,{afnFiltering:u.search,aTypes:u.type.detect,ofnSearch:u.type.search,oSort:u.type.order,afnSortData:u.order,aoFeatures:u.feature,oApi:u.internal,oStdClasses:u.classes,oPagination:u.pager});h.extend(m.ext.classes,{sTable:"dataTable",sNoFooter:"no-footer",sPageButton:"paginate_button",sPageButtonActive:"current",sPageButtonDisabled:"disabled",sStripeOdd:"odd", -sStripeEven:"even",sRowEmpty:"dataTables_empty",sWrapper:"dataTables_wrapper",sFilter:"dataTables_filter",sInfo:"dataTables_info",sPaging:"dataTables_paginate paging_",sLength:"dataTables_length",sProcessing:"dataTables_processing",sSortAsc:"sorting_asc",sSortDesc:"sorting_desc",sSortable:"sorting",sSortableAsc:"sorting_asc_disabled",sSortableDesc:"sorting_desc_disabled",sSortableNone:"sorting_disabled",sSortColumn:"sorting_",sFilterInput:"",sLengthSelect:"",sScrollWrapper:"dataTables_scroll",sScrollHead:"dataTables_scrollHead", -sScrollHeadInner:"dataTables_scrollHeadInner",sScrollBody:"dataTables_scrollBody",sScrollFoot:"dataTables_scrollFoot",sScrollFootInner:"dataTables_scrollFootInner",sHeaderTH:"",sFooterTH:"",sSortJUIAsc:"",sSortJUIDesc:"",sSortJUI:"",sSortJUIAscAllowed:"",sSortJUIDescAllowed:"",sSortJUIWrapper:"",sSortIcon:"",sJUIHeader:"",sJUIFooter:""});var Da="",Da="",F=Da+"ui-state-default",ja=Da+"css_right ui-icon ui-icon-",Xb=Da+"fg-toolbar ui-toolbar ui-widget-header ui-helper-clearfix";h.extend(m.ext.oJUIClasses, -m.ext.classes,{sPageButton:"fg-button ui-button "+F,sPageButtonActive:"ui-state-disabled",sPageButtonDisabled:"ui-state-disabled",sPaging:"dataTables_paginate fg-buttonset ui-buttonset fg-buttonset-multi ui-buttonset-multi paging_",sSortAsc:F+" sorting_asc",sSortDesc:F+" sorting_desc",sSortable:F+" sorting",sSortableAsc:F+" sorting_asc_disabled",sSortableDesc:F+" sorting_desc_disabled",sSortableNone:F+" sorting_disabled",sSortJUIAsc:ja+"triangle-1-n",sSortJUIDesc:ja+"triangle-1-s",sSortJUI:ja+"carat-2-n-s", -sSortJUIAscAllowed:ja+"carat-1-n",sSortJUIDescAllowed:ja+"carat-1-s",sSortJUIWrapper:"DataTables_sort_wrapper",sSortIcon:"DataTables_sort_icon",sScrollHead:"dataTables_scrollHead "+F,sScrollFoot:"dataTables_scrollFoot "+F,sHeaderTH:F,sFooterTH:F,sJUIHeader:Xb+" ui-corner-tl ui-corner-tr",sJUIFooter:Xb+" ui-corner-bl ui-corner-br"});var Mb=m.ext.pager;h.extend(Mb,{simple:function(){return["previous","next"]},full:function(){return["first","previous","next","last"]},simple_numbers:function(a,b){return["previous", -Wa(a,b),"next"]},full_numbers:function(a,b){return["first","previous",Wa(a,b),"next","last"]},_numbers:Wa,numbers_length:7});h.extend(!0,m.ext.renderer,{pageButton:{_:function(a,b,c,e,d,f){var g=a.oClasses,j=a.oLanguage.oPaginate,i,k,l=0,m=function(b,e){var n,r,t,s,u=function(b){Ta(a,b.data.action,true)};n=0;for(r=e.length;n").appendTo(b);m(t,s)}else{k=i="";switch(s){case "ellipsis":b.append('…');break; -case "first":i=j.sFirst;k=s+(d>0?"":" "+g.sPageButtonDisabled);break;case "previous":i=j.sPrevious;k=s+(d>0?"":" "+g.sPageButtonDisabled);break;case "next":i=j.sNext;k=s+(d",{"class":g.sPageButton+" "+k,"aria-controls":a.sTableId,"data-dt-idx":l,tabindex:a.iTabIndex,id:c===0&&typeof s==="string"?a.sTableId+"_"+s:null}).html(i).appendTo(b); -Va(t,{action:s},u);l++}}}},n;try{n=h(Q.activeElement).data("dt-idx")}catch(r){}m(h(b).empty(),e);n&&h(b).find("[data-dt-idx="+n+"]").focus()}}});h.extend(m.ext.type.detect,[function(a,b){var c=b.oLanguage.sDecimal;return Za(a,c)?"num"+c:null},function(a){if(a&&!(a instanceof Date)&&(!ac.test(a)||!bc.test(a)))return null;var b=Date.parse(a);return null!==b&&!isNaN(b)||J(a)?"date":null},function(a,b){var c=b.oLanguage.sDecimal;return Za(a,c,!0)?"num-fmt"+c:null},function(a,b){var c=b.oLanguage.sDecimal; -return Rb(a,c)?"html-num"+c:null},function(a,b){var c=b.oLanguage.sDecimal;return Rb(a,c,!0)?"html-num-fmt"+c:null},function(a){return J(a)||"string"===typeof a&&-1!==a.indexOf("<")?"html":null}]);h.extend(m.ext.type.search,{html:function(a){return J(a)?a:"string"===typeof a?a.replace(Ob," ").replace(Ba,""):""},string:function(a){return J(a)?a:"string"===typeof a?a.replace(Ob," "):a}});var Aa=function(a,b,c,e){if(0!==a&&(!a||"-"===a))return-Infinity;b&&(a=Qb(a,b));a.replace&&(c&&(a=a.replace(c,"")), -e&&(a=a.replace(e,"")));return 1*a};h.extend(u.type.order,{"date-pre":function(a){return Date.parse(a)||0},"html-pre":function(a){return J(a)?"":a.replace?a.replace(/<.*?>/g,"").toLowerCase():a+""},"string-pre":function(a){return J(a)?"":"string"===typeof a?a.toLowerCase():!a.toString?"":a.toString()},"string-asc":function(a,b){return ab?1:0},"string-desc":function(a,b){return ab?-1:0}});db("");h.extend(!0,m.ext.renderer,{header:{_:function(a,b,c,e){h(a.nTable).on("order.dt.DT",function(d, -f,g,h){if(a===f){d=c.idx;b.removeClass(c.sSortingClass+" "+e.sSortAsc+" "+e.sSortDesc).addClass(h[d]=="asc"?e.sSortAsc:h[d]=="desc"?e.sSortDesc:c.sSortingClass)}})},jqueryui:function(a,b,c,e){h("

    ").addClass(e.sSortJUIWrapper).append(b.contents()).append(h("").addClass(e.sSortIcon+" "+c.sSortingClassJUI)).appendTo(b);h(a.nTable).on("order.dt.DT",function(d,f,g,h){if(a===f){d=c.idx;b.removeClass(e.sSortAsc+" "+e.sSortDesc).addClass(h[d]=="asc"?e.sSortAsc:h[d]=="desc"?e.sSortDesc:c.sSortingClass); -b.find("span."+e.sSortIcon).removeClass(e.sSortJUIAsc+" "+e.sSortJUIDesc+" "+e.sSortJUI+" "+e.sSortJUIAscAllowed+" "+e.sSortJUIDescAllowed).addClass(h[d]=="asc"?e.sSortJUIAsc:h[d]=="desc"?e.sSortJUIDesc:c.sSortingClassJUI)}})}}});m.render={number:function(a,b,c,e){return{display:function(d){if("number"!==typeof d&&"string"!==typeof d)return d;var f=0>d?"-":"",d=Math.abs(parseFloat(d)),g=parseInt(d,10),d=c?b+(d-g).toFixed(c).substring(2):"";return f+(e||"")+g.toString().replace(/\B(?=(\d{3})+(?!\d))/g, -a)+d}}}};h.extend(m.ext.internal,{_fnExternApiFunc:Nb,_fnBuildAjax:ra,_fnAjaxUpdate:kb,_fnAjaxParameters:tb,_fnAjaxUpdateDraw:ub,_fnAjaxDataSrc:sa,_fnAddColumn:Fa,_fnColumnOptions:ka,_fnAdjustColumnSizing:X,_fnVisibleToColumnIndex:la,_fnColumnIndexToVisible:$,_fnVisbleColumns:aa,_fnGetColumns:Z,_fnColumnTypes:Ha,_fnApplyColumnDefs:ib,_fnHungarianMap:W,_fnCamelToHungarian:H,_fnLanguageCompat:P,_fnBrowserDetect:gb,_fnAddData:K,_fnAddTr:ma,_fnNodeToDataIndex:function(a,b){return b._DT_RowIndex!==k?b._DT_RowIndex: -null},_fnNodeToColumnIndex:function(a,b,c){return h.inArray(c,a.aoData[b].anCells)},_fnGetCellData:x,_fnSetCellData:Ia,_fnSplitObjNotation:Ka,_fnGetObjectDataFn:R,_fnSetObjectDataFn:S,_fnGetDataMaster:La,_fnClearTable:oa,_fnDeleteIndex:pa,_fnInvalidate:ca,_fnGetRowElements:na,_fnCreateTr:Ja,_fnBuildHead:jb,_fnDrawHead:ea,_fnDraw:M,_fnReDraw:N,_fnAddOptionsHtml:mb,_fnDetectHeader:da,_fnGetUniqueThs:qa,_fnFeatureHtmlFilter:ob,_fnFilterComplete:fa,_fnFilterCustom:xb,_fnFilterColumn:wb,_fnFilter:vb,_fnFilterCreateSearch:Qa, -_fnEscapeRegex:va,_fnFilterData:yb,_fnFeatureHtmlInfo:rb,_fnUpdateInfo:Bb,_fnInfoMacros:Cb,_fnInitialise:ga,_fnInitComplete:ta,_fnLengthChange:Ra,_fnFeatureHtmlLength:nb,_fnFeatureHtmlPaginate:sb,_fnPageChange:Ta,_fnFeatureHtmlProcessing:pb,_fnProcessingDisplay:C,_fnFeatureHtmlTable:qb,_fnScrollDraw:Y,_fnApplyToChildren:G,_fnCalculateColumnWidths:Ga,_fnThrottle:ua,_fnConvertToWidth:Db,_fnScrollingWidthAdjust:Fb,_fnGetWidestNode:Eb,_fnGetMaxLenString:Gb,_fnStringToCss:s,_fnScrollBarWidth:Hb,_fnSortFlatten:U, -_fnSort:lb,_fnSortAria:Jb,_fnSortListener:Ua,_fnSortAttachListener:Oa,_fnSortingClasses:xa,_fnSortData:Ib,_fnSaveState:ya,_fnLoadState:Kb,_fnSettingsFromNode:za,_fnLog:I,_fnMap:E,_fnBindAction:Va,_fnCallbackReg:z,_fnCallbackFire:w,_fnLengthOverflow:Sa,_fnRenderer:Pa,_fnDataSource:B,_fnRowAttributes:Ma,_fnCalculateEnd:function(){}});h.fn.dataTable=m;h.fn.dataTableSettings=m.settings;h.fn.dataTableExt=m.ext;h.fn.DataTable=function(a){return h(this).dataTable(a).api()};h.each(m,function(a,b){h.fn.DataTable[a]= -b});return h.fn.dataTable};"function"===typeof define&&define.amd?define("datatables",["jquery"],P):"object"===typeof exports?module.exports=P(require("jquery")):jQuery&&!jQuery.fn.dataTable&&P(jQuery)})(window,document); +/*! + DataTables 1.10.19 + ©2008-2018 SpryMedia Ltd - datatables.net/license +*/ +(function(h){"function"===typeof define&&define.amd?define(["jquery"],function(E){return h(E,window,document)}):"object"===typeof exports?module.exports=function(E,H){E||(E=window);H||(H="undefined"!==typeof window?require("jquery"):require("jquery")(E));return h(H,E,E.document)}:h(jQuery,window,document)})(function(h,E,H,k){function Z(a){var b,c,d={};h.each(a,function(e){if((b=e.match(/^([^A-Z]+?)([A-Z])/))&&-1!=="a aa ai ao as b fn i m o s ".indexOf(b[1]+" "))c=e.replace(b[0],b[2].toLowerCase()), +d[c]=e,"o"===b[1]&&Z(a[e])});a._hungarianMap=d}function J(a,b,c){a._hungarianMap||Z(a);var d;h.each(b,function(e){d=a._hungarianMap[e];if(d!==k&&(c||b[d]===k))"o"===d.charAt(0)?(b[d]||(b[d]={}),h.extend(!0,b[d],b[e]),J(a[d],b[d],c)):b[d]=b[e]})}function Ca(a){var b=n.defaults.oLanguage,c=b.sDecimal;c&&Da(c);if(a){var d=a.sZeroRecords;!a.sEmptyTable&&(d&&"No data available in table"===b.sEmptyTable)&&F(a,a,"sZeroRecords","sEmptyTable");!a.sLoadingRecords&&(d&&"Loading..."===b.sLoadingRecords)&&F(a, +a,"sZeroRecords","sLoadingRecords");a.sInfoThousands&&(a.sThousands=a.sInfoThousands);(a=a.sDecimal)&&c!==a&&Da(a)}}function fb(a){A(a,"ordering","bSort");A(a,"orderMulti","bSortMulti");A(a,"orderClasses","bSortClasses");A(a,"orderCellsTop","bSortCellsTop");A(a,"order","aaSorting");A(a,"orderFixed","aaSortingFixed");A(a,"paging","bPaginate");A(a,"pagingType","sPaginationType");A(a,"pageLength","iDisplayLength");A(a,"searching","bFilter");"boolean"===typeof a.sScrollX&&(a.sScrollX=a.sScrollX?"100%": +"");"boolean"===typeof a.scrollX&&(a.scrollX=a.scrollX?"100%":"");if(a=a.aoSearchCols)for(var b=0,c=a.length;b").css({position:"fixed",top:0,left:-1*h(E).scrollLeft(),height:1,width:1, +overflow:"hidden"}).append(h("
    ").css({position:"absolute",top:1,left:1,width:100,overflow:"scroll"}).append(h("
    ").css({width:"100%",height:10}))).appendTo("body"),d=c.children(),e=d.children();b.barWidth=d[0].offsetWidth-d[0].clientWidth;b.bScrollOversize=100===e[0].offsetWidth&&100!==d[0].clientWidth;b.bScrollbarLeft=1!==Math.round(e.offset().left);b.bBounding=c[0].getBoundingClientRect().width?!0:!1;c.remove()}h.extend(a.oBrowser,n.__browser);a.oScroll.iBarWidth=n.__browser.barWidth} +function ib(a,b,c,d,e,f){var g,j=!1;c!==k&&(g=c,j=!0);for(;d!==e;)a.hasOwnProperty(d)&&(g=j?b(g,a[d],d,a):a[d],j=!0,d+=f);return g}function Ea(a,b){var c=n.defaults.column,d=a.aoColumns.length,c=h.extend({},n.models.oColumn,c,{nTh:b?b:H.createElement("th"),sTitle:c.sTitle?c.sTitle:b?b.innerHTML:"",aDataSort:c.aDataSort?c.aDataSort:[d],mData:c.mData?c.mData:d,idx:d});a.aoColumns.push(c);c=a.aoPreSearchCols;c[d]=h.extend({},n.models.oSearch,c[d]);ka(a,d,h(b).data())}function ka(a,b,c){var b=a.aoColumns[b], +d=a.oClasses,e=h(b.nTh);if(!b.sWidthOrig){b.sWidthOrig=e.attr("width")||null;var f=(e.attr("style")||"").match(/width:\s*(\d+[pxem%]+)/);f&&(b.sWidthOrig=f[1])}c!==k&&null!==c&&(gb(c),J(n.defaults.column,c),c.mDataProp!==k&&!c.mData&&(c.mData=c.mDataProp),c.sType&&(b._sManualType=c.sType),c.className&&!c.sClass&&(c.sClass=c.className),c.sClass&&e.addClass(c.sClass),h.extend(b,c),F(b,c,"sWidth","sWidthOrig"),c.iDataSort!==k&&(b.aDataSort=[c.iDataSort]),F(b,c,"aDataSort"));var g=b.mData,j=S(g),i=b.mRender? +S(b.mRender):null,c=function(a){return"string"===typeof a&&-1!==a.indexOf("@")};b._bAttrSrc=h.isPlainObject(g)&&(c(g.sort)||c(g.type)||c(g.filter));b._setter=null;b.fnGetData=function(a,b,c){var d=j(a,b,k,c);return i&&b?i(d,b,a,c):d};b.fnSetData=function(a,b,c){return N(g)(a,b,c)};"number"!==typeof g&&(a._rowReadObject=!0);a.oFeatures.bSort||(b.bSortable=!1,e.addClass(d.sSortableNone));a=-1!==h.inArray("asc",b.asSorting);c=-1!==h.inArray("desc",b.asSorting);!b.bSortable||!a&&!c?(b.sSortingClass=d.sSortableNone, +b.sSortingClassJUI=""):a&&!c?(b.sSortingClass=d.sSortableAsc,b.sSortingClassJUI=d.sSortJUIAscAllowed):!a&&c?(b.sSortingClass=d.sSortableDesc,b.sSortingClassJUI=d.sSortJUIDescAllowed):(b.sSortingClass=d.sSortable,b.sSortingClassJUI=d.sSortJUI)}function $(a){if(!1!==a.oFeatures.bAutoWidth){var b=a.aoColumns;Fa(a);for(var c=0,d=b.length;cq[f])d(l.length+q[f],m);else if("string"=== +typeof q[f]){j=0;for(i=l.length;jb&&a[e]--; -1!=d&&c===k&&a.splice(d, +1)}function da(a,b,c,d){var e=a.aoData[b],f,g=function(c,d){for(;c.childNodes.length;)c.removeChild(c.firstChild);c.innerHTML=B(a,b,d,"display")};if("dom"===c||(!c||"auto"===c)&&"dom"===e.src)e._aData=Ia(a,e,d,d===k?k:e._aData).data;else{var j=e.anCells;if(j)if(d!==k)g(j[d],d);else{c=0;for(f=j.length;c").appendTo(g));b=0;for(c=l.length;btr").attr("role","row");h(g).find(">tr>th, >tr>td").addClass(m.sHeaderTH);h(j).find(">tr>th, >tr>td").addClass(m.sFooterTH);if(null!==j){a=a.aoFooter[0];b=0;for(c=a.length;b=a.fnRecordsDisplay()?0:g,a.iInitDisplayStart=-1);var g=a._iDisplayStart,m=a.fnDisplayEnd();if(a.bDeferLoading)a.bDeferLoading=!1,a.iDraw++,C(a,!1);else if(j){if(!a.bDestroying&&!mb(a))return}else a.iDraw++;if(0!==i.length){f=j?a.aoData.length:m;for(j=j?0:g;j",{"class":e?d[0]:""}).append(h("",{valign:"top",colSpan:V(a),"class":a.oClasses.sRowEmpty}).html(c))[0];r(a,"aoHeaderCallback","header",[h(a.nTHead).children("tr")[0],Ka(a),g,m,i]);r(a,"aoFooterCallback","footer",[h(a.nTFoot).children("tr")[0],Ka(a),g,m,i]);d=h(a.nTBody);d.children().detach(); +d.append(h(b));r(a,"aoDrawCallback","draw",[a]);a.bSorted=!1;a.bFiltered=!1;a.bDrawing=!1}}function T(a,b){var c=a.oFeatures,d=c.bFilter;c.bSort&&nb(a);d?ga(a,a.oPreviousSearch):a.aiDisplay=a.aiDisplayMaster.slice();!0!==b&&(a._iDisplayStart=0);a._drawHold=b;P(a);a._drawHold=!1}function ob(a){var b=a.oClasses,c=h(a.nTable),c=h("
    ").insertBefore(c),d=a.oFeatures,e=h("
    ",{id:a.sTableId+"_wrapper","class":b.sWrapper+(a.nTFoot?"":" "+b.sNoFooter)});a.nHolding=c[0];a.nTableWrapper=e[0];a.nTableReinsertBefore= +a.nTable.nextSibling;for(var f=a.sDom.split(""),g,j,i,m,l,q,k=0;k")[0];m=f[k+1];if("'"==m||'"'==m){l="";for(q=2;f[k+q]!=m;)l+=f[k+q],q++;"H"==l?l=b.sJUIHeader:"F"==l&&(l=b.sJUIFooter);-1!=l.indexOf(".")?(m=l.split("."),i.id=m[0].substr(1,m[0].length-1),i.className=m[1]):"#"==l.charAt(0)?i.id=l.substr(1,l.length-1):i.className=l;k+=q}e.append(i);e=h(i)}else if(">"==j)e=e.parent();else if("l"==j&&d.bPaginate&&d.bLengthChange)g=pb(a);else if("f"==j&& +d.bFilter)g=qb(a);else if("r"==j&&d.bProcessing)g=rb(a);else if("t"==j)g=sb(a);else if("i"==j&&d.bInfo)g=tb(a);else if("p"==j&&d.bPaginate)g=ub(a);else if(0!==n.ext.feature.length){i=n.ext.feature;q=0;for(m=i.length;q',j=d.sSearch,j=j.match(/_INPUT_/)?j.replace("_INPUT_", +g):j+g,b=h("
    ",{id:!f.f?c+"_filter":null,"class":b.sFilter}).append(h("
    ").addClass(b.sLength);a.aanFeatures.l||(i[0].id=c+"_length");i.children().append(a.oLanguage.sLengthMenu.replace("_MENU_",e[0].outerHTML));h("select",i).val(a._iDisplayLength).on("change.DT",function(){Ra(a,h(this).val());P(a)});h(a.nTable).on("length.dt.DT",function(b,c,d){a=== +c&&h("select",i).val(d)});return i[0]}function ub(a){var b=a.sPaginationType,c=n.ext.pager[b],d="function"===typeof c,e=function(a){P(a)},b=h("
    ").addClass(a.oClasses.sPaging+b)[0],f=a.aanFeatures;d||c.fnInit(a,b,e);f.p||(b.id=a.sTableId+"_paginate",a.aoDrawCallback.push({fn:function(a){if(d){var b=a._iDisplayStart,i=a._iDisplayLength,h=a.fnRecordsDisplay(),l=-1===i,b=l?0:Math.ceil(b/i),i=l?1:Math.ceil(h/i),h=c(b,i),k,l=0;for(k=f.p.length;lf&&(d=0)):"first"==b?d=0:"previous"==b?(d=0<=e?d-e:0,0>d&&(d=0)):"next"==b?d+e",{id:!a.aanFeatures.r?a.sTableId+"_processing":null,"class":a.oClasses.sProcessing}).html(a.oLanguage.sProcessing).insertBefore(a.nTable)[0]} +function C(a,b){a.oFeatures.bProcessing&&h(a.aanFeatures.r).css("display",b?"block":"none");r(a,null,"processing",[a,b])}function sb(a){var b=h(a.nTable);b.attr("role","grid");var c=a.oScroll;if(""===c.sX&&""===c.sY)return a.nTable;var d=c.sX,e=c.sY,f=a.oClasses,g=b.children("caption"),j=g.length?g[0]._captionSide:null,i=h(b[0].cloneNode(!1)),m=h(b[0].cloneNode(!1)),l=b.children("tfoot");l.length||(l=null);i=h("
    ",{"class":f.sScrollWrapper}).append(h("
    ",{"class":f.sScrollHead}).css({overflow:"hidden", +position:"relative",border:0,width:d?!d?null:v(d):"100%"}).append(h("
    ",{"class":f.sScrollHeadInner}).css({"box-sizing":"content-box",width:c.sXInner||"100%"}).append(i.removeAttr("id").css("margin-left",0).append("top"===j?g:null).append(b.children("thead"))))).append(h("
    ",{"class":f.sScrollBody}).css({position:"relative",overflow:"auto",width:!d?null:v(d)}).append(b));l&&i.append(h("
    ",{"class":f.sScrollFoot}).css({overflow:"hidden",border:0,width:d?!d?null:v(d):"100%"}).append(h("
    ", +{"class":f.sScrollFootInner}).append(m.removeAttr("id").css("margin-left",0).append("bottom"===j?g:null).append(b.children("tfoot")))));var b=i.children(),k=b[0],f=b[1],t=l?b[2]:null;if(d)h(f).on("scroll.DT",function(){var a=this.scrollLeft;k.scrollLeft=a;l&&(t.scrollLeft=a)});h(f).css(e&&c.bCollapse?"max-height":"height",e);a.nScrollHead=k;a.nScrollBody=f;a.nScrollFoot=t;a.aoDrawCallback.push({fn:la,sName:"scrolling"});return i[0]}function la(a){var b=a.oScroll,c=b.sX,d=b.sXInner,e=b.sY,b=b.iBarWidth, +f=h(a.nScrollHead),g=f[0].style,j=f.children("div"),i=j[0].style,m=j.children("table"),j=a.nScrollBody,l=h(j),q=j.style,t=h(a.nScrollFoot).children("div"),n=t.children("table"),o=h(a.nTHead),p=h(a.nTable),s=p[0],r=s.style,u=a.nTFoot?h(a.nTFoot):null,x=a.oBrowser,U=x.bScrollOversize,Xb=D(a.aoColumns,"nTh"),Q,L,R,w,Ua=[],y=[],z=[],A=[],B,C=function(a){a=a.style;a.paddingTop="0";a.paddingBottom="0";a.borderTopWidth="0";a.borderBottomWidth="0";a.height=0};L=j.scrollHeight>j.clientHeight;if(a.scrollBarVis!== +L&&a.scrollBarVis!==k)a.scrollBarVis=L,$(a);else{a.scrollBarVis=L;p.children("thead, tfoot").remove();u&&(R=u.clone().prependTo(p),Q=u.find("tr"),R=R.find("tr"));w=o.clone().prependTo(p);o=o.find("tr");L=w.find("tr");w.find("th, td").removeAttr("tabindex");c||(q.width="100%",f[0].style.width="100%");h.each(ra(a,w),function(b,c){B=aa(a,b);c.style.width=a.aoColumns[B].sWidth});u&&I(function(a){a.style.width=""},R);f=p.outerWidth();if(""===c){r.width="100%";if(U&&(p.find("tbody").height()>j.offsetHeight|| +"scroll"==l.css("overflow-y")))r.width=v(p.outerWidth()-b);f=p.outerWidth()}else""!==d&&(r.width=v(d),f=p.outerWidth());I(C,L);I(function(a){z.push(a.innerHTML);Ua.push(v(h(a).css("width")))},L);I(function(a,b){if(h.inArray(a,Xb)!==-1)a.style.width=Ua[b]},o);h(L).height(0);u&&(I(C,R),I(function(a){A.push(a.innerHTML);y.push(v(h(a).css("width")))},R),I(function(a,b){a.style.width=y[b]},Q),h(R).height(0));I(function(a,b){a.innerHTML='
    '+z[b]+"
    ";a.childNodes[0].style.height= +"0";a.childNodes[0].style.overflow="hidden";a.style.width=Ua[b]},L);u&&I(function(a,b){a.innerHTML='
    '+A[b]+"
    ";a.childNodes[0].style.height="0";a.childNodes[0].style.overflow="hidden";a.style.width=y[b]},R);if(p.outerWidth()j.offsetHeight||"scroll"==l.css("overflow-y")?f+b:f;if(U&&(j.scrollHeight>j.offsetHeight||"scroll"==l.css("overflow-y")))r.width=v(Q-b);(""===c||""!==d)&&K(a,1,"Possible column misalignment",6)}else Q="100%";q.width=v(Q); +g.width=v(Q);u&&(a.nScrollFoot.style.width=v(Q));!e&&U&&(q.height=v(s.offsetHeight+b));c=p.outerWidth();m[0].style.width=v(c);i.width=v(c);d=p.height()>j.clientHeight||"scroll"==l.css("overflow-y");e="padding"+(x.bScrollbarLeft?"Left":"Right");i[e]=d?b+"px":"0px";u&&(n[0].style.width=v(c),t[0].style.width=v(c),t[0].style[e]=d?b+"px":"0px");p.children("colgroup").insertBefore(p.children("thead"));l.scroll();if((a.bSorted||a.bFiltered)&&!a._drawHold)j.scrollTop=0}}function I(a,b,c){for(var d=0,e=0, +f=b.length,g,j;e").appendTo(j.find("tbody"));j.find("thead, tfoot").remove();j.append(h(a.nTHead).clone()).append(h(a.nTFoot).clone());j.find("tfoot th, tfoot td").css("width","");m=ra(a,j.find("thead")[0]);for(n=0;n").css({width:o.sWidthOrig,margin:0,padding:0,border:0,height:1}));if(a.aoData.length)for(n=0;n").css(f||e?{position:"absolute",top:0,left:0,height:1,right:0,overflow:"hidden"}:{}).append(j).appendTo(k);f&&g?j.width(g):f?(j.css("width","auto"),j.removeAttr("width"),j.width()").css("width",v(a)).appendTo(b||H.body),d=c[0].offsetWidth;c.remove();return d}function Gb(a, +b){var c=Hb(a,b);if(0>c)return null;var d=a.aoData[c];return!d.nTr?h("").html(B(a,c,b,"display"))[0]:d.anCells[b]}function Hb(a,b){for(var c,d=-1,e=-1,f=0,g=a.aoData.length;fd&&(d=c.length,e=f);return e}function v(a){return null===a?"0px":"number"==typeof a?0>a?"0px":a+"px":a.match(/\d$/)?a+"px":a}function X(a){var b,c,d=[],e=a.aoColumns,f,g,j,i;b=a.aaSortingFixed;c=h.isPlainObject(b);var m=[];f=function(a){a.length&& +!h.isArray(a[0])?m.push(a):h.merge(m,a)};h.isArray(b)&&f(b);c&&b.pre&&f(b.pre);f(a.aaSorting);c&&b.post&&f(b.post);for(a=0;ae?1:0,0!==c)return"asc"===j.dir?c:-c;c=d[a];e=d[b];return ce?1:0}):i.sort(function(a,b){var c,g,j,i,k=h.length,n=f[a]._aSortData,o=f[b]._aSortData;for(j=0;jg?1:0})}a.bSorted=!0}function Jb(a){for(var b,c,d=a.aoColumns,e=X(a),a=a.oLanguage.oAria,f=0,g=d.length;f/g,"");var i=c.nTh;i.removeAttribute("aria-sort");c.bSortable&&(0e?e+1:3));e=0;for(f=d.length;ee?e+1:3))}a.aLastSort=d}function Ib(a,b){var c=a.aoColumns[b],d=n.ext.order[c.sSortDataType],e;d&&(e=d.call(a.oInstance,a,b,ba(a,b)));for(var f,g=n.ext.type.order[c.sType+"-pre"],j=0,i=a.aoData.length;j=f.length?[0,c[1]]:c)}));b.search!==k&&h.extend(a.oPreviousSearch,Cb(b.search));if(b.columns){d=0;for(e=b.columns.length;d=c&&(b=c-d);b-=b%d;if(-1===d||0>b)b=0;a._iDisplayStart=b}function Na(a,b){var c=a.renderer,d=n.ext.renderer[b];return h.isPlainObject(c)&&c[b]?d[c[b]]||d._:"string"=== +typeof c?d[c]||d._:d._}function y(a){return a.oFeatures.bServerSide?"ssp":a.ajax||a.sAjaxSource?"ajax":"dom"}function ia(a,b){var c=[],c=Lb.numbers_length,d=Math.floor(c/2);b<=c?c=Y(0,b):a<=d?(c=Y(0,c-2),c.push("ellipsis"),c.push(b-1)):(a>=b-1-d?c=Y(b-(c-2),b):(c=Y(a-d+2,a+d-1),c.push("ellipsis"),c.push(b-1)),c.splice(0,0,"ellipsis"),c.splice(0,0,0));c.DT_el="span";return c}function Da(a){h.each({num:function(b){return za(b,a)},"num-fmt":function(b){return za(b,a,Ya)},"html-num":function(b){return za(b, +a,Aa)},"html-num-fmt":function(b){return za(b,a,Aa,Ya)}},function(b,c){x.type.order[b+a+"-pre"]=c;b.match(/^html\-/)&&(x.type.search[b+a]=x.type.search.html)})}function Mb(a){return function(){var b=[ya(this[n.ext.iApiIndex])].concat(Array.prototype.slice.call(arguments));return n.ext.internal[a].apply(this,b)}}var n=function(a){this.$=function(a,b){return this.api(!0).$(a,b)};this._=function(a,b){return this.api(!0).rows(a,b).data()};this.api=function(a){return a?new s(ya(this[x.iApiIndex])):new s(this)}; +this.fnAddData=function(a,b){var c=this.api(!0),d=h.isArray(a)&&(h.isArray(a[0])||h.isPlainObject(a[0]))?c.rows.add(a):c.row.add(a);(b===k||b)&&c.draw();return d.flatten().toArray()};this.fnAdjustColumnSizing=function(a){var b=this.api(!0).columns.adjust(),c=b.settings()[0],d=c.oScroll;a===k||a?b.draw(!1):(""!==d.sX||""!==d.sY)&&la(c)};this.fnClearTable=function(a){var b=this.api(!0).clear();(a===k||a)&&b.draw()};this.fnClose=function(a){this.api(!0).row(a).child.hide()};this.fnDeleteRow=function(a, +b,c){var d=this.api(!0),a=d.rows(a),e=a.settings()[0],h=e.aoData[a[0][0]];a.remove();b&&b.call(this,e,h);(c===k||c)&&d.draw();return h};this.fnDestroy=function(a){this.api(!0).destroy(a)};this.fnDraw=function(a){this.api(!0).draw(a)};this.fnFilter=function(a,b,c,d,e,h){e=this.api(!0);null===b||b===k?e.search(a,c,d,h):e.column(b).search(a,c,d,h);e.draw()};this.fnGetData=function(a,b){var c=this.api(!0);if(a!==k){var d=a.nodeName?a.nodeName.toLowerCase():"";return b!==k||"td"==d||"th"==d?c.cell(a,b).data(): +c.row(a).data()||null}return c.data().toArray()};this.fnGetNodes=function(a){var b=this.api(!0);return a!==k?b.row(a).node():b.rows().nodes().flatten().toArray()};this.fnGetPosition=function(a){var b=this.api(!0),c=a.nodeName.toUpperCase();return"TR"==c?b.row(a).index():"TD"==c||"TH"==c?(a=b.cell(a).index(),[a.row,a.columnVisible,a.column]):null};this.fnIsOpen=function(a){return this.api(!0).row(a).child.isShown()};this.fnOpen=function(a,b,c){return this.api(!0).row(a).child(b,c).show().child()[0]}; +this.fnPageChange=function(a,b){var c=this.api(!0).page(a);(b===k||b)&&c.draw(!1)};this.fnSetColumnVis=function(a,b,c){a=this.api(!0).column(a).visible(b);(c===k||c)&&a.columns.adjust().draw()};this.fnSettings=function(){return ya(this[x.iApiIndex])};this.fnSort=function(a){this.api(!0).order(a).draw()};this.fnSortListener=function(a,b,c){this.api(!0).order.listener(a,b,c)};this.fnUpdate=function(a,b,c,d,e){var h=this.api(!0);c===k||null===c?h.row(b).data(a):h.cell(b,c).data(a);(e===k||e)&&h.columns.adjust(); +(d===k||d)&&h.draw();return 0};this.fnVersionCheck=x.fnVersionCheck;var b=this,c=a===k,d=this.length;c&&(a={});this.oApi=this.internal=x.internal;for(var e in n.ext.internal)e&&(this[e]=Mb(e));this.each(function(){var e={},g=1").appendTo(q)); +p.nTHead=b[0];b=q.children("tbody");b.length===0&&(b=h("").appendTo(q));p.nTBody=b[0];b=q.children("tfoot");if(b.length===0&&a.length>0&&(p.oScroll.sX!==""||p.oScroll.sY!==""))b=h("").appendTo(q);if(b.length===0||b.children().length===0)q.addClass(u.sNoFooter);else if(b.length>0){p.nTFoot=b[0];ea(p.aoFooter,p.nTFoot)}if(g.aaData)for(j=0;j/g,Zb=/^\d{2,4}[\.\/\-]\d{1,2}[\.\/\-]\d{1,2}([T ]{1}\d{1,2}[:\.]\d{2}([\.:]\d{2})?)?$/,$b=RegExp("(\\/|\\.|\\*|\\+|\\?|\\||\\(|\\)|\\[|\\]|\\{|\\}|\\\\|\\$|\\^|\\-)","g"),Ya=/[',$£€¥%\u2009\u202F\u20BD\u20a9\u20BArfkɃΞ]/gi,M=function(a){return!a||!0===a||"-"===a?!0:!1},Ob=function(a){var b=parseInt(a,10);return!isNaN(b)&& +isFinite(a)?b:null},Pb=function(a,b){Za[b]||(Za[b]=RegExp(Qa(b),"g"));return"string"===typeof a&&"."!==b?a.replace(/\./g,"").replace(Za[b],"."):a},$a=function(a,b,c){var d="string"===typeof a;if(M(a))return!0;b&&d&&(a=Pb(a,b));c&&d&&(a=a.replace(Ya,""));return!isNaN(parseFloat(a))&&isFinite(a)},Qb=function(a,b,c){return M(a)?!0:!(M(a)||"string"===typeof a)?null:$a(a.replace(Aa,""),b,c)?!0:null},D=function(a,b,c){var d=[],e=0,f=a.length;if(c!==k)for(;ea.length)){b=a.slice().sort();for(var c=b[0],d=1,e=b.length;d")[0],Wb=va.textContent!==k,Yb= +/<.*?>/g,Oa=n.util.throttle,Sb=[],w=Array.prototype,ac=function(a){var b,c,d=n.settings,e=h.map(d,function(a){return a.nTable});if(a){if(a.nTable&&a.oApi)return[a];if(a.nodeName&&"table"===a.nodeName.toLowerCase())return b=h.inArray(a,e),-1!==b?[d[b]]:null;if(a&&"function"===typeof a.settings)return a.settings().toArray();"string"===typeof a?c=h(a):a instanceof h&&(c=a)}else return[];if(c)return c.map(function(){b=h.inArray(this,e);return-1!==b?d[b]:null}).toArray()};s=function(a,b){if(!(this instanceof +s))return new s(a,b);var c=[],d=function(a){(a=ac(a))&&(c=c.concat(a))};if(h.isArray(a))for(var e=0,f=a.length;ea?new s(b[a],this[a]):null},filter:function(a){var b=[];if(w.filter)b=w.filter.call(this,a,this);else for(var c=0,d=this.length;c").addClass(b),h("td",c).addClass(b).html(a)[0].colSpan=V(d),e.push(c[0]))};f(a,b);c._details&&c._details.detach();c._details=h(e); +c._detailsShow&&c._details.insertAfter(c.nTr)}return this});o(["row().child.show()","row().child().show()"],function(){Ub(this,!0);return this});o(["row().child.hide()","row().child().hide()"],function(){Ub(this,!1);return this});o(["row().child.remove()","row().child().remove()"],function(){db(this);return this});o("row().child.isShown()",function(){var a=this.context;return a.length&&this.length?a[0].aoData[this[0]]._detailsShow||!1:!1});var bc=/^([^:]+):(name|visIdx|visible)$/,Vb=function(a,b, +c,d,e){for(var c=[],d=0,f=e.length;d=0?b:g.length+b];if(typeof a==="function"){var e=Ba(c,f);return h.map(g,function(b,f){return a(f,Vb(c,f,0,0,e),i[f])?f:null})}var k=typeof a==="string"?a.match(bc): +"";if(k)switch(k[2]){case "visIdx":case "visible":b=parseInt(k[1],10);if(b<0){var n=h.map(g,function(a,b){return a.bVisible?b:null});return[n[n.length+b]]}return[aa(c,b)];case "name":return h.map(j,function(a,b){return a===k[1]?b:null});default:return[]}if(a.nodeName&&a._DT_CellIndex)return[a._DT_CellIndex.column];b=h(i).filter(a).map(function(){return h.inArray(this,i)}).toArray();if(b.length||!a.nodeName)return b;b=h(a).closest("*[data-dt-column]");return b.length?[b.data("dt-column")]:[]},c,f)}, +1);c.selector.cols=a;c.selector.opts=b;return c});u("columns().header()","column().header()",function(){return this.iterator("column",function(a,b){return a.aoColumns[b].nTh},1)});u("columns().footer()","column().footer()",function(){return this.iterator("column",function(a,b){return a.aoColumns[b].nTf},1)});u("columns().data()","column().data()",function(){return this.iterator("column-rows",Vb,1)});u("columns().dataSrc()","column().dataSrc()",function(){return this.iterator("column",function(a,b){return a.aoColumns[b].mData}, +1)});u("columns().cache()","column().cache()",function(a){return this.iterator("column-rows",function(b,c,d,e,f){return ja(b.aoData,f,"search"===a?"_aFilterData":"_aSortData",c)},1)});u("columns().nodes()","column().nodes()",function(){return this.iterator("column-rows",function(a,b,c,d,e){return ja(a.aoData,e,"anCells",b)},1)});u("columns().visible()","column().visible()",function(a,b){var c=this.iterator("column",function(b,c){if(a===k)return b.aoColumns[c].bVisible;var f=b.aoColumns,g=f[c],j=b.aoData, +i,m,l;if(a!==k&&g.bVisible!==a){if(a){var n=h.inArray(!0,D(f,"bVisible"),c+1);i=0;for(m=j.length;id;return!0};n.isDataTable= +n.fnIsDataTable=function(a){var b=h(a).get(0),c=!1;if(a instanceof n.Api)return!0;h.each(n.settings,function(a,e){var f=e.nScrollHead?h("table",e.nScrollHead)[0]:null,g=e.nScrollFoot?h("table",e.nScrollFoot)[0]:null;if(e.nTable===b||f===b||g===b)c=!0});return c};n.tables=n.fnTables=function(a){var b=!1;h.isPlainObject(a)&&(b=a.api,a=a.visible);var c=h.map(n.settings,function(b){if(!a||a&&h(b.nTable).is(":visible"))return b.nTable});return b?new s(c):c};n.camelToHungarian=J;o("$()",function(a,b){var c= +this.rows(b).nodes(),c=h(c);return h([].concat(c.filter(a).toArray(),c.find(a).toArray()))});h.each(["on","one","off"],function(a,b){o(b+"()",function(){var a=Array.prototype.slice.call(arguments);a[0]=h.map(a[0].split(/\s/),function(a){return!a.match(/\.dt\b/)?a+".dt":a}).join(" ");var d=h(this.tables().nodes());d[b].apply(d,a);return this})});o("clear()",function(){return this.iterator("table",function(a){oa(a)})});o("settings()",function(){return new s(this.context,this.context)});o("init()",function(){var a= +this.context;return a.length?a[0].oInit:null});o("data()",function(){return this.iterator("table",function(a){return D(a.aoData,"_aData")}).flatten()});o("destroy()",function(a){a=a||!1;return this.iterator("table",function(b){var c=b.nTableWrapper.parentNode,d=b.oClasses,e=b.nTable,f=b.nTBody,g=b.nTHead,j=b.nTFoot,i=h(e),f=h(f),k=h(b.nTableWrapper),l=h.map(b.aoData,function(a){return a.nTr}),o;b.bDestroying=!0;r(b,"aoDestroyCallback","destroy",[b]);a||(new s(b)).columns().visible(!0);k.off(".DT").find(":not(tbody *)").off(".DT"); +h(E).off(".DT-"+b.sInstance);e!=g.parentNode&&(i.children("thead").detach(),i.append(g));j&&e!=j.parentNode&&(i.children("tfoot").detach(),i.append(j));b.aaSorting=[];b.aaSortingFixed=[];wa(b);h(l).removeClass(b.asStripeClasses.join(" "));h("th, td",g).removeClass(d.sSortable+" "+d.sSortableAsc+" "+d.sSortableDesc+" "+d.sSortableNone);f.children().detach();f.append(l);g=a?"remove":"detach";i[g]();k[g]();!a&&c&&(c.insertBefore(e,b.nTableReinsertBefore),i.css("width",b.sDestroyWidth).removeClass(d.sTable), +(o=b.asDestroyStripes.length)&&f.children().each(function(a){h(this).addClass(b.asDestroyStripes[a%o])}));c=h.inArray(b,n.settings);-1!==c&&n.settings.splice(c,1)})});h.each(["column","row","cell"],function(a,b){o(b+"s().every()",function(a){var d=this.selector.opts,e=this;return this.iterator(b,function(f,g,h,i,m){a.call(e[b](g,"cell"===b?h:d,"cell"===b?d:k),g,h,i,m)})})});o("i18n()",function(a,b,c){var d=this.context[0],a=S(a)(d.oLanguage);a===k&&(a=b);c!==k&&h.isPlainObject(a)&&(a=a[c]!==k?a[c]: +a._);return a.replace("%d",c)});n.version="1.10.19";n.settings=[];n.models={};n.models.oSearch={bCaseInsensitive:!0,sSearch:"",bRegex:!1,bSmart:!0};n.models.oRow={nTr:null,anCells:null,_aData:[],_aSortData:null,_aFilterData:null,_sFilterRow:null,_sRowStripe:"",src:null,idx:-1};n.models.oColumn={idx:null,aDataSort:null,asSorting:null,bSearchable:null,bSortable:null,bVisible:null,_sManualType:null,_bAttrSrc:!1,fnCreatedCell:null,fnGetData:null,fnSetData:null,mData:null,mRender:null,nTh:null,nTf:null, +sClass:null,sContentPadding:null,sDefaultContent:null,sName:null,sSortDataType:"std",sSortingClass:null,sSortingClassJUI:null,sTitle:null,sType:null,sWidth:null,sWidthOrig:null};n.defaults={aaData:null,aaSorting:[[0,"asc"]],aaSortingFixed:[],ajax:null,aLengthMenu:[10,25,50,100],aoColumns:null,aoColumnDefs:null,aoSearchCols:[],asStripeClasses:null,bAutoWidth:!0,bDeferRender:!1,bDestroy:!1,bFilter:!0,bInfo:!0,bLengthChange:!0,bPaginate:!0,bProcessing:!1,bRetrieve:!1,bScrollCollapse:!1,bServerSide:!1, +bSort:!0,bSortMulti:!0,bSortCellsTop:!1,bSortClasses:!0,bStateSave:!1,fnCreatedRow:null,fnDrawCallback:null,fnFooterCallback:null,fnFormatNumber:function(a){return a.toString().replace(/\B(?=(\d{3})+(?!\d))/g,this.oLanguage.sThousands)},fnHeaderCallback:null,fnInfoCallback:null,fnInitComplete:null,fnPreDrawCallback:null,fnRowCallback:null,fnServerData:null,fnServerParams:null,fnStateLoadCallback:function(a){try{return JSON.parse((-1===a.iStateDuration?sessionStorage:localStorage).getItem("DataTables_"+ +a.sInstance+"_"+location.pathname))}catch(b){}},fnStateLoadParams:null,fnStateLoaded:null,fnStateSaveCallback:function(a,b){try{(-1===a.iStateDuration?sessionStorage:localStorage).setItem("DataTables_"+a.sInstance+"_"+location.pathname,JSON.stringify(b))}catch(c){}},fnStateSaveParams:null,iStateDuration:7200,iDeferLoading:null,iDisplayLength:10,iDisplayStart:0,iTabIndex:0,oClasses:{},oLanguage:{oAria:{sSortAscending:": activate to sort column ascending",sSortDescending:": activate to sort column descending"}, +oPaginate:{sFirst:"First",sLast:"Last",sNext:"Next",sPrevious:"Previous"},sEmptyTable:"No data available in table",sInfo:"Showing _START_ to _END_ of _TOTAL_ entries",sInfoEmpty:"Showing 0 to 0 of 0 entries",sInfoFiltered:"(filtered from _MAX_ total entries)",sInfoPostFix:"",sDecimal:"",sThousands:",",sLengthMenu:"Show _MENU_ entries",sLoadingRecords:"Loading...",sProcessing:"Processing...",sSearch:"Search:",sSearchPlaceholder:"",sUrl:"",sZeroRecords:"No matching records found"},oSearch:h.extend({}, +n.models.oSearch),sAjaxDataProp:"data",sAjaxSource:null,sDom:"lfrtip",searchDelay:null,sPaginationType:"simple_numbers",sScrollX:"",sScrollXInner:"",sScrollY:"",sServerMethod:"GET",renderer:null,rowId:"DT_RowId"};Z(n.defaults);n.defaults.column={aDataSort:null,iDataSort:-1,asSorting:["asc","desc"],bSearchable:!0,bSortable:!0,bVisible:!0,fnCreatedCell:null,mData:null,mRender:null,sCellType:"td",sClass:"",sContentPadding:"",sDefaultContent:null,sName:"",sSortDataType:"std",sTitle:null,sType:null,sWidth:null}; +Z(n.defaults.column);n.models.oSettings={oFeatures:{bAutoWidth:null,bDeferRender:null,bFilter:null,bInfo:null,bLengthChange:null,bPaginate:null,bProcessing:null,bServerSide:null,bSort:null,bSortMulti:null,bSortClasses:null,bStateSave:null},oScroll:{bCollapse:null,iBarWidth:0,sX:null,sXInner:null,sY:null},oLanguage:{fnInfoCallback:null},oBrowser:{bScrollOversize:!1,bScrollbarLeft:!1,bBounding:!1,barWidth:0},ajax:null,aanFeatures:[],aoData:[],aiDisplay:[],aiDisplayMaster:[],aIds:{},aoColumns:[],aoHeader:[], +aoFooter:[],oPreviousSearch:{},aoPreSearchCols:[],aaSorting:null,aaSortingFixed:[],asStripeClasses:null,asDestroyStripes:[],sDestroyWidth:0,aoRowCallback:[],aoHeaderCallback:[],aoFooterCallback:[],aoDrawCallback:[],aoRowCreatedCallback:[],aoPreDrawCallback:[],aoInitComplete:[],aoStateSaveParams:[],aoStateLoadParams:[],aoStateLoaded:[],sTableId:"",nTable:null,nTHead:null,nTFoot:null,nTBody:null,nTableWrapper:null,bDeferLoading:!1,bInitialised:!1,aoOpenRows:[],sDom:null,searchDelay:null,sPaginationType:"two_button", +iStateDuration:0,aoStateSave:[],aoStateLoad:[],oSavedState:null,oLoadedState:null,sAjaxSource:null,sAjaxDataProp:null,bAjaxDataGet:!0,jqXHR:null,json:k,oAjaxData:k,fnServerData:null,aoServerParams:[],sServerMethod:null,fnFormatNumber:null,aLengthMenu:null,iDraw:0,bDrawing:!1,iDrawError:-1,_iDisplayLength:10,_iDisplayStart:0,_iRecordsTotal:0,_iRecordsDisplay:0,oClasses:{},bFiltered:!1,bSorted:!1,bSortCellsTop:null,oInit:null,aoDestroyCallback:[],fnRecordsTotal:function(){return"ssp"==y(this)?1*this._iRecordsTotal: +this.aiDisplayMaster.length},fnRecordsDisplay:function(){return"ssp"==y(this)?1*this._iRecordsDisplay:this.aiDisplay.length},fnDisplayEnd:function(){var a=this._iDisplayLength,b=this._iDisplayStart,c=b+a,d=this.aiDisplay.length,e=this.oFeatures,f=e.bPaginate;return e.bServerSide?!1===f||-1===a?b+d:Math.min(b+a,this._iRecordsDisplay):!f||c>d||-1===a?d:c},oInstance:null,sInstance:null,iTabIndex:0,nScrollHead:null,nScrollFoot:null,aLastSort:[],oPlugins:{},rowIdFn:null,rowId:null};n.ext=x={buttons:{}, +classes:{},builder:"-source-",errMode:"alert",feature:[],search:[],selector:{cell:[],column:[],row:[]},internal:{},legacy:{ajax:null},pager:{},renderer:{pageButton:{},header:{}},order:{},type:{detect:[],search:{},order:{}},_unique:0,fnVersionCheck:n.fnVersionCheck,iApiIndex:0,oJUIClasses:{},sVersion:n.version};h.extend(x,{afnFiltering:x.search,aTypes:x.type.detect,ofnSearch:x.type.search,oSort:x.type.order,afnSortData:x.order,aoFeatures:x.feature,oApi:x.internal,oStdClasses:x.classes,oPagination:x.pager}); +h.extend(n.ext.classes,{sTable:"dataTable",sNoFooter:"no-footer",sPageButton:"paginate_button",sPageButtonActive:"current",sPageButtonDisabled:"disabled",sStripeOdd:"odd",sStripeEven:"even",sRowEmpty:"dataTables_empty",sWrapper:"dataTables_wrapper",sFilter:"dataTables_filter",sInfo:"dataTables_info",sPaging:"dataTables_paginate paging_",sLength:"dataTables_length",sProcessing:"dataTables_processing",sSortAsc:"sorting_asc",sSortDesc:"sorting_desc",sSortable:"sorting",sSortableAsc:"sorting_asc_disabled", +sSortableDesc:"sorting_desc_disabled",sSortableNone:"sorting_disabled",sSortColumn:"sorting_",sFilterInput:"",sLengthSelect:"",sScrollWrapper:"dataTables_scroll",sScrollHead:"dataTables_scrollHead",sScrollHeadInner:"dataTables_scrollHeadInner",sScrollBody:"dataTables_scrollBody",sScrollFoot:"dataTables_scrollFoot",sScrollFootInner:"dataTables_scrollFootInner",sHeaderTH:"",sFooterTH:"",sSortJUIAsc:"",sSortJUIDesc:"",sSortJUI:"",sSortJUIAscAllowed:"",sSortJUIDescAllowed:"",sSortJUIWrapper:"",sSortIcon:"", +sJUIHeader:"",sJUIFooter:""});var Lb=n.ext.pager;h.extend(Lb,{simple:function(){return["previous","next"]},full:function(){return["first","previous","next","last"]},numbers:function(a,b){return[ia(a,b)]},simple_numbers:function(a,b){return["previous",ia(a,b),"next"]},full_numbers:function(a,b){return["first","previous",ia(a,b),"next","last"]},first_last_numbers:function(a,b){return["first",ia(a,b),"last"]},_numbers:ia,numbers_length:7});h.extend(!0,n.ext.renderer,{pageButton:{_:function(a,b,c,d,e, +f){var g=a.oClasses,j=a.oLanguage.oPaginate,i=a.oLanguage.oAria.paginate||{},m,l,n=0,o=function(b,d){var k,s,u,r,v=function(b){Ta(a,b.data.action,true)};k=0;for(s=d.length;k").appendTo(b);o(u,r)}else{m=null;l="";switch(r){case "ellipsis":b.append('…');break;case "first":m=j.sFirst;l=r+(e>0?"":" "+g.sPageButtonDisabled);break;case "previous":m=j.sPrevious;l=r+(e>0?"":" "+g.sPageButtonDisabled);break;case "next":m= +j.sNext;l=r+(e",{"class":g.sPageButton+" "+l,"aria-controls":a.sTableId,"aria-label":i[r],"data-dt-idx":n,tabindex:a.iTabIndex,id:c===0&&typeof r==="string"?a.sTableId+"_"+r:null}).html(m).appendTo(b);Wa(u,{action:r},v);n++}}}},s;try{s=h(b).find(H.activeElement).data("dt-idx")}catch(u){}o(h(b).empty(),d);s!==k&&h(b).find("[data-dt-idx="+ +s+"]").focus()}}});h.extend(n.ext.type.detect,[function(a,b){var c=b.oLanguage.sDecimal;return $a(a,c)?"num"+c:null},function(a){if(a&&!(a instanceof Date)&&!Zb.test(a))return null;var b=Date.parse(a);return null!==b&&!isNaN(b)||M(a)?"date":null},function(a,b){var c=b.oLanguage.sDecimal;return $a(a,c,!0)?"num-fmt"+c:null},function(a,b){var c=b.oLanguage.sDecimal;return Qb(a,c)?"html-num"+c:null},function(a,b){var c=b.oLanguage.sDecimal;return Qb(a,c,!0)?"html-num-fmt"+c:null},function(a){return M(a)|| +"string"===typeof a&&-1!==a.indexOf("<")?"html":null}]);h.extend(n.ext.type.search,{html:function(a){return M(a)?a:"string"===typeof a?a.replace(Nb," ").replace(Aa,""):""},string:function(a){return M(a)?a:"string"===typeof a?a.replace(Nb," "):a}});var za=function(a,b,c,d){if(0!==a&&(!a||"-"===a))return-Infinity;b&&(a=Pb(a,b));a.replace&&(c&&(a=a.replace(c,"")),d&&(a=a.replace(d,"")));return 1*a};h.extend(x.type.order,{"date-pre":function(a){a=Date.parse(a);return isNaN(a)?-Infinity:a},"html-pre":function(a){return M(a)? +"":a.replace?a.replace(/<.*?>/g,"").toLowerCase():a+""},"string-pre":function(a){return M(a)?"":"string"===typeof a?a.toLowerCase():!a.toString?"":a.toString()},"string-asc":function(a,b){return ab?1:0},"string-desc":function(a,b){return ab?-1:0}});Da("");h.extend(!0,n.ext.renderer,{header:{_:function(a,b,c,d){h(a.nTable).on("order.dt.DT",function(e,f,g,h){if(a===f){e=c.idx;b.removeClass(c.sSortingClass+" "+d.sSortAsc+" "+d.sSortDesc).addClass(h[e]=="asc"?d.sSortAsc:h[e]=="desc"?d.sSortDesc: +c.sSortingClass)}})},jqueryui:function(a,b,c,d){h("
    ").addClass(d.sSortJUIWrapper).append(b.contents()).append(h("").addClass(d.sSortIcon+" "+c.sSortingClassJUI)).appendTo(b);h(a.nTable).on("order.dt.DT",function(e,f,g,h){if(a===f){e=c.idx;b.removeClass(d.sSortAsc+" "+d.sSortDesc).addClass(h[e]=="asc"?d.sSortAsc:h[e]=="desc"?d.sSortDesc:c.sSortingClass);b.find("span."+d.sSortIcon).removeClass(d.sSortJUIAsc+" "+d.sSortJUIDesc+" "+d.sSortJUI+" "+d.sSortJUIAscAllowed+" "+d.sSortJUIDescAllowed).addClass(h[e]== +"asc"?d.sSortJUIAsc:h[e]=="desc"?d.sSortJUIDesc:c.sSortingClassJUI)}})}}});var eb=function(a){return"string"===typeof a?a.replace(//g,">").replace(/"/g,"""):a};n.render={number:function(a,b,c,d,e){return{display:function(f){if("number"!==typeof f&&"string"!==typeof f)return f;var g=0>f?"-":"",h=parseFloat(f);if(isNaN(h))return eb(f);h=h.toFixed(c);f=Math.abs(h);h=parseInt(f,10);f=c?b+(f-h).toFixed(c).substring(2):"";return g+(d||"")+h.toString().replace(/\B(?=(\d{3})+(?!\d))/g, +a)+f+(e||"")}}},text:function(){return{display:eb,filter:eb}}};h.extend(n.ext.internal,{_fnExternApiFunc:Mb,_fnBuildAjax:sa,_fnAjaxUpdate:mb,_fnAjaxParameters:vb,_fnAjaxUpdateDraw:wb,_fnAjaxDataSrc:ta,_fnAddColumn:Ea,_fnColumnOptions:ka,_fnAdjustColumnSizing:$,_fnVisibleToColumnIndex:aa,_fnColumnIndexToVisible:ba,_fnVisbleColumns:V,_fnGetColumns:ma,_fnColumnTypes:Ga,_fnApplyColumnDefs:jb,_fnHungarianMap:Z,_fnCamelToHungarian:J,_fnLanguageCompat:Ca,_fnBrowserDetect:hb,_fnAddData:O,_fnAddTr:na,_fnNodeToDataIndex:function(a, +b){return b._DT_RowIndex!==k?b._DT_RowIndex:null},_fnNodeToColumnIndex:function(a,b,c){return h.inArray(c,a.aoData[b].anCells)},_fnGetCellData:B,_fnSetCellData:kb,_fnSplitObjNotation:Ja,_fnGetObjectDataFn:S,_fnSetObjectDataFn:N,_fnGetDataMaster:Ka,_fnClearTable:oa,_fnDeleteIndex:pa,_fnInvalidate:da,_fnGetRowElements:Ia,_fnCreateTr:Ha,_fnBuildHead:lb,_fnDrawHead:fa,_fnDraw:P,_fnReDraw:T,_fnAddOptionsHtml:ob,_fnDetectHeader:ea,_fnGetUniqueThs:ra,_fnFeatureHtmlFilter:qb,_fnFilterComplete:ga,_fnFilterCustom:zb, +_fnFilterColumn:yb,_fnFilter:xb,_fnFilterCreateSearch:Pa,_fnEscapeRegex:Qa,_fnFilterData:Ab,_fnFeatureHtmlInfo:tb,_fnUpdateInfo:Db,_fnInfoMacros:Eb,_fnInitialise:ha,_fnInitComplete:ua,_fnLengthChange:Ra,_fnFeatureHtmlLength:pb,_fnFeatureHtmlPaginate:ub,_fnPageChange:Ta,_fnFeatureHtmlProcessing:rb,_fnProcessingDisplay:C,_fnFeatureHtmlTable:sb,_fnScrollDraw:la,_fnApplyToChildren:I,_fnCalculateColumnWidths:Fa,_fnThrottle:Oa,_fnConvertToWidth:Fb,_fnGetWidestNode:Gb,_fnGetMaxLenString:Hb,_fnStringToCss:v, +_fnSortFlatten:X,_fnSort:nb,_fnSortAria:Jb,_fnSortListener:Va,_fnSortAttachListener:Ma,_fnSortingClasses:wa,_fnSortData:Ib,_fnSaveState:xa,_fnLoadState:Kb,_fnSettingsFromNode:ya,_fnLog:K,_fnMap:F,_fnBindAction:Wa,_fnCallbackReg:z,_fnCallbackFire:r,_fnLengthOverflow:Sa,_fnRenderer:Na,_fnDataSource:y,_fnRowAttributes:La,_fnExtend:Xa,_fnCalculateEnd:function(){}});h.fn.dataTable=n;n.$=h;h.fn.dataTableSettings=n.settings;h.fn.dataTableExt=n.ext;h.fn.DataTable=function(a){return h(this).dataTable(a).api()}; +h.each(n,function(a,b){h.fn.DataTable[a]=b});return h.fn.dataTable}); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSErasureCoding.md b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSErasureCoding.md index 7d26883161873..d28aefde03040 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSErasureCoding.md +++ b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSErasureCoding.md @@ -234,7 +234,7 @@ Certain HDFS operations, i.e., `hflush`, `hsync`, `concat`, `setReplication`, `t are not supported on erasure coded files due to substantial technical challenges. -* `append()` and `truncate()` on an erasure coded file will throw `IOException`. +* `append()` and `truncate()` on an erasure coded file will throw `IOException`. However, with `NEW_BLOCK` flag enabled, append to a closed striped file is supported. * `concat()` will throw `IOException` if files are mixed with different erasure coding policies or with replicated files. * `setReplication()` is no-op on erasure coded files. diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSClientAdapter.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSClientAdapter.java index 6a211ef51c194..ccf45f6afc9a4 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSClientAdapter.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSClientAdapter.java @@ -54,8 +54,8 @@ public static DFSClient getClient(DistributedFileSystem dfs) return dfs.dfs; } - public static ExtendedBlock getPreviousBlock(DFSClient client, long fileId) { - return client.getPreviousBlock(fileId); + public static ExtendedBlock getPreviousBlock(DFSClient client, String renewLeaseKey) { + return client.getPreviousBlock(renewLeaseKey); } public static long getFileId(DFSOutputStream out) { diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java index 484958e3c302c..dd8bb2043828b 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java @@ -2159,10 +2159,11 @@ public void shutdown(boolean deleteDfsDir, boolean closeFileSystem) { LOG.info("Shutting down the Mini HDFS Cluster"); if (checkExitOnShutdown) { if (ExitUtil.terminateCalled()) { - LOG.error("Test resulted in an unexpected exit", - ExitUtil.getFirstExitException()); + Exception cause = ExitUtil.getFirstExitException(); + LOG.error("Test resulted in an unexpected exit", cause); ExitUtil.resetFirstExitException(); - throw new AssertionError("Test resulted in an unexpected exit"); + throw new AssertionError("Test resulted in an unexpected exit: " + + cause.toString(), cause); } } if (closeFileSystem) { @@ -2308,6 +2309,8 @@ public synchronized void restartNameNode(int nnIndex, boolean waitActive, nn.getHttpServer() .setAttribute(ImageServlet.RECENT_IMAGE_CHECK_ENABLED, false); info.nameNode = nn; + info.nameserviceId = info.conf.get(DFS_NAMESERVICE_ID); + info.nnId = info.conf.get(DFS_HA_NAMENODE_ID_KEY); info.setStartOpt(startOpt); if (waitActive) { if (numDataNodes > 0) { diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSClientRetries.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSClientRetries.java index 970003b0e58cc..c335d38f73311 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSClientRetries.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSClientRetries.java @@ -384,7 +384,7 @@ public void testLeaseRenewSocketTimeout() throws Exception cluster.waitActive(); NamenodeProtocols spyNN = spy(cluster.getNameNodeRpc()); Mockito.doThrow(new SocketTimeoutException()).when(spyNN).renewLease( - Mockito.anyString()); + Mockito.anyString(), any()); DFSClient client = new DFSClient(null, spyNN, conf, null); // Get hold of the lease renewer instance used by the client final LeaseRenewer leaseRenewer1 = client.getLeaseRenewer(); @@ -392,7 +392,7 @@ public void testLeaseRenewSocketTimeout() throws Exception OutputStream out1 = client.create(file1, false); Mockito.verify(spyNN, timeout(10000).times(1)).renewLease( - Mockito.anyString()); + Mockito.anyString(), any()); verifyEmptyLease(leaseRenewer1); GenericTestUtils.waitFor(() -> !(leaseRenewer1.isRunning()), 100, 10000); try { @@ -406,12 +406,12 @@ public void testLeaseRenewSocketTimeout() throws Exception // Verify DFSClient can do write operation after renewLease no longer // throws SocketTimeoutException. Mockito.doNothing().when(spyNN).renewLease( - Mockito.anyString()); + Mockito.anyString(), any()); final LeaseRenewer leaseRenewer2 = client.getLeaseRenewer(); leaseRenewer2.setRenewalTime(100); OutputStream out2 = client.create(file2, false); Mockito.verify(spyNN, timeout(10000).times(2)).renewLease( - Mockito.anyString()); + Mockito.anyString(), any()); out2.write(new byte[256]); out2.close(); verifyEmptyLease(leaseRenewer2); @@ -1309,7 +1309,7 @@ public void delayWhenRenewLeaseTimeout() { try { //1. trigger get LeaseRenewer lock Mockito.doThrow(new SocketTimeoutException()).when(spyNN) - .renewLease(Mockito.anyString()); + .renewLease(Mockito.anyString(), any()); } catch (IOException e) { e.printStackTrace(); } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSOutputStream.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSOutputStream.java index 4c1633a1e8605..8b90287e82fe1 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSOutputStream.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSOutputStream.java @@ -68,8 +68,9 @@ import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; import static org.mockito.ArgumentMatchers.anyBoolean; -import static org.mockito.ArgumentMatchers.anyLong; import org.mockito.Mockito; + +import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; @@ -433,7 +434,7 @@ public void testEndLeaseCall() throws Exception { EnumSet.of(CreateFlag.CREATE), (short) 3, 1024, null , 1024, null); DFSOutputStream spyDFSOutputStream = Mockito.spy(dfsOutputStream); spyDFSOutputStream.closeThreads(anyBoolean()); - verify(spyClient, times(1)).endFileLease(anyLong()); + verify(spyClient, times(1)).endFileLease(anyString()); } @Test diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileChecksum.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileChecksum.java index c19d8c3e4a5e2..29ab1baa705ef 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileChecksum.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileChecksum.java @@ -215,6 +215,39 @@ public void testStripedAndReplicatedFileChecksum() throws Exception { } } + /** + * Test the corner case of the COMPOSITE_CRC. + * For Stripe File, last block size in the file is (int)(blockSize * 0.5), + * but the last block size in the check length is (int)(blockSize * 0.6). + * For Replicate File, the last block size in the file is (int)(blockSize * 0.5), + * but the last block size in the check length is ((dataBlocks - 1) * blockSize + * + (int) (blockSize * 0.6)) + */ + @Test(timeout = 90000) + public void testStripedAndReplicatedFileChecksum2() throws Exception { + final int lastBlockSize = (int) (blockSize * 0.5); + final int fullStripeLength = dataBlocks * blockSize; + final int testFileSize = fullStripeLength + lastBlockSize; + prepareTestFiles(testFileSize, new String[] {stripedFile1, replicatedFile}); + + final int specialLength = (dataBlocks - 1) * blockSize + + (int) (blockSize * 0.6); + + Assert.assertTrue(specialLength % blockSize > lastBlockSize); + Assert.assertTrue(specialLength % fullStripeLength > lastBlockSize); + + FileChecksum stripedFileChecksum = getFileChecksum(stripedFile1, + specialLength, false); + FileChecksum replicatedFileChecksum = getFileChecksum(replicatedFile, + specialLength, false); + + if (checksumCombineMode.equals(ChecksumCombineMode.COMPOSITE_CRC.name())) { + Assert.assertEquals(replicatedFileChecksum, stripedFileChecksum); + } else { + Assert.assertNotEquals(replicatedFileChecksum, stripedFileChecksum); + } + } + @Test(timeout = 90000) public void testDifferentBlockSizeReplicatedFileChecksum() throws Exception { byte[] fileData = StripedFileTestUtil.generateBytes(fileSize); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestLease.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestLease.java index 5d7b62a42846a..8b527d07a29b8 100755 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestLease.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestLease.java @@ -90,7 +90,8 @@ public void testLeaseAbort() throws Exception { // stub the renew method. doThrow(new RemoteException(InvalidToken.class.getName(), - "Your token is worthless")).when(spyNN).renewLease(anyString()); + "Your token is worthless")).when(spyNN).renewLease( + anyString(), any()); // We don't need to wait the lease renewer thread to act. // call renewLease() manually. @@ -131,7 +132,7 @@ public void testLeaseAbort() throws Exception { Assert.assertTrue(originalRenewer.isEmpty()); // unstub - doNothing().when(spyNN).renewLease(anyString()); + doNothing().when(spyNN).renewLease(anyString(), any()); // existing input streams should work try { diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReconstructStripedFile.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReconstructStripedFile.java index fa3c1aa68130f..cda714b2cb3fb 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReconstructStripedFile.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReconstructStripedFile.java @@ -472,7 +472,7 @@ public void testProcessErasureCodingTasksSubmitionShouldSucceed() BlockECReconstructionInfo invalidECInfo = new BlockECReconstructionInfo( new ExtendedBlock("bp-id", 123456), dataDNs, dnStorageInfo, liveIndices, - ecPolicy); + new byte[0], ecPolicy); List ecTasks = new ArrayList<>(); ecTasks.add(invalidECInfo); dataNode.getErasureCodingWorker().processErasureCodingTasks(ecTasks); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestRollingUpgrade.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestRollingUpgrade.java index c428d20e840a8..6e7014c42eb13 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestRollingUpgrade.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestRollingUpgrade.java @@ -33,6 +33,8 @@ import javax.management.ReflectionException; import javax.management.openmbean.CompositeDataSupport; +import org.junit.Rule; +import org.junit.rules.TemporaryFolder; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.hadoop.conf.Configuration; @@ -80,16 +82,38 @@ public static void runCmd(DFSAdmin dfsadmin, boolean success, } } + @Rule + public TemporaryFolder folder = new TemporaryFolder(); + + /** + * Create a default HDFS configuration which has test-specific data directories. This is + * intended to protect against interactions between test runs that might corrupt results. Each + * test run's data is automatically cleaned-up by JUnit. + * + * @return a default configuration with test-specific data directories + */ + public Configuration getHdfsConfiguration() throws IOException { + Configuration conf = new HdfsConfiguration(); + + // Override the file system locations with test-specific temporary folders + conf.set(DFSConfigKeys.DFS_NAMENODE_NAME_DIR_KEY, + folder.newFolder("dfs/name").toString()); + conf.set(DFSConfigKeys.DFS_NAMENODE_CHECKPOINT_DIR_KEY, + folder.newFolder("dfs/namesecondary").toString()); + conf.set(DFSConfigKeys.DFS_DATANODE_DATA_DIR_KEY, + folder.newFolder("dfs/data").toString()); + + return conf; + } + /** * Test DFSAdmin Upgrade Command. */ @Test public void testDFSAdminRollingUpgradeCommands() throws Exception { // start a cluster - final Configuration conf = new HdfsConfiguration(); - MiniDFSCluster cluster = null; - try { - cluster = new MiniDFSCluster.Builder(conf).numDataNodes(0).build(); + final Configuration conf = getHdfsConfiguration(); + try (MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).numDataNodes(0).build()) { cluster.waitActive(); final Path foo = new Path("/foo"); @@ -149,8 +173,6 @@ public void testDFSAdminRollingUpgradeCommands() throws Exception { Assert.assertTrue(dfs.exists(bar)); Assert.assertTrue(dfs.exists(baz)); } - } finally { - if(cluster != null) cluster.shutdown(); } } @@ -172,115 +194,116 @@ public void testRollingUpgradeWithQJM() throws Exception { LOG.info("nn1Dir=" + nn1Dir); LOG.info("nn2Dir=" + nn2Dir); - final Configuration conf = new HdfsConfiguration(); - final MiniJournalCluster mjc = new MiniJournalCluster.Builder(conf).build(); - mjc.waitActive(); - setConf(conf, nn1Dir, mjc); - - { - // Start the cluster once to generate the dfs dirs - final MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf) - .numDataNodes(0) - .manageNameDfsDirs(false) - .checkExitOnShutdown(false) - .build(); - // Shutdown the cluster before making a copy of the namenode dir to release - // all file locks, otherwise, the copy will fail on some platforms. - cluster.shutdown(); - } - - MiniDFSCluster cluster2 = null; - try { - // Start a second NN pointed to the same quorum. - // We need to copy the image dir from the first NN -- or else - // the new NN will just be rejected because of Namespace mismatch. - FileUtil.fullyDelete(nn2Dir); - FileUtil.copy(nn1Dir, FileSystem.getLocal(conf).getRaw(), - new Path(nn2Dir.getAbsolutePath()), false, conf); - - // Start the cluster again - final MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf) - .numDataNodes(0) - .format(false) - .manageNameDfsDirs(false) - .checkExitOnShutdown(false) - .build(); + final Configuration conf = getHdfsConfiguration(); + try (MiniJournalCluster mjc = new MiniJournalCluster.Builder(conf).build()) { + mjc.waitActive(); + setConf(conf, nn1Dir, mjc); - final Path foo = new Path("/foo"); - final Path bar = new Path("/bar"); - final Path baz = new Path("/baz"); - - final RollingUpgradeInfo info1; { - final DistributedFileSystem dfs = cluster.getFileSystem(); - dfs.mkdirs(foo); - - //start rolling upgrade - dfs.setSafeMode(SafeModeAction.SAFEMODE_ENTER); - info1 = dfs.rollingUpgrade(RollingUpgradeAction.PREPARE); - dfs.setSafeMode(SafeModeAction.SAFEMODE_LEAVE); - LOG.info("START\n" + info1); - - //query rolling upgrade - assertEquals(info1, dfs.rollingUpgrade(RollingUpgradeAction.QUERY)); - - dfs.mkdirs(bar); + // Start the cluster once to generate the dfs dirs + final MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf) + .numDataNodes(0) + .manageNameDfsDirs(false) + .checkExitOnShutdown(false) + .build(); + // Shutdown the cluster before making a copy of the namenode dir to release + // all file locks, otherwise, the copy will fail on some platforms. cluster.shutdown(); } - // cluster2 takes over QJM - final Configuration conf2 = setConf(new Configuration(), nn2Dir, mjc); - cluster2 = new MiniDFSCluster.Builder(conf2) - .numDataNodes(0) - .format(false) - .manageNameDfsDirs(false) - .build(); - final DistributedFileSystem dfs2 = cluster2.getFileSystem(); - - // Check that cluster2 sees the edits made on cluster1 - Assert.assertTrue(dfs2.exists(foo)); - Assert.assertTrue(dfs2.exists(bar)); - Assert.assertFalse(dfs2.exists(baz)); - - //query rolling upgrade in cluster2 - assertEquals(info1, dfs2.rollingUpgrade(RollingUpgradeAction.QUERY)); - - dfs2.mkdirs(baz); - - LOG.info("RESTART cluster 2"); - cluster2.restartNameNode(); - assertEquals(info1, dfs2.rollingUpgrade(RollingUpgradeAction.QUERY)); - Assert.assertTrue(dfs2.exists(foo)); - Assert.assertTrue(dfs2.exists(bar)); - Assert.assertTrue(dfs2.exists(baz)); - - //restart cluster with -upgrade should fail. + MiniDFSCluster cluster2 = null; try { - cluster2.restartNameNode("-upgrade"); - } catch(IOException e) { - LOG.info("The exception is expected.", e); - } + // Start a second NN pointed to the same quorum. + // We need to copy the image dir from the first NN -- or else + // the new NN will just be rejected because of Namespace mismatch. + FileUtil.fullyDelete(nn2Dir); + FileUtil.copy(nn1Dir, FileSystem.getLocal(conf).getRaw(), + new Path(nn2Dir.getAbsolutePath()), false, conf); + + // Start the cluster again + final MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf) + .numDataNodes(0) + .format(false) + .manageNameDfsDirs(false) + .checkExitOnShutdown(false) + .build(); + + final Path foo = new Path("/foo"); + final Path bar = new Path("/bar"); + final Path baz = new Path("/baz"); + + final RollingUpgradeInfo info1; + { + final DistributedFileSystem dfs = cluster.getFileSystem(); + dfs.mkdirs(foo); + + //start rolling upgrade + dfs.setSafeMode(SafeModeAction.SAFEMODE_ENTER); + info1 = dfs.rollingUpgrade(RollingUpgradeAction.PREPARE); + dfs.setSafeMode(SafeModeAction.SAFEMODE_LEAVE); + LOG.info("START\n" + info1); + + //query rolling upgrade + assertEquals(info1, dfs.rollingUpgrade(RollingUpgradeAction.QUERY)); + + dfs.mkdirs(bar); + cluster.shutdown(); + } - LOG.info("RESTART cluster 2 again"); - cluster2.restartNameNode(); - assertEquals(info1, dfs2.rollingUpgrade(RollingUpgradeAction.QUERY)); - Assert.assertTrue(dfs2.exists(foo)); - Assert.assertTrue(dfs2.exists(bar)); - Assert.assertTrue(dfs2.exists(baz)); - - //finalize rolling upgrade - final RollingUpgradeInfo finalize = dfs2.rollingUpgrade( - RollingUpgradeAction.FINALIZE); - Assert.assertTrue(finalize.isFinalized()); - - LOG.info("RESTART cluster 2 with regular startup option"); - cluster2.getNameNodeInfos()[0].setStartOpt(StartupOption.REGULAR); - cluster2.restartNameNode(); - Assert.assertTrue(dfs2.exists(foo)); - Assert.assertTrue(dfs2.exists(bar)); - Assert.assertTrue(dfs2.exists(baz)); - } finally { - if (cluster2 != null) cluster2.shutdown(); + // cluster2 takes over QJM + final Configuration conf2 = setConf(new Configuration(), nn2Dir, mjc); + cluster2 = new MiniDFSCluster.Builder(conf2) + .numDataNodes(0) + .format(false) + .manageNameDfsDirs(false) + .build(); + final DistributedFileSystem dfs2 = cluster2.getFileSystem(); + + // Check that cluster2 sees the edits made on cluster1 + Assert.assertTrue(dfs2.exists(foo)); + Assert.assertTrue(dfs2.exists(bar)); + Assert.assertFalse(dfs2.exists(baz)); + + //query rolling upgrade in cluster2 + assertEquals(info1, dfs2.rollingUpgrade(RollingUpgradeAction.QUERY)); + + dfs2.mkdirs(baz); + + LOG.info("RESTART cluster 2"); + cluster2.restartNameNode(); + assertEquals(info1, dfs2.rollingUpgrade(RollingUpgradeAction.QUERY)); + Assert.assertTrue(dfs2.exists(foo)); + Assert.assertTrue(dfs2.exists(bar)); + Assert.assertTrue(dfs2.exists(baz)); + + //restart cluster with -upgrade should fail. + try { + cluster2.restartNameNode("-upgrade"); + } catch (IOException e) { + LOG.info("The exception is expected.", e); + } + + LOG.info("RESTART cluster 2 again"); + cluster2.restartNameNode(); + assertEquals(info1, dfs2.rollingUpgrade(RollingUpgradeAction.QUERY)); + Assert.assertTrue(dfs2.exists(foo)); + Assert.assertTrue(dfs2.exists(bar)); + Assert.assertTrue(dfs2.exists(baz)); + + //finalize rolling upgrade + final RollingUpgradeInfo finalize = dfs2.rollingUpgrade( + RollingUpgradeAction.FINALIZE); + Assert.assertTrue(finalize.isFinalized()); + + LOG.info("RESTART cluster 2 with regular startup option"); + cluster2.getNameNodeInfos()[0].setStartOpt(StartupOption.REGULAR); + cluster2.restartNameNode(); + Assert.assertTrue(dfs2.exists(foo)); + Assert.assertTrue(dfs2.exists(bar)); + Assert.assertTrue(dfs2.exists(baz)); + } finally { + if (cluster2 != null) cluster2.shutdown(); + } } } @@ -309,10 +332,8 @@ private static void checkMxBean() throws Exception { @Test public void testRollback() throws Exception { // start a cluster - final Configuration conf = new HdfsConfiguration(); - MiniDFSCluster cluster = null; - try { - cluster = new MiniDFSCluster.Builder(conf).numDataNodes(1).build(); + final Configuration conf = getHdfsConfiguration(); + try (MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).numDataNodes(1).build()) { cluster.waitActive(); final Path foo = new Path("/foo"); @@ -352,8 +373,6 @@ public void testRollback() throws Exception { startRollingUpgrade(foo, bar, file, data, cluster); rollbackRollingUpgrade(foo, bar, file, data, cluster); - } finally { - if(cluster != null) cluster.shutdown(); } } @@ -407,10 +426,8 @@ private static void rollbackRollingUpgrade(Path foo, Path bar, @Test public void testDFSAdminDatanodeUpgradeControlCommands() throws Exception { // start a cluster - final Configuration conf = new HdfsConfiguration(); - MiniDFSCluster cluster = null; - try { - cluster = new MiniDFSCluster.Builder(conf).numDataNodes(1).build(); + final Configuration conf = getHdfsConfiguration(); + try (MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).numDataNodes(1).build()) { cluster.waitActive(); final DFSAdmin dfsadmin = new DFSAdmin(conf); DataNode dn = cluster.getDataNodes().get(0); @@ -431,8 +448,6 @@ public void testDFSAdminDatanodeUpgradeControlCommands() throws Exception { // ping should fail. assertEquals(-1, dfsadmin.run(args1)); - } finally { - if (cluster != null) cluster.shutdown(); } } @@ -462,7 +477,7 @@ private void testFinalize(int nnCount) throws Exception { private void testFinalize(int nnCount, boolean skipImageDeltaCheck) throws Exception { - final Configuration conf = new HdfsConfiguration(); + final Configuration conf = getHdfsConfiguration(); MiniQJMHACluster cluster = null; final Path foo = new Path("/foo"); final Path bar = new Path("/bar"); @@ -528,10 +543,8 @@ public void testQueryWithMultipleNN() throws Exception { } private void testQuery(int nnCount) throws Exception{ - final Configuration conf = new Configuration(); - MiniQJMHACluster cluster = null; - try { - cluster = new MiniQJMHACluster.Builder(conf).setNumNameNodes(nnCount).build(); + final Configuration conf = getHdfsConfiguration(); + try (MiniQJMHACluster cluster = new MiniQJMHACluster.Builder(conf).setNumNameNodes(nnCount).build()) { MiniDFSCluster dfsCluster = cluster.getDfsCluster(); dfsCluster.waitActive(); @@ -561,19 +574,13 @@ private void testQuery(int nnCount) throws Exception{ // The NN should have a copy of the fsimage in case of rollbacks. Assert.assertTrue(dfsCluster.getNamesystem(0).getFSImage() .hasRollbackFSImage()); - } finally { - if (cluster != null) { - cluster.shutdown(); - } } } @Test (timeout = 300000) public void testQueryAfterRestart() throws IOException, InterruptedException { - final Configuration conf = new Configuration(); - MiniDFSCluster cluster = null; - try { - cluster = new MiniDFSCluster.Builder(conf).numDataNodes(0).build(); + final Configuration conf = getHdfsConfiguration(); + try (MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).numDataNodes(0).build()) { cluster.waitActive(); DistributedFileSystem dfs = cluster.getFileSystem(); @@ -587,10 +594,6 @@ public void testQueryAfterRestart() throws IOException, InterruptedException { cluster.restartNameNodes(); dfs.rollingUpgrade(RollingUpgradeAction.QUERY); - } finally { - if (cluster != null) { - cluster.shutdown(); - } } } @@ -606,7 +609,7 @@ public void testCheckpointWithMultipleNN() throws IOException, InterruptedExcept @Test(timeout = 60000) public void testRollBackImage() throws Exception { - final Configuration conf = new Configuration(); + final Configuration conf = getHdfsConfiguration(); conf.setInt(DFSConfigKeys.DFS_NAMENODE_CHECKPOINT_TXNS_KEY, 10); conf.setInt(DFSConfigKeys.DFS_HA_TAILEDITS_PERIOD_KEY, 1); conf.setInt(DFSConfigKeys.DFS_NAMENODE_CHECKPOINT_CHECK_PERIOD_KEY, 2); @@ -651,15 +654,14 @@ public void duringUploadInProgess() } public void testCheckpoint(int nnCount) throws IOException, InterruptedException { - final Configuration conf = new Configuration(); + final Configuration conf = getHdfsConfiguration(); conf.setInt(DFSConfigKeys.DFS_HA_TAILEDITS_PERIOD_KEY, 1); conf.setInt(DFSConfigKeys.DFS_NAMENODE_CHECKPOINT_PERIOD_KEY, 1); - MiniQJMHACluster cluster = null; final Path foo = new Path("/foo"); - try { - cluster = new MiniQJMHACluster.Builder(conf).setNumNameNodes(nnCount).build(); + try (MiniQJMHACluster cluster = new MiniQJMHACluster.Builder(conf).setNumNameNodes(nnCount) + .build()) { MiniDFSCluster dfsCluster = cluster.getDfsCluster(); dfsCluster.waitActive(); @@ -681,17 +683,14 @@ public void testCheckpoint(int nnCount) throws IOException, InterruptedException verifyNNCheckpoint(dfsCluster, txid, i); } - } finally { - if (cluster != null) { - cluster.shutdown(); - } } } /** * Verify that the namenode at the given index has an FSImage with a TxId up to txid-1 */ - private void verifyNNCheckpoint(MiniDFSCluster dfsCluster, long txid, int nnIndex) throws InterruptedException { + private void verifyNNCheckpoint(MiniDFSCluster dfsCluster, long txid, int nnIndex) + throws InterruptedException { int retries = 0; while (++retries < 5) { NNStorage storage = dfsCluster.getNamesystem(nnIndex).getFSImage() @@ -732,7 +731,7 @@ public void testCheckpointWithSNN() throws Exception { SecondaryNameNode snn = null; try { - Configuration conf = new HdfsConfiguration(); + Configuration conf = getHdfsConfiguration(); cluster = new MiniDFSCluster.Builder(conf).build(); cluster.waitActive(); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/protocolPB/TestPBHelper.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/protocolPB/TestPBHelper.java index a1b2e633edd3f..3b0b8d1564b84 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/protocolPB/TestPBHelper.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/protocolPB/TestPBHelper.java @@ -749,9 +749,10 @@ public void testBlockECRecoveryCommand() { DatanodeStorageInfo[] targetDnInfos0 = new DatanodeStorageInfo[] { targetDnInfos_0, targetDnInfos_1 }; byte[] liveBlkIndices0 = new byte[2]; + byte[] excludeReconstructedIndices0=new byte[2]; BlockECReconstructionInfo blkECRecoveryInfo0 = new BlockECReconstructionInfo( new ExtendedBlock("bp1", 1234), dnInfos0, targetDnInfos0, - liveBlkIndices0, StripedFileTestUtil.getDefaultECPolicy()); + liveBlkIndices0, excludeReconstructedIndices0, StripedFileTestUtil.getDefaultECPolicy()); DatanodeInfo[] dnInfos1 = new DatanodeInfo[] { DFSTestUtil.getLocalDatanodeInfo(), DFSTestUtil.getLocalDatanodeInfo() }; DatanodeStorageInfo targetDnInfos_2 = BlockManagerTestUtil @@ -763,9 +764,10 @@ public void testBlockECRecoveryCommand() { DatanodeStorageInfo[] targetDnInfos1 = new DatanodeStorageInfo[] { targetDnInfos_2, targetDnInfos_3 }; byte[] liveBlkIndices1 = new byte[2]; + byte[] excludeReconstructedIndices = new byte[2]; BlockECReconstructionInfo blkECRecoveryInfo1 = new BlockECReconstructionInfo( new ExtendedBlock("bp2", 3256), dnInfos1, targetDnInfos1, - liveBlkIndices1, StripedFileTestUtil.getDefaultECPolicy()); + liveBlkIndices1, excludeReconstructedIndices, StripedFileTestUtil.getDefaultECPolicy()); List blkRecoveryInfosList = new ArrayList(); blkRecoveryInfosList.add(blkECRecoveryInfo0); blkRecoveryInfosList.add(blkECRecoveryInfo1); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/MiniQJMHACluster.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/MiniQJMHACluster.java index 3ece3d7e47a68..0791e0ace1c0a 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/MiniQJMHACluster.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/MiniQJMHACluster.java @@ -35,7 +35,7 @@ import java.util.List; import java.util.Random; -public class MiniQJMHACluster { +public class MiniQJMHACluster implements AutoCloseable { private MiniDFSCluster cluster; private MiniJournalCluster journalCluster; private final Configuration conf; @@ -189,4 +189,15 @@ public void shutdown() throws IOException { cluster.shutdown(); journalCluster.shutdown(); } + + @Override + public void close() { + try { + shutdown(); + } catch (IOException shutdownFailure) { + LOG.warn("Exception while closing journal cluster", shutdownFailure); + } + + } + } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/client/TestQJMWithFaults.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/client/TestQJMWithFaults.java index c6e9fb7aa034c..e9f46e335c6e6 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/client/TestQJMWithFaults.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/client/TestQJMWithFaults.java @@ -198,7 +198,7 @@ public void testRecoverAfterDoubleFailures() throws Exception { public void testUnresolvableHostName() throws Exception { expectedException.expect(UnknownHostException.class); new QuorumJournalManager(conf, - new URI("qjournal://" + "bogus:12345" + "/" + JID), FAKE_NSINFO); + new URI("qjournal://" + "bogus.invalid:12345" + "/" + JID), FAKE_NSINFO); } /** diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/client/TestQuorumJournalManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/client/TestQuorumJournalManager.java index 5666ae58b0f41..e2ee2e365d9ff 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/client/TestQuorumJournalManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/client/TestQuorumJournalManager.java @@ -28,7 +28,7 @@ import static org.junit.Assert.assertNull; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; - +import static org.mockito.ArgumentMatchers.eq; import java.io.Closeable; import java.io.File; import java.io.IOException; @@ -40,11 +40,17 @@ import java.util.ArrayList; import java.util.List; import java.util.concurrent.ExecutorService; +import java.util.concurrent.Semaphore; import java.util.concurrent.TimeoutException; +import java.util.concurrent.atomic.AtomicInteger; import org.apache.hadoop.hdfs.server.common.Util; +import org.apache.hadoop.hdfs.server.protocol.RemoteEditLogManifest; import org.apache.hadoop.net.MockDomainNameResolver; +import org.apache.hadoop.thirdparty.com.google.common.util.concurrent.ListenableFuture; +import org.apache.hadoop.thirdparty.com.google.common.util.concurrent.SettableFuture; import org.apache.hadoop.util.Lists; +import org.mockito.stubbing.Answer; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.hadoop.conf.Configuration; @@ -53,6 +59,7 @@ import org.apache.hadoop.hdfs.qjournal.MiniJournalCluster; import org.apache.hadoop.hdfs.qjournal.QJMTestUtil; import org.apache.hadoop.hdfs.qjournal.protocol.QJournalProtocolProtos.SegmentStateProto; +import org.apache.hadoop.hdfs.qjournal.protocol.QJournalProtocolProtos.GetJournaledEditsResponseProto; import org.apache.hadoop.hdfs.qjournal.server.JournalFaultInjector; import org.apache.hadoop.hdfs.qjournal.server.JournalNode; import org.apache.hadoop.hdfs.server.namenode.EditLogInputStream; @@ -1101,6 +1108,56 @@ public void testSelectViaRpcTwoJNsError() throws Exception { } } + /** + * Test selecting EditLogInputStream after some journalNode jitter. + * Suppose there are 3 journalNodes, JN0 ~ JN2. + * 1. JN0 has some abnormal cases when Active Namenode is syncing 10 Edits with first txid 11. + * 2. NameNode just ignore the abnormal JN0 and continue to sync Edits to Journal 1 and 2. + * 3. JN0 backed to health. + * 4. NameNode continue sync 10 Edits with first txid 21. + * 5. At this point, there are no Edits 11 ~ 30 in the cache of JN0. + * 6. Observer NameNode try to select EditLogInputStream through + * getJournaledEdits with since txId 21. + * 7. JN2 has some abnormal cases and caused a slow response. + */ + @Test + public void testSelectViaRPCAfterJNJitter() throws Exception { + EditLogOutputStream stm = qjm.startLogSegment( + 1, NameNodeLayoutVersion.CURRENT_LAYOUT_VERSION); + SettableFuture slowLog = SettableFuture.create(); + Mockito.doReturn(slowLog).when(spies.get(0)) + .sendEdits(eq(1L), eq(11L), eq(10), Mockito.any()); + // Successfully write these edits to JN0 ~ JN2 + writeTxns(stm, 1, 10); + // Failed write these edits to JN0, but successfully write them to JN1 ~ JN2 + writeTxns(stm, 11, 10); + // Successfully write these edits to JN1 ~ JN2 + writeTxns(stm, 21, 20); + + Semaphore semaphore = new Semaphore(0); + spyGetJournaledEdits(0, 21, () -> semaphore.release(1)); + spyGetJournaledEdits(1, 21, () -> semaphore.release(1)); + spyGetJournaledEdits(2, 21, () -> semaphore.acquireUninterruptibly(2)); + + List streams = new ArrayList<>(); + qjm.selectInputStreams(streams, 21, true, true); + + assertEquals(1, streams.size()); + assertEquals(21, streams.get(0).getFirstTxId()); + assertEquals(40, streams.get(0).getLastTxId()); + } + + private void spyGetJournaledEdits(int jnSpyIdx, long fromTxId, Runnable preHook) { + Mockito.doAnswer((Answer>) invocation -> { + preHook.run(); + @SuppressWarnings("unchecked") + ListenableFuture result = + (ListenableFuture) invocation.callRealMethod(); + return result; + }).when(spies.get(jnSpyIdx)).getJournaledEdits(fromTxId, + QuorumJournalManager.QJM_RPC_MAX_TXNS_DEFAULT); + } + @Test public void testSelectViaRpcAfterJNRestart() throws Exception { EditLogOutputStream stm = @@ -1208,4 +1265,39 @@ private void checkRecovery(MiniJournalCluster cluster, segmentTxId); } } + + @Test + public void testSelectLatestEditsWithoutStreaming() throws Exception { + EditLogOutputStream stm = qjm.startLogSegment( + 1, NameNodeLayoutVersion.CURRENT_LAYOUT_VERSION); + // Successfully write these edits to JN0 ~ JN2 + writeTxns(stm, 1, 10); + + AtomicInteger atomicInteger = new AtomicInteger(0); + spyGetEditLogManifest(0, 11, true, atomicInteger::incrementAndGet); + spyGetEditLogManifest(1, 11, true, atomicInteger::incrementAndGet); + spyGetEditLogManifest(2, 11, true, atomicInteger::incrementAndGet); + + List streams = new ArrayList<>(); + qjm.selectInputStreams(streams, 1, true, true); + assertEquals(1, streams.size()); + assertEquals(1, streams.get(0).getFirstTxId()); + assertEquals(10, streams.get(0).getLastTxId()); + + streams.clear(); + qjm.selectInputStreams(streams, 11, true, true); + assertEquals(0, streams.size()); + assertEquals(0, atomicInteger.get()); + } + + private void spyGetEditLogManifest(int jnSpyIdx, long fromTxId, + boolean inProgressOk, Runnable preHook) { + Mockito.doAnswer((Answer>) invocation -> { + preHook.run(); + @SuppressWarnings("unchecked") + ListenableFuture result = + (ListenableFuture) invocation.callRealMethod(); + return result; + }).when(spies.get(jnSpyIdx)).getEditLogManifest(fromTxId, inProgressOk); + } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/server/TestGetJournalEditServlet.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/server/TestGetJournalEditServlet.java new file mode 100644 index 0000000000000..7d9dea0351651 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/server/TestGetJournalEditServlet.java @@ -0,0 +1,106 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package org.apache.hadoop.hdfs.qjournal.server; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hdfs.DFSConfigKeys; +import org.apache.hadoop.hdfs.HdfsConfiguration; +import org.apache.hadoop.hdfs.web.resources.UserParam; +import org.apache.hadoop.security.UserGroupInformation; +import org.junit.BeforeClass; +import org.junit.Test; + +import javax.servlet.ServletConfig; +import javax.servlet.ServletException; +import javax.servlet.http.HttpServletRequest; +import java.io.IOException; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +public class TestGetJournalEditServlet { + + private final static Configuration CONF = new HdfsConfiguration(); + + private final static GetJournalEditServlet SERVLET = new GetJournalEditServlet(); + + @BeforeClass + public static void setUp() throws ServletException { + // Configure Hadoop + CONF.set(DFSConfigKeys.FS_DEFAULT_NAME_KEY, "hdfs://localhost:4321/"); + CONF.set(DFSConfigKeys.HADOOP_SECURITY_AUTH_TO_LOCAL, + "RULE:[2:$1/$2@$0]([nsdj]n/.*@REALM\\.TLD)s/.*/hdfs/\nDEFAULT"); + CONF.set(DFSConfigKeys.DFS_NAMESERVICES, "ns"); + CONF.set(DFSConfigKeys.DFS_NAMENODE_KERBEROS_PRINCIPAL_KEY, "nn/_HOST@REALM.TLD"); + + // Configure Kerberos UGI + UserGroupInformation.setConfiguration(CONF); + UserGroupInformation.setLoginUser(UserGroupInformation.createRemoteUser( + "jn/somehost@REALM.TLD")); + + // Initialize the servlet + ServletConfig config = mock(ServletConfig.class); + SERVLET.init(config); + } + + /** + * Unauthenticated user should be rejected. + * + * @throws IOException for unexpected validation failures + */ + @Test + public void testWithoutUser() throws IOException { + // Test: Make a request without specifying a user + HttpServletRequest request = mock(HttpServletRequest.class); + boolean isValid = SERVLET.isValidRequestor(request, CONF); + + // Verify: The request is invalid + assertThat(isValid).isFalse(); + } + + /** + * Namenode requests should be authorized, since it will match the configured namenode. + * + * @throws IOException for unexpected validation failures + */ + @Test + public void testRequestNameNode() throws IOException, ServletException { + // Test: Make a request from a namenode + HttpServletRequest request = mock(HttpServletRequest.class); + when(request.getParameter(UserParam.NAME)).thenReturn("nn/localhost@REALM.TLD"); + boolean isValid = SERVLET.isValidRequestor(request, CONF); + + assertThat(isValid).isTrue(); + } + + /** + * There is a fallback using the short name, which is used by journalnodes. + * + * @throws IOException for unexpected validation failures + */ + @Test + public void testRequestShortName() throws IOException { + // Test: Make a request from a namenode + HttpServletRequest request = mock(HttpServletRequest.class); + when(request.getParameter(UserParam.NAME)).thenReturn("jn/localhost@REALM.TLD"); + boolean isValid = SERVLET.isValidRequestor(request, CONF); + + assertThat(isValid).isTrue(); + } + +} \ No newline at end of file diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/server/TestJournalNodeSync.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/server/TestJournalNodeSync.java index 1564e41031aba..28e36e03bfaa5 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/server/TestJournalNodeSync.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/server/TestJournalNodeSync.java @@ -17,6 +17,8 @@ */ package org.apache.hadoop.hdfs.qjournal.server; +import java.net.InetSocketAddress; +import java.net.URISyntaxException; import java.util.function.Supplier; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; @@ -34,6 +36,7 @@ import static org.apache.hadoop.hdfs.server.namenode.FileJournalManager .getLogFile; import static org.assertj.core.api.Assertions.assertThat; +import static org.junit.Assert.assertEquals; import org.apache.hadoop.hdfs.server.protocol.NamespaceInfo; import org.apache.hadoop.test.GenericTestUtils; @@ -96,12 +99,45 @@ public void shutDownMiniCluster() throws IOException { } } + /** + * Test that the "self exclusion" works when there are multiple JournalNode instances running on + * the same server, but on different ports. + */ + @Test + public void testJournalNodeExcludesSelfMultilpePorts() throws URISyntaxException, IOException { + String uri = qjmhaCluster.getJournalCluster().getQuorumJournalURI("ns1").toString(); + JournalNodeSyncer syncer = jCluster.getJournalNode(0).getJournalSyncer("ns1"); + + // Test: Get the Journal address list for the default configuration + List addrList = syncer.getJournalAddrList(uri); + + // Verify: One of the addresses should be excluded so that the node isn't syncing with itself + assertEquals(2, addrList.size()); + } + + /** + * Test that the "self exclusion" works when there a host uses a wildcard address. + */ + @Test + public void testJournalNodeExcludesSelfWildCard() throws URISyntaxException, IOException { + String uri = qjmhaCluster.getJournalCluster().getQuorumJournalURI("ns1").toString(); + JournalNodeSyncer syncer = jCluster.getJournalNode(0).getJournalSyncer("ns1"); + + // Test: Request the same Journal address list, but using the IPv4 "0.0.0.0" which is commonly + // used as a bind host. + String boundHostUri = uri.replaceAll("127.0.0.1", "0.0.0.0"); + List boundHostAddrList = syncer.getJournalAddrList(boundHostUri); + + // Verify: One of the address should be excluded so that the node isn't syncing with itself + assertEquals(2, boundHostAddrList.size()); + } + @Test(timeout=30000) public void testJournalNodeSync() throws Exception { //As by default 3 journal nodes are started; for(int i=0; i<3; i++) { - Assert.assertEquals(true, + assertEquals(true, jCluster.getJournalNode(i).getJournalSyncerStatus("ns1")); } @@ -386,13 +422,13 @@ public void testSyncDuringRollingUpgrade() throws Exception { HdfsConstants.RollingUpgradeAction.PREPARE); //query rolling upgrade - Assert.assertEquals(info, dfsActive.rollingUpgrade( + assertEquals(info, dfsActive.rollingUpgrade( HdfsConstants.RollingUpgradeAction.QUERY)); // Restart the Standby NN with rollingUpgrade option dfsCluster.restartNameNode(standbyNNindex, true, "-rollingUpgrade", "started"); - Assert.assertEquals(info, dfsActive.rollingUpgrade( + assertEquals(info, dfsActive.rollingUpgrade( HdfsConstants.RollingUpgradeAction.QUERY)); // Do some edits and delete some edit logs @@ -420,7 +456,7 @@ public void testSyncDuringRollingUpgrade() throws Exception { // Restart the current standby NN (previously active) dfsCluster.restartNameNode(standbyNNindex, true, "-rollingUpgrade", "started"); - Assert.assertEquals(info, dfsActive.rollingUpgrade( + assertEquals(info, dfsActive.rollingUpgrade( HdfsConstants.RollingUpgradeAction.QUERY)); dfsCluster.waitActive(); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockManager.java index f444b927b610e..8ebcbfe2e347f 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockManager.java @@ -700,6 +700,7 @@ public void testHighestPriReplSrcChosenDespiteMaxReplLimit() throws Exception { new NumberReplicas(), new ArrayList(), new ArrayList(), + new ArrayList(), LowRedundancyBlocks.QUEUE_HIGHEST_PRIORITY)[0]); assertEquals("Does not choose a source node for a less-than-highest-priority" @@ -712,6 +713,7 @@ public void testHighestPriReplSrcChosenDespiteMaxReplLimit() throws Exception { new NumberReplicas(), new ArrayList(), new ArrayList(), + new ArrayList(), LowRedundancyBlocks.QUEUE_VERY_LOW_REDUNDANCY).length); // Increase the replication count to test replication count > hard limit @@ -727,6 +729,7 @@ public void testHighestPriReplSrcChosenDespiteMaxReplLimit() throws Exception { new NumberReplicas(), new ArrayList(), new ArrayList(), + new ArrayList(), LowRedundancyBlocks.QUEUE_HIGHEST_PRIORITY).length); } @@ -773,6 +776,7 @@ public void testChooseSrcDatanodesWithDupEC() throws Exception { NumberReplicas numReplicas = new NumberReplicas(); List liveBlockIndices = new ArrayList<>(); List liveBusyBlockIndices = new ArrayList<>(); + List excludeReconstructedIndices = new ArrayList<>(); bm.chooseSourceDatanodes( aBlockInfoStriped, @@ -780,6 +784,7 @@ public void testChooseSrcDatanodesWithDupEC() throws Exception { liveNodes, numReplicas, liveBlockIndices, liveBusyBlockIndices, + excludeReconstructedIndices, LowRedundancyBlocks.QUEUE_VERY_LOW_REDUNDANCY); assertEquals("Choose the source node for reconstruction with one node reach" @@ -836,6 +841,7 @@ public void testChooseSrcDNWithDupECInDecommissioningNode() throws Exception { NumberReplicas numReplicas = new NumberReplicas(); List liveBlockIndices = new ArrayList<>(); List liveBusyBlockIndices = new ArrayList<>(); + List excludeReconstructedIndices = new ArrayList<>(); bm.chooseSourceDatanodes( aBlockInfoStriped, @@ -843,6 +849,7 @@ public void testChooseSrcDNWithDupECInDecommissioningNode() throws Exception { nodesContainingLiveReplicas, numReplicas, liveBlockIndices, liveBusyBlockIndices, + excludeReconstructedIndices, LowRedundancyBlocks.QUEUE_HIGHEST_PRIORITY); assertEquals("There are 5 live replicas in " + "[ds2, ds3, ds4, ds5, ds6] datanodes ", @@ -975,6 +982,7 @@ public void testFavorDecomUntilHardLimit() throws Exception { new NumberReplicas(), new LinkedList(), new ArrayList(), + new ArrayList(), LowRedundancyBlocks.QUEUE_LOW_REDUNDANCY)[0]); @@ -991,6 +999,7 @@ public void testFavorDecomUntilHardLimit() throws Exception { new NumberReplicas(), new LinkedList(), new ArrayList(), + new ArrayList(), LowRedundancyBlocks.QUEUE_LOW_REDUNDANCY).length); } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestFsDatasetImpl.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestFsDatasetImpl.java index 1e4de75148679..0805257a287eb 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestFsDatasetImpl.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestFsDatasetImpl.java @@ -70,6 +70,7 @@ import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsDatasetSpi; import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsDatasetSpi.FsVolumeReferences; import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsVolumeSpi; +import org.apache.hadoop.hdfs.server.datanode.metrics.DataNodeMetrics; import org.apache.hadoop.hdfs.server.protocol.NamespaceInfo; import org.apache.hadoop.io.MultipleIOException; import org.apache.hadoop.test.GenericTestUtils; @@ -226,6 +227,8 @@ public void setUp() throws IOException { final ShortCircuitRegistry shortCircuitRegistry = new ShortCircuitRegistry(conf); when(datanode.getShortCircuitRegistry()).thenReturn(shortCircuitRegistry); + DataNodeMetrics dataNodeMetrics = DataNodeMetrics.create(conf, "mockName"); + when(datanode.getMetrics()).thenReturn(dataNodeMetrics); createStorageDirs(storage, conf, NUM_INIT_VOLUMES); dataset = new FsDatasetImpl(datanode, storage, conf); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/web/TestDatanodeHttpXFrame.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/web/TestDatanodeHttpXFrame.java index 62827a2e5d3bc..b6ad5a9a5d6b7 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/web/TestDatanodeHttpXFrame.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/web/TestDatanodeHttpXFrame.java @@ -57,8 +57,7 @@ public void testDataNodeXFrameOptionsEnabled() throws Exception { cluster = createCluster(xFrameEnabled, null); HttpURLConnection conn = getConn(cluster); String xfoHeader = conn.getHeaderField("X-FRAME-OPTIONS"); - Assert.assertTrue("X-FRAME-OPTIONS is absent in the header", - xfoHeader != null); + Assert.assertNotNull("X-FRAME-OPTIONS is absent in the header", xfoHeader); Assert.assertTrue(xfoHeader.endsWith(HttpServer2.XFrameOption .SAMEORIGIN.toString())); } @@ -69,7 +68,7 @@ public void testNameNodeXFrameOptionsDisabled() throws Exception { cluster = createCluster(xFrameEnabled, null); HttpURLConnection conn = getConn(cluster); String xfoHeader = conn.getHeaderField("X-FRAME-OPTIONS"); - Assert.assertTrue("unexpected X-FRAME-OPTION in header", xfoHeader == null); + Assert.assertNull("unexpected X-FRAME-OPTION in header", xfoHeader); } @Test diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestEditLogRace.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestEditLogRace.java index 083d9e5070dd7..9dc2cc9182bba 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestEditLogRace.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestEditLogRace.java @@ -581,8 +581,8 @@ public void run() { } static SetOwnerOp getSetOwnerOp(OpInstanceCache cache, String group) { - return ((SetOwnerOp)cache.get(OP_SET_OWNER)) - .setSource("/").setUser("u").setGroup(group); + SetOwnerOp setOwnerOp = cache.get(OP_SET_OWNER); + return setOwnerOp.setSource("/").setUser("u").setGroup(group); } static class BlockingOpMatcher implements ArgumentMatcher { diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeRpcServer.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeRpcServer.java index 74d85bc637efa..2960a7ee6d4bb 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeRpcServer.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeRpcServer.java @@ -28,22 +28,29 @@ import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_RPC_BIND_HOST_KEY; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotEquals; +import static org.junit.Assert.assertNotNull; import java.io.IOException; import java.nio.charset.StandardCharsets; +import java.security.PrivilegedExceptionAction; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FSDataOutputStream; +import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.permission.FsPermission; import org.apache.hadoop.hdfs.DFSTestUtil; import org.apache.hadoop.hdfs.DistributedFileSystem; import org.apache.hadoop.hdfs.HdfsConfiguration; import org.apache.hadoop.hdfs.MiniDFSCluster; import org.apache.hadoop.hdfs.protocol.LocatedBlocks; +import org.apache.hadoop.hdfs.qjournal.MiniQJMHACluster; import org.apache.hadoop.ipc.CallerContext; import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.test.GenericTestUtils; import org.junit.Test; +import org.junit.jupiter.api.Timeout; public class TestNameNodeRpcServer { @@ -91,6 +98,66 @@ private static String getPreferredLocation(DistributedFileSystem fs, // trials. 1/3^20=3e-10, so that should be good enough. static final int ITERATIONS_TO_USE = 20; + @Test + @Timeout(30000) + public void testNamenodeRpcClientIpProxyWithFailBack() throws Exception { + // Make 3 nodes & racks so that we have a decent shot of detecting when + // our change overrides the random choice of datanode. + Configuration conf = new HdfsConfiguration(); + conf.set(DFS_NAMENODE_IP_PROXY_USERS, "fake_joe"); + final CallerContext original = CallerContext.getCurrent(); + + MiniQJMHACluster qjmhaCluster = null; + try { + String baseDir = GenericTestUtils.getRandomizedTempPath(); + MiniQJMHACluster.Builder builder = new MiniQJMHACluster.Builder(conf); + builder.getDfsBuilder().numDataNodes(3); + qjmhaCluster = builder.baseDir(baseDir).build(); + MiniDFSCluster dfsCluster = qjmhaCluster.getDfsCluster(); + dfsCluster.waitActive(); + dfsCluster.transitionToActive(0); + + // Set the caller context to set the ip address + CallerContext.setCurrent( + new CallerContext.Builder("test", conf) + .build()); + + dfsCluster.getFileSystem(0).setPermission( + new Path("/"), FsPermission.getDirDefault()); + + // Run as fake joe to authorize the test + UserGroupInformation joe = + UserGroupInformation.createUserForTesting("fake_joe", + new String[]{"fake_group"}); + + FileSystem joeFs = joe.doAs((PrivilegedExceptionAction) () -> + FileSystem.get(dfsCluster.getURI(0), conf)); + + Path testPath = new Path("/foo"); + // Write a sample file + FSDataOutputStream stream = joeFs.create(testPath); + stream.write("Hello world!\n".getBytes(StandardCharsets.UTF_8)); + stream.close(); + + qjmhaCluster.getDfsCluster().transitionToStandby(0); + qjmhaCluster.getDfsCluster().transitionToActive(1); + + DistributedFileSystem nn1 = dfsCluster.getFileSystem(1); + assertNotNull(nn1.getFileStatus(testPath)); + } finally { + CallerContext.setCurrent(original); + if (qjmhaCluster != null) { + try { + qjmhaCluster.shutdown(); + } catch (IOException e) { + e.printStackTrace(); + } + } + // Reset the config + conf.unset(DFS_NAMENODE_IP_PROXY_USERS); + } + } + /** * A test to make sure that if an authorized user adds "clientIp:" to their * caller context, it will be used to make locality decisions on the NN. diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestReconstructStripedBlocks.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestReconstructStripedBlocks.java index fc307bf84d86f..0a7874fc7a808 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestReconstructStripedBlocks.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestReconstructStripedBlocks.java @@ -25,6 +25,7 @@ import org.apache.hadoop.hdfs.HdfsConfiguration; import org.apache.hadoop.hdfs.MiniDFSCluster; import org.apache.hadoop.hdfs.StripedFileTestUtil; +import org.apache.hadoop.hdfs.client.HdfsDataInputStream; import org.apache.hadoop.hdfs.protocol.Block; import org.apache.hadoop.hdfs.protocol.DatanodeInfo; import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy; @@ -45,6 +46,7 @@ import org.apache.hadoop.hdfs.server.protocol.BlockECReconstructionCommand.BlockECReconstructionInfo; import org.apache.hadoop.hdfs.util.StripedBlockUtil; +import org.apache.hadoop.test.GenericTestUtils; import org.junit.Assert; import org.junit.Test; import org.slf4j.Logger; @@ -84,6 +86,7 @@ private void initConf(Configuration conf) { // chooseUnderReplicatedBlocks at once. conf.setInt( DFSConfigKeys.DFS_NAMENODE_REPLICATION_WORK_MULTIPLIER_PER_ITERATION, 5); + } @Test @@ -430,4 +433,86 @@ public void testReconstructionWork() throws Exception { dfsCluster.shutdown(); } } + private byte[] writeStripedFile(DistributedFileSystem fs, Path ecFile, + int writeBytes) throws Exception { + byte[] bytes = StripedFileTestUtil.generateBytes(writeBytes); + DFSTestUtil.writeFile(fs, ecFile, new String(bytes)); + StripedFileTestUtil.waitBlockGroupsReported(fs, ecFile.toString()); + + return bytes; + } + @Test + public void testReconstrutionWithBusyBlock1() throws Exception { + //When the index of busy block is smaller than the missing block + //[0(busy),1(busy),3,4,5,6,7,8] + int busyNodeIndex1 = 0; + int busyNodeIndex2 = 1; + int deadNodeIndex = 2; + final Path ecDir = new Path(GenericTestUtils.getRandomizedTempPath()); + final Path ecFile = new Path(ecDir, "testReconstrutionWithBusyBlock1"); + int writeBytes = cellSize * dataBlocks; + HdfsConfiguration conf = new HdfsConfiguration(); + initConf(conf); + conf.setBoolean(DFSConfigKeys.DFS_DISK_BALANCER_ENABLED, false); + conf.setInt(DFSConfigKeys.DFS_NAMENODE_HEARTBEAT_RECHECK_INTERVAL_KEY, + 2000); + conf.setInt(DFSConfigKeys.DFS_HEARTBEAT_INTERVAL_KEY, 1); + conf.setInt(DFSConfigKeys.DFS_NAMENODE_REDUNDANCY_INTERVAL_SECONDS_KEY, 1); + conf.setInt(DFSConfigKeys.DFS_BLOCKREPORT_INTERVAL_MSEC_KEY, + 1000); + conf.setInt(DFSConfigKeys.DFS_NAMENODE_RECONSTRUCTION_PENDING_TIMEOUT_SEC_KEY, + 4); + conf.setInt(DFSConfigKeys.DFS_NAMENODE_REDUNDANCY_INTERVAL_SECONDS_KEY, + 1); + cluster = new MiniDFSCluster.Builder(conf).numDataNodes(groupSize + 5) + .build(); + cluster.waitActive(); + DistributedFileSystem dfs = cluster.getFileSystem(0); + dfs.enableErasureCodingPolicy( + StripedFileTestUtil.getDefaultECPolicy().getName()); + dfs.mkdirs(ecDir); + dfs.setErasureCodingPolicy(ecDir, + StripedFileTestUtil.getDefaultECPolicy().getName()); + byte[] originBytesArray = writeStripedFile(dfs, ecFile, writeBytes); + List lbs = ((HdfsDataInputStream) dfs.open(ecFile)) + .getAllBlocks(); + LocatedStripedBlock lsb = (LocatedStripedBlock) lbs.get(0); + DatanodeInfo[] dnList = lsb.getLocations(); + BlockManager bm = cluster.getNamesystem().getBlockManager(); + BlockInfoStriped blockInfo = + (BlockInfoStriped) bm.getStoredBlock( + new Block(lsb.getBlock().getBlockId())); + + //1.Make nodes busy + DatanodeDescriptor busyNode = bm.getDatanodeManager() + .getDatanode(dnList[busyNodeIndex1].getDatanodeUuid()); + for (int j = 0; j < maxReplicationStreams; j++) { + busyNode.incrementPendingReplicationWithoutTargets(); + } + DatanodeDescriptor busyNode2 = bm.getDatanodeManager() + .getDatanode(dnList[busyNodeIndex2].getDatanodeUuid()); + for (int j = 0; j < maxReplicationStreams; j++) { + busyNode2.incrementPendingReplicationWithoutTargets(); + } + + //2.Make a node missing + DataNode dn = cluster.getDataNode(dnList[deadNodeIndex].getIpcPort()); + cluster.stopDataNode(dnList[deadNodeIndex].getXferAddr()); + cluster.setDataNodeDead(dn.getDatanodeId()); + + //3.Whether there is excess replicas or not during the recovery? + assertEquals(8, bm.countNodes(blockInfo).liveReplicas()); + + GenericTestUtils.waitFor( + () -> { + return bm.countNodes(blockInfo).liveReplicas() == 9|| + bm.countNodes(blockInfo).excessReplicas() >= 1|| + bm.countNodes(blockInfo).redundantInternalBlocks() >= 1; + }, + 10, 100000); + + assertEquals(0, bm.countNodes(blockInfo).excessReplicas()); + assertEquals(9, bm.countNodes(blockInfo).liveReplicas()); + } + } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestRefreshNamenodeReplicationConfig.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestRefreshNamenodeReplicationConfig.java index 8dc81f8c1a21d..8336a432b9a41 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestRefreshNamenodeReplicationConfig.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestRefreshNamenodeReplicationConfig.java @@ -49,6 +49,9 @@ public void setup() throws IOException { config.setInt( DFSConfigKeys.DFS_NAMENODE_REPLICATION_WORK_MULTIPLIER_PER_ITERATION, 12); + config.setInt( + DFSConfigKeys.DFS_NAMENODE_RECONSTRUCTION_PENDING_TIMEOUT_SEC_KEY, + 300); cluster = new MiniDFSCluster.Builder(config) .nnTopology(MiniDFSNNTopology.simpleSingleNN(0, 0)) @@ -72,6 +75,7 @@ public void testParamsCanBeReconfigured() throws ReconfigurationException { assertEquals(8, bm.getMaxReplicationStreams()); assertEquals(10, bm.getReplicationStreamsHardLimit()); assertEquals(12, bm.getBlocksReplWorkMultiplier()); + assertEquals(300, bm.getReconstructionPendingTimeout()); cluster.getNameNode().reconfigurePropertyImpl( DFSConfigKeys.DFS_NAMENODE_REPLICATION_MAX_STREAMS_KEY, "20"); @@ -81,10 +85,14 @@ public void testParamsCanBeReconfigured() throws ReconfigurationException { cluster.getNameNode().reconfigurePropertyImpl( DFSConfigKeys.DFS_NAMENODE_REPLICATION_WORK_MULTIPLIER_PER_ITERATION, "24"); + cluster.getNameNode().reconfigurePropertyImpl( + DFSConfigKeys.DFS_NAMENODE_RECONSTRUCTION_PENDING_TIMEOUT_SEC_KEY, + "180"); assertEquals(20, bm.getMaxReplicationStreams()); assertEquals(22, bm.getReplicationStreamsHardLimit()); assertEquals(24, bm.getBlocksReplWorkMultiplier()); + assertEquals(180, bm.getReconstructionPendingTimeout()); } /** @@ -96,7 +104,8 @@ public void testReconfigureFailsWithInvalidValues() throws Exception { String[] keys = new String[]{ DFSConfigKeys.DFS_NAMENODE_REPLICATION_MAX_STREAMS_KEY, DFSConfigKeys.DFS_NAMENODE_REPLICATION_STREAMS_HARD_LIMIT_KEY, - DFSConfigKeys.DFS_NAMENODE_REPLICATION_WORK_MULTIPLIER_PER_ITERATION + DFSConfigKeys.DFS_NAMENODE_REPLICATION_WORK_MULTIPLIER_PER_ITERATION, + DFSConfigKeys.DFS_NAMENODE_RECONSTRUCTION_PENDING_TIMEOUT_SEC_KEY }; // Ensure we cannot set any of the parameters negative @@ -112,6 +121,7 @@ public void testReconfigureFailsWithInvalidValues() throws Exception { assertEquals(8, bm.getMaxReplicationStreams()); assertEquals(10, bm.getReplicationStreamsHardLimit()); assertEquals(12, bm.getBlocksReplWorkMultiplier()); + assertEquals(300, bm.getReconstructionPendingTimeout()); for (String key : keys) { ReconfigurationException e = @@ -126,6 +136,7 @@ public void testReconfigureFailsWithInvalidValues() throws Exception { assertEquals(8, bm.getMaxReplicationStreams()); assertEquals(10, bm.getReplicationStreamsHardLimit()); assertEquals(12, bm.getBlocksReplWorkMultiplier()); + assertEquals(300, bm.getReconstructionPendingTimeout()); // Ensure none of the parameters can be set to a string value for (String key : keys) { @@ -139,5 +150,6 @@ public void testReconfigureFailsWithInvalidValues() throws Exception { assertEquals(8, bm.getMaxReplicationStreams()); assertEquals(10, bm.getReplicationStreamsHardLimit()); assertEquals(12, bm.getBlocksReplWorkMultiplier()); + assertEquals(300, bm.getReconstructionPendingTimeout()); } -} \ No newline at end of file +} diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestHASafeMode.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestHASafeMode.java index 37e279dfa5ee7..16a57c6867242 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestHASafeMode.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestHASafeMode.java @@ -852,13 +852,13 @@ public void testOpenFileWhenNNAndClientCrashAfterAddBlock() throws Exception { null); create.write(testData.getBytes()); create.hflush(); - long fileId = ((DFSOutputStream)create. - getWrappedStream()).getFileId(); + String renewLeaseKey = ((DFSOutputStream)create. + getWrappedStream()).getUniqKey(); FileStatus fileStatus = dfs.getFileStatus(filePath); DFSClient client = DFSClientAdapter.getClient(dfs); // add one dummy block at NN, but not write to DataNode ExtendedBlock previousBlock = - DFSClientAdapter.getPreviousBlock(client, fileId); + DFSClientAdapter.getPreviousBlock(client, renewLeaseKey); DFSClientAdapter.getNamenode(client).addBlock( pathString, client.getClientName(), diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestObserverNode.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestObserverNode.java index 29cae6f13adbc..60728284e59b4 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestObserverNode.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestObserverNode.java @@ -21,6 +21,7 @@ import static org.apache.hadoop.hdfs.server.namenode.NameNodeAdapter.getServiceState; import static org.apache.hadoop.hdfs.server.namenode.ha.ObserverReadProxyProvider.*; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; import static org.mockito.ArgumentMatchers.any; @@ -36,7 +37,10 @@ import java.util.List; import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; import java.util.concurrent.Future; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.ScheduledFuture; import java.util.concurrent.TimeUnit; import org.apache.hadoop.conf.Configuration; @@ -61,9 +65,12 @@ import org.apache.hadoop.hdfs.server.namenode.FSEditLog; import org.apache.hadoop.hdfs.server.namenode.FSNamesystem; import org.apache.hadoop.hdfs.server.namenode.NameNodeAdapter; +import org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer; import org.apache.hadoop.hdfs.server.namenode.TestFsck; import org.apache.hadoop.hdfs.tools.GetGroups; import org.apache.hadoop.ipc.ObserverRetryOnActiveException; +import org.apache.hadoop.ipc.metrics.RpcMetrics; +import org.apache.hadoop.test.GenericTestUtils; import org.apache.hadoop.util.Time; import org.apache.hadoop.util.concurrent.HadoopExecutors; import org.junit.After; @@ -124,6 +131,43 @@ public static void shutDownCluster() throws IOException { } } + @Test + public void testObserverRequeue() throws Exception { + ScheduledExecutorService interruptor = + Executors.newScheduledThreadPool(1); + + FSNamesystem observerFsNS = dfsCluster.getNamesystem(2); + RpcMetrics obRpcMetrics = ((NameNodeRpcServer)dfsCluster + .getNameNodeRpc(2)).getClientRpcServer().getRpcMetrics(); + try { + // Stop EditlogTailer of Observer NameNode. + observerFsNS.getEditLogTailer().stop(); + long oldRequeueNum = obRpcMetrics.getRpcRequeueCalls(); + ScheduledFuture scheduledFuture = interruptor.schedule( + () -> { + Path tmpTestPath = new Path("/TestObserverRequeue"); + dfs.create(tmpTestPath, (short)1).close(); + assertSentTo(0); + // This operation will be blocked in ObserverNameNode + // until EditlogTailer tailed edits from journalNode. + FileStatus fileStatus = dfs.getFileStatus(tmpTestPath); + assertSentTo(2); + return fileStatus; + }, 0, TimeUnit.SECONDS); + + GenericTestUtils.waitFor(() -> obRpcMetrics.getRpcRequeueCalls() > oldRequeueNum, + 50, 10000); + + observerFsNS.getEditLogTailer().doTailEdits(); + FileStatus fileStatus = scheduledFuture.get(10000, TimeUnit.MILLISECONDS); + assertNotNull(fileStatus); + } finally { + EditLogTailer editLogTailer = new EditLogTailer(observerFsNS, conf); + observerFsNS.setEditLogTailerForTests(editLogTailer); + editLogTailer.start(); + } + } + @Test public void testNoActiveToObserver() throws Exception { try { @@ -369,6 +413,12 @@ public void testObserverNodeBlockMissingRetry() throws Exception { dfs.open(testPath); assertSentTo(0); + dfs.getClient().listPaths("/", new byte[0], true); + assertSentTo(0); + + dfs.getClient().getLocatedFileInfo(testPath.toString(), false); + assertSentTo(0); + Mockito.reset(bmSpy); } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/metrics/TestNameNodeMetrics.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/metrics/TestNameNodeMetrics.java index aaedb8288e410..4d472d74b46a8 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/metrics/TestNameNodeMetrics.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/metrics/TestNameNodeMetrics.java @@ -48,6 +48,9 @@ import java.util.EnumSet; import java.util.List; import java.util.Random; + +import org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer; +import org.apache.hadoop.ipc.metrics.RpcDetailedMetrics; import org.apache.hadoop.thirdparty.com.google.common.collect.ImmutableList; import org.slf4j.Logger; @@ -1128,4 +1131,14 @@ public void testEditLogTailing() throws Exception { } } + + @Test + public void testNNRPCMetricIntegrity() { + RpcDetailedMetrics metrics = + ((NameNodeRpcServer) cluster.getNameNode() + .getRpcServer()).getClientRpcServer().getRpcDetailedMetrics(); + MetricsRecordBuilder rb = getMetrics(metrics.name()); + // CommitBlockSynchronizationNumOps should exist. + assertCounter("CommitBlockSynchronizationNumOps", 0L, rb); + } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestDFSAdmin.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestDFSAdmin.java index 3df873a51ceae..99e4b348f6157 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestDFSAdmin.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestDFSAdmin.java @@ -438,7 +438,7 @@ public void testNameNodeGetReconfigurableProperties() throws IOException, Interr final List outs = Lists.newArrayList(); final List errs = Lists.newArrayList(); getReconfigurableProperties("namenode", address, outs, errs); - assertEquals(19, outs.size()); + assertEquals(20, outs.size()); assertTrue(outs.get(0).contains("Reconfigurable properties:")); assertEquals(DFS_BLOCK_INVALIDATE_LIMIT_KEY, outs.get(1)); assertEquals(DFS_BLOCK_PLACEMENT_EC_CLASSNAME_KEY, outs.get(2)); @@ -1266,4 +1266,4 @@ public void testAllDatanodesReconfig() outs.get(8)); } -} \ No newline at end of file +} diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/TestOfflineImageViewerForErasureCodingPolicy.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/TestOfflineImageViewerForErasureCodingPolicy.java new file mode 100644 index 0000000000000..c4e5622f41326 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/TestOfflineImageViewerForErasureCodingPolicy.java @@ -0,0 +1,186 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hdfs.tools.offlineImageViewer; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FSDataOutputStream; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hdfs.DFSConfigKeys; +import org.apache.hadoop.hdfs.DFSTestUtil; +import org.apache.hadoop.hdfs.DistributedFileSystem; +import org.apache.hadoop.hdfs.MiniDFSCluster; +import org.apache.hadoop.hdfs.protocol.HdfsConstants; +import org.apache.hadoop.hdfs.server.namenode.FSImageTestUtil; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.BufferedReader; +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; + +import static org.junit.Assert.assertEquals; + +/** + * Tests OfflineImageViewer if the input fsimage has HDFS ErasureCodingPolicy + * entries. + */ +public class TestOfflineImageViewerForErasureCodingPolicy { + + private static final Logger LOG = + LoggerFactory.getLogger(TestOfflineImageViewerForErasureCodingPolicy.class); + + private static File originalFsimage = null; + private static File tempDir; + + /** + * Create a populated namespace for later testing. Save its contents to a + * data structure and store its fsimage location. + */ + @BeforeClass + public static void createOriginalFSImage() throws IOException { + MiniDFSCluster cluster = null; + try { + Configuration conf = new Configuration(); + conf.setBoolean(DFSConfigKeys.DFS_NAMENODE_ACLS_ENABLED_KEY, true); + conf.setBoolean(DFSConfigKeys.DFS_STORAGE_POLICY_ENABLED_KEY, true); + + File[] nnDirs = MiniDFSCluster.getNameNodeDirectory( + MiniDFSCluster.getBaseDirectory(), 0, 0); + tempDir = nnDirs[0]; + + cluster = new MiniDFSCluster.Builder(conf).numDataNodes(10).build(); + cluster.waitActive(); + DistributedFileSystem hdfs = cluster.getFileSystem(); + + hdfs.enableErasureCodingPolicy("RS-6-3-1024k"); + hdfs.enableErasureCodingPolicy("RS-3-2-1024k"); + + Path dir = new Path("/dir_wo_ec_rs63"); + hdfs.mkdirs(dir); + hdfs.setErasureCodingPolicy(dir, "RS-6-3-1024k"); + + dir = new Path("/dir_wo_ec_rs63/sub_dir_1"); + hdfs.mkdirs(dir); + + dir = new Path("/dir_wo_ec_rs63/sub_dir_2"); + hdfs.mkdirs(dir); + + Path file = new Path("/dir_wo_ec_rs63/file_wo_ec_1"); + try (FSDataOutputStream o = hdfs.create(file)) { + o.write(123); + } + + file = new Path("/dir_wo_ec_rs63/file_wo_ec_2"); + try (FSDataOutputStream o = hdfs.create(file)) { + o.write(123); + } + + dir = new Path("/dir_wo_ec_rs32"); + hdfs.mkdirs(dir); + hdfs.setErasureCodingPolicy(dir, "RS-3-2-1024k"); + + dir = new Path("/dir_wo_ec_rs32/sub_dir_1"); + hdfs.mkdirs(dir); + + file = new Path("/dir_wo_ec_rs32/file_wo_ec"); + try (FSDataOutputStream o = hdfs.create(file)) { + o.write(123); + } + + dir = new Path("/dir_wo_rep"); + hdfs.mkdirs(dir); + + dir = new Path("/dir_wo_rep/sub_dir_1"); + hdfs.mkdirs(dir); + + file = new Path("/dir_wo_rep/file_rep"); + try (FSDataOutputStream o = hdfs.create(file)) { + o.write(123); + } + + // Write results to the fsimage file + hdfs.setSafeMode(HdfsConstants.SafeModeAction.SAFEMODE_ENTER, false); + hdfs.saveNamespace(); + + // Determine the location of the fsimage file + originalFsimage = FSImageTestUtil.findLatestImageFile(FSImageTestUtil + .getFSImage(cluster.getNameNode()).getStorage().getStorageDir(0)); + if (originalFsimage == null) { + throw new RuntimeException("Didn't generate or can't find fsimage"); + } + LOG.debug("original FS image file is " + originalFsimage); + } finally { + if (cluster != null) { + cluster.shutdown(); + } + } + } + + @AfterClass + public static void deleteOriginalFSImage() throws IOException { + if (originalFsimage != null && originalFsimage.exists()) { + originalFsimage.delete(); + } + } + + @Test + public void testPBDelimitedWriterForErasureCodingPolicy() throws Exception { + String expected = DFSTestUtil.readResoucePlainFile( + "testErasureCodingPolicy.csv"); + String result = readECPolicyFromFsimageFile(); + assertEquals(expected, result); + } + + private String readECPolicyFromFsimageFile() throws Exception { + StringBuilder builder = new StringBuilder(); + String delemiter = "\t"; + + File delimitedOutput = new File(tempDir, "delimitedOutput"); + + if (OfflineImageViewerPB.run(new String[] {"-p", "Delimited", + "-i", originalFsimage.getAbsolutePath(), + "-o", delimitedOutput.getAbsolutePath(), + "-ec"}) != 0) { + throw new IOException("oiv returned failure creating " + + "delimited output with ec."); + } + + try (InputStream input = new FileInputStream(delimitedOutput); + BufferedReader reader = + new BufferedReader(new InputStreamReader(input))) { + String line; + boolean header = true; + while ((line = reader.readLine()) != null) { + String[] fields = line.split(delemiter); + if (!header) { + String path = fields[0]; + String ecPolicy = fields[12]; + builder.append(path).append(",").append(ecPolicy).append("\n"); + } + header = false; + } + } + return builder.toString(); + } +} \ No newline at end of file diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testErasureCodingPolicy.csv b/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testErasureCodingPolicy.csv new file mode 100644 index 0000000000000..862a19cec8e67 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testErasureCodingPolicy.csv @@ -0,0 +1,28 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +#dir,erasure coding policy +/,- +/dir_wo_ec_rs63,RS-6-3-1024k +/dir_wo_ec_rs63/sub_dir_1,- +/dir_wo_ec_rs63/sub_dir_2,- +/dir_wo_ec_rs63/file_wo_ec_1,RS-6-3-1024k +/dir_wo_ec_rs63/file_wo_ec_2,RS-6-3-1024k +/dir_wo_ec_rs32,RS-3-2-1024k +/dir_wo_ec_rs32/sub_dir_1,- +/dir_wo_ec_rs32/file_wo_ec,RS-3-2-1024k +/dir_wo_rep,- +/dir_wo_rep/sub_dir_1,- +/dir_wo_rep/file_rep,- \ No newline at end of file diff --git a/hadoop-mapreduce-project/dev-support/jdiff/Apache_Hadoop_MapReduce_Common_3.2.4.xml b/hadoop-mapreduce-project/dev-support/jdiff/Apache_Hadoop_MapReduce_Common_3.2.4.xml new file mode 100644 index 0000000000000..ff74bb7afb254 --- /dev/null +++ b/hadoop-mapreduce-project/dev-support/jdiff/Apache_Hadoop_MapReduce_Common_3.2.4.xml @@ -0,0 +1,113 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/hadoop-mapreduce-project/dev-support/jdiff/Apache_Hadoop_MapReduce_Common_3.3.4.xml b/hadoop-mapreduce-project/dev-support/jdiff/Apache_Hadoop_MapReduce_Common_3.3.4.xml new file mode 100644 index 0000000000000..b8737e922030e --- /dev/null +++ b/hadoop-mapreduce-project/dev-support/jdiff/Apache_Hadoop_MapReduce_Common_3.3.4.xml @@ -0,0 +1,113 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/hadoop-mapreduce-project/dev-support/jdiff/Apache_Hadoop_MapReduce_Core_3.2.4.xml b/hadoop-mapreduce-project/dev-support/jdiff/Apache_Hadoop_MapReduce_Core_3.2.4.xml new file mode 100644 index 0000000000000..239b01b46126d --- /dev/null +++ b/hadoop-mapreduce-project/dev-support/jdiff/Apache_Hadoop_MapReduce_Core_3.2.4.xml @@ -0,0 +1,28073 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + FileStatus of a given cache file on hdfs + @throws IOException]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + DistributedCache is a facility provided by the Map-Reduce + framework to cache files (text, archives, jars etc.) needed by applications. +

    + +

    Applications specify the files, via urls (hdfs:// or http://) to be cached + via the {@link org.apache.hadoop.mapred.JobConf}. The + DistributedCache assumes that the files specified via urls are + already present on the {@link FileSystem} at the path specified by the url + and are accessible by every machine in the cluster.

    + +

    The framework will copy the necessary files on to the worker node before + any tasks for the job are executed on that node. Its efficiency stems from + the fact that the files are only copied once per job and the ability to + cache archives which are un-archived on the workers.

    + +

    DistributedCache can be used to distribute simple, read-only + data/text files and/or more complex types such as archives, jars etc. + Archives (zip, tar and tgz/tar.gz files) are un-archived at the worker nodes. + Jars may be optionally added to the classpath of the tasks, a rudimentary + software distribution mechanism. Files have execution permissions. + In older version of Hadoop Map/Reduce users could optionally ask for symlinks + to be created in the working directory of the child task. In the current + version symlinks are always created. If the URL does not have a fragment + the name of the file or directory will be used. If multiple files or + directories map to the same link name, the last one added, will be used. All + others will not even be downloaded.

    + +

    DistributedCache tracks modification timestamps of the cache + files. Clearly the cache files should not be modified by the application + or externally while the job is executing.

    + +

    Here is an illustrative example on how to use the + DistributedCache:

    + 

    +     // Setting up the cache for the application
+
+     1. Copy the requisite files to the FileSystem:
+
+     $ bin/hadoop fs -copyFromLocal lookup.dat /myapp/lookup.dat
+     $ bin/hadoop fs -copyFromLocal map.zip /myapp/map.zip
+     $ bin/hadoop fs -copyFromLocal mylib.jar /myapp/mylib.jar
+     $ bin/hadoop fs -copyFromLocal mytar.tar /myapp/mytar.tar
+     $ bin/hadoop fs -copyFromLocal mytgz.tgz /myapp/mytgz.tgz
+     $ bin/hadoop fs -copyFromLocal mytargz.tar.gz /myapp/mytargz.tar.gz
+
+     2. Setup the application's JobConf:
+
+     JobConf job = new JobConf();
+     DistributedCache.addCacheFile(new URI("/myapp/lookup.dat#lookup.dat"),
+                                   job);
+     DistributedCache.addCacheArchive(new URI("/myapp/map.zip"), job);
+     DistributedCache.addFileToClassPath(new Path("/myapp/mylib.jar"), job);
+     DistributedCache.addCacheArchive(new URI("/myapp/mytar.tar"), job);
+     DistributedCache.addCacheArchive(new URI("/myapp/mytgz.tgz"), job);
+     DistributedCache.addCacheArchive(new URI("/myapp/mytargz.tar.gz"), job);
+
+     3. Use the cached files in the {@link org.apache.hadoop.mapred.Mapper}
+     or {@link org.apache.hadoop.mapred.Reducer}:
+
+     public static class MapClass extends MapReduceBase
+     implements Mapper<K, V, K, V> {
+
+       private Path[] localArchives;
+       private Path[] localFiles;
+
+       public void configure(JobConf job) {
+         // Get the cached archives/files
+         File f = new File("./map.zip/some/file/in/zip.txt");
+       }
+
+       public void map(K key, V value,
+                       OutputCollector<K, V> output, Reporter reporter)
+       throws IOException {
+         // Use data from the cached archives/files here
+         // ...
+         // ...
+         output.collect(k, v);
+       }
+     }
+
+
    + + It is also very common to use the DistributedCache by using + {@link org.apache.hadoop.util.GenericOptionsParser}. + + This class includes methods that should be used by users + (specifically those mentioned in the example above, as well + as {@link DistributedCache#addArchiveToClassPath(Path, Configuration)}), + as well as methods intended for use by the MapReduce framework + (e.g., {@link org.apache.hadoop.mapred.JobClient}). + + @see org.apache.hadoop.mapred.JobConf + @see org.apache.hadoop.mapred.JobClient + @see org.apache.hadoop.mapreduce.Job]]> +     +     + +     + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + JobTracker, + as {@link JobTracker.State} + + {@link JobTracker.State} should no longer be used on M/R 2.x. The function + is kept to be compatible with M/R 1.x applications. + + @return the invalid state of the JobTracker.]]> + + + + + + + + + + + + + + ClusterStatus provides clients with information such as: +
      +
    1. + Size of the cluster. +
      2. +
    2. + Name of the trackers. +
      3. +
    3. + Task capacity of the cluster. +
      4. +
    4. + The number of currently running map and reduce tasks. +
      5. +
    5. + State of the JobTracker. +
      6. +
    6. + Details regarding black listed trackers. +
      7. +
    + +

    Clients can query for the latest ClusterStatus, via + {@link JobClient#getClusterStatus()}.

    + + @see JobClient]]> +     +     + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Counters represent global counters, defined either by the + Map-Reduce framework or applications. Each Counter can be of + any {@link Enum} type.

    + +

    Counters are bunched into {@link Group}s, each comprising of + counters from a particular Enum class.]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Group of counters, comprising of counters from a particular + counter {@link Enum} class. + +

    Grouphandles localization of the class name and the + counter names.

    ]]> +     +     + + + + + + + + + + + + + + + + + + + + + + + + + FileInputFormat always returns + true. Implementations that may deal with non-splittable files must + override this method. + + FileInputFormat implementations can override this and return + false to ensure that individual input files are never split-up + so that {@link Mapper}s process entire files. + + @param fs the file system that the file is on + @param filename the file name to check + @return is this file splitable?]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + FileInputFormat is the base class for all file-based + InputFormats. This provides a generic implementation of + {@link #getSplits(JobConf, int)}. + + Implementations of FileInputFormat can also override the + {@link #isSplitable(FileSystem, Path)} method to prevent input files + from being split-up in certain situations. Implementations that may + deal with non-splittable files must override this method, since + the default implementation assumes splitting is always possible.]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + true if the job output should be compressed, + false otherwise]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Tasks' Side-Effect Files + +

    Note: The following is valid only if the {@link OutputCommitter} + is {@link FileOutputCommitter}. If OutputCommitter is not + a FileOutputCommitter, the task's temporary output + directory is same as {@link #getOutputPath(JobConf)} i.e. + ${mapreduce.output.fileoutputformat.outputdir}$

    + +

    Some applications need to create/write-to side-files, which differ from + the actual job-outputs. + +

    In such cases there could be issues with 2 instances of the same TIP + (running simultaneously e.g. speculative tasks) trying to open/write-to the + same file (path) on HDFS. Hence the application-writer will have to pick + unique names per task-attempt (e.g. using the attemptid, say + attempt_200709221812_0001_m_000000_0), not just per TIP.

    + +

    To get around this the Map-Reduce framework helps the application-writer + out by maintaining a special + ${mapreduce.output.fileoutputformat.outputdir}/_temporary/_${taskid} + sub-directory for each task-attempt on HDFS where the output of the + task-attempt goes. On successful completion of the task-attempt the files + in the ${mapreduce.output.fileoutputformat.outputdir}/_temporary/_${taskid} (only) + are promoted to ${mapreduce.output.fileoutputformat.outputdir}. Of course, the + framework discards the sub-directory of unsuccessful task-attempts. This + is completely transparent to the application.

    + +

    The application-writer can take advantage of this by creating any + side-files required in ${mapreduce.task.output.dir} during execution + of his reduce-task i.e. via {@link #getWorkOutputPath(JobConf)}, and the + framework will move them out similarly - thus she doesn't have to pick + unique paths per task-attempt.

    + +

    Note: the value of ${mapreduce.task.output.dir} during + execution of a particular task-attempt is actually + ${mapreduce.output.fileoutputformat.outputdir}/_temporary/_{$taskid}, and this value is + set by the map-reduce framework. So, just create any side-files in the + path returned by {@link #getWorkOutputPath(JobConf)} from map/reduce + task to take advantage of this feature.

    + +

    The entire discussion holds true for maps of jobs with + reducer=NONE (i.e. 0 reduces) since output of the map, in that case, + goes directly to HDFS.

    + + @return the {@link Path} to the task's temporary output directory + for the map-reduce job.]]> +     +     + + + + + + + + + + + + + The generated name can be used to create custom files from within the + different tasks for the job, the names for different tasks will not collide + with each other.

    + +

    The given name is postfixed with the task type, 'm' for maps, 'r' for + reduces and the task partition number. For example, give a name 'test' + running on the first map o the job the generated name will be + 'test-m-00000'.

    + + @param conf the configuration for the job. + @param name the name to make unique. + @return a unique name accross all tasks of the job.]]> +     +     + + + + + The path can be used to create custom files from within the map and + reduce tasks. The path name will be unique for each task. The path parent + will be the job output directory.

    ls + +

    This method uses the {@link #getUniqueName} method to make the file name + unique for the task.

    + + @param conf the configuration for the job. + @param name the name for the file. + @return a unique path accross all tasks of the job.]]> +     +     + + + +     + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
    or + conf.setInt(FixedLengthInputFormat.FIXED_RECORD_LENGTH, recordLength); +

    + @see FixedLengthRecordReader]]> +     +     + + + + + + + + + + + + + + + + + + + + + + + Each {@link InputSplit} is then assigned to an individual {@link Mapper} + for processing.

    + +

    Note: The split is a logical split of the inputs and the + input files are not physically split into chunks. For e.g. a split could + be <input-file-path, start, offset> tuple. + + @param job job configuration. + @param numSplits the desired number of splits, a hint. + @return an array of {@link InputSplit}s for the job.]]> + + + + + + + + + It is the responsibility of the RecordReader to respect + record boundaries while processing the logical split to present a + record-oriented view to the individual task.

    + + @param split the {@link InputSplit} + @param job the job that this split belongs to + @return a {@link RecordReader}]]> +     +     + + InputFormat describes the input-specification for a + Map-Reduce job. + +

    The Map-Reduce framework relies on the InputFormat of the + job to:

    +

      +
    1. + Validate the input-specification of the job. +
    2. + Split-up the input file(s) into logical {@link InputSplit}s, each of + which is then assigned to an individual {@link Mapper}. +
      3. +
    3. + Provide the {@link RecordReader} implementation to be used to glean + input records from the logical InputSplit for processing by + the {@link Mapper}. +
      4. +
    + +

    The default behavior of file-based {@link InputFormat}s, typically + sub-classes of {@link FileInputFormat}, is to split the + input into logical {@link InputSplit}s based on the total size, in + bytes, of the input files. However, the {@link FileSystem} blocksize of + the input files is treated as an upper bound for input splits. A lower bound + on the split size can be set via + + mapreduce.input.fileinputformat.split.minsize.

    + +

    Clearly, logical splits based on input-size is insufficient for many + applications since record boundaries are to be respected. In such cases, the + application has to also implement a {@link RecordReader} on whom lies the + responsibilty to respect record-boundaries and present a record-oriented + view of the logical InputSplit to the individual task. + + @see InputSplit + @see RecordReader + @see JobClient + @see FileInputFormat]]> + + + + + + + + + + InputSplit. + + @return the number of bytes in the input split. + @throws IOException]]> + + + + + + InputSplit is + located as an array of Strings. + @throws IOException]]> + + + + InputSplit represents the data to be processed by an + individual {@link Mapper}. + +

    Typically, it presents a byte-oriented view on the input and is the + responsibility of {@link RecordReader} of the job to process this and present + a record-oriented view. + + @see InputFormat + @see RecordReader]]> + + + + + + + + + + SplitLocationInfos describing how the split + data is stored at each location. A null value indicates that all the + locations have the data stored on disk. + @throws IOException]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + JobClient.]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + jobid doesn't correspond to any known job. + @throws IOException]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + JobClient is the primary interface for the user-job to interact + with the cluster. + + JobClient provides facilities to submit jobs, track their + progress, access component-tasks' reports/logs, get the Map-Reduce cluster + status information etc. + +

    The job submission process involves: +

      +
    1. + Checking the input and output specifications of the job. +
      2. +
    2. + Computing the {@link InputSplit}s for the job. +
      3. +
    3. + Setup the requisite accounting information for the {@link DistributedCache} + of the job, if necessary. +
      4. +
    4. + Copying the job's jar and configuration to the map-reduce system directory + on the distributed file-system. +
      5. +
    5. + Submitting the job to the cluster and optionally monitoring + it's status. +
      6. +
    + + Normally the user creates the application, describes various facets of the + job via {@link JobConf} and then uses the JobClient to submit + the job and monitor its progress. + +

    Here is an example on how to use JobClient:

    + 

    +     // Create a new JobConf
+     JobConf job = new JobConf(new Configuration(), MyJob.class);
+     
+     // Specify various job-specific parameters     
+     job.setJobName("myjob");
+     
+     job.setInputPath(new Path("in"));
+     job.setOutputPath(new Path("out"));
+     
+     job.setMapperClass(MyJob.MyMapper.class);
+     job.setReducerClass(MyJob.MyReducer.class);
+
+     // Submit the job, then poll for progress until the job is complete
+     JobClient.runJob(job);
+
    + + Job Control + +

    At times clients would chain map-reduce jobs to accomplish complex tasks + which cannot be done via a single map-reduce job. This is fairly easy since + the output of the job, typically, goes to distributed file-system and that + can be used as the input for the next job.

    + +

    However, this also means that the onus on ensuring jobs are complete + (success/failure) lies squarely on the clients. In such situations the + various job-control options are: +

      +
    1. + {@link #runJob(JobConf)} : submits the job and returns only after + the job has completed. +
      2. +
    2. + {@link #submitJob(JobConf)} : only submits the job, then poll the + returned handle to the {@link RunningJob} to query status and make + scheduling decisions. +
      3. +
    3. + {@link JobConf#setJobEndNotificationURI(String)} : setup a notification + on job-completion, thus avoiding polling. +
      4. +
    + + @see JobConf + @see ClusterStatus + @see Tool + @see DistributedCache]]> +     + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + If the parameter {@code loadDefaults} is false, the new instance + will not load resources from the default files. + + @param loadDefaults specifies whether to load from the default files]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + true if framework should keep the intermediate files + for failed tasks, false otherwise.]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + true if the outputs of the maps are to be compressed, + false otherwise.]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + This comparator should be provided if the equivalence rules for keys + for sorting the intermediates are different from those for grouping keys + before each call to + {@link Reducer#reduce(Object, java.util.Iterator, OutputCollector, Reporter)}.

    + +

    For key-value pairs (K1,V1) and (K2,V2), the values (V1, V2) are passed + in a single call to the reduce function if K1 and K2 compare as equal.

    + +

    Since {@link #setOutputKeyComparatorClass(Class)} can be used to control + how keys are sorted, this can be used in conjunction to simulate + secondary sort on values.

    + +

    Note: This is not a guarantee of the combiner sort being + stable in any sense. (In any case, with the order of available + map-outputs to the combiner being non-deterministic, it wouldn't make + that much sense.)

    + + @param theClass the comparator class to be used for grouping keys for the + combiner. It should implement RawComparator. + @see #setOutputKeyComparatorClass(Class)]]> +     +     + + + + This comparator should be provided if the equivalence rules for keys + for sorting the intermediates are different from those for grouping keys + before each call to + {@link Reducer#reduce(Object, java.util.Iterator, OutputCollector, Reporter)}.

    + +

    For key-value pairs (K1,V1) and (K2,V2), the values (V1, V2) are passed + in a single call to the reduce function if K1 and K2 compare as equal.

    + +

    Since {@link #setOutputKeyComparatorClass(Class)} can be used to control + how keys are sorted, this can be used in conjunction to simulate + secondary sort on values.

    + +

    Note: This is not a guarantee of the reduce sort being + stable in any sense. (In any case, with the order of available + map-outputs to the reduce being non-deterministic, it wouldn't make + that much sense.)

    + + @param theClass the comparator class to be used for grouping keys. + It should implement RawComparator. + @see #setOutputKeyComparatorClass(Class) + @see #setCombinerKeyGroupingComparator(Class)]]> +     +     + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + combiner class used to combine map-outputs + before being sent to the reducers. Typically the combiner is same as the + the {@link Reducer} for the job i.e. {@link #getReducerClass()}. + + @return the user-defined combiner class used to combine map-outputs.]]> + + + + + + combiner class used to combine map-outputs + before being sent to the reducers. + +

    The combiner is an application-specified aggregation operation, which + can help cut down the amount of data transferred between the + {@link Mapper} and the {@link Reducer}, leading to better performance.

    + +

    The framework may invoke the combiner 0, 1, or multiple times, in both + the mapper and reducer tasks. In general, the combiner is called as the + sort/merge result is written to disk. The combiner must: +

      +
    • be side-effect free
      • +
    • have the same input and output key types and the same input and + output value types
      • +
    + +

    Typically the combiner is same as the Reducer for the + job i.e. {@link #setReducerClass(Class)}.

    + + @param theClass the user-defined combiner class used to combine + map-outputs.]]> +     +     + + + true. + + @return true if speculative execution be used for this job, + false otherwise.]]> + + + + + + true if speculative execution + should be turned on, else false.]]> + + + + + true. + + @return true if speculative execution be + used for this job for map tasks, + false otherwise.]]> + + + + + + true if speculative execution + should be turned on for map tasks, + else false.]]> + + + + + true. + + @return true if speculative execution be used + for reduce tasks for this job, + false otherwise.]]> + + + + + + true if speculative execution + should be turned on for reduce tasks, + else false.]]> + + + + + 1. + + @return the number of map tasks for this job.]]> + + + + + + Note: This is only a hint to the framework. The actual + number of spawned map tasks depends on the number of {@link InputSplit}s + generated by the job's {@link InputFormat#getSplits(JobConf, int)}. + + A custom {@link InputFormat} is typically used to accurately control + the number of map tasks for the job.

    + + How many maps? + +

    The number of maps is usually driven by the total size of the inputs + i.e. total number of blocks of the input files.

    + +

    The right level of parallelism for maps seems to be around 10-100 maps + per-node, although it has been set up to 300 or so for very cpu-light map + tasks. Task setup takes awhile, so it is best if the maps take at least a + minute to execute.

    + +

    The default behavior of file-based {@link InputFormat}s is to split the + input into logical {@link InputSplit}s based on the total size, in + bytes, of input files. However, the {@link FileSystem} blocksize of the + input files is treated as an upper bound for input splits. A lower bound + on the split size can be set via + + mapreduce.input.fileinputformat.split.minsize.

    + +

    Thus, if you expect 10TB of input data and have a blocksize of 128MB, + you'll end up with 82,000 maps, unless {@link #setNumMapTasks(int)} is + used to set it even higher.

    + + @param n the number of map tasks for this job. + @see InputFormat#getSplits(JobConf, int) + @see FileInputFormat + @see FileSystem#getDefaultBlockSize() + @see FileStatus#getBlockSize()]]> +     +     + + + 1. + + @return the number of reduce tasks for this job.]]> + + + + + + How many reduces? + +

    The right number of reduces seems to be 0.95 or + 1.75 multiplied by ( + available memory for reduce tasks + (The value of this should be smaller than + numNodes * yarn.nodemanager.resource.memory-mb + since the resource of memory is shared by map tasks and other + applications) / + + mapreduce.reduce.memory.mb). +

    + +

    With 0.95 all of the reduces can launch immediately and + start transfering map outputs as the maps finish. With 1.75 + the faster nodes will finish their first round of reduces and launch a + second wave of reduces doing a much better job of load balancing.

    + +

    Increasing the number of reduces increases the framework overhead, but + increases load balancing and lowers the cost of failures.

    + +

    The scaling factors above are slightly less than whole numbers to + reserve a few reduce slots in the framework for speculative-tasks, failures + etc.

    + + Reducer NONE + +

    It is legal to set the number of reduce-tasks to zero.

    + +

    In this case the output of the map-tasks directly go to distributed + file-system, to the path set by + {@link FileOutputFormat#setOutputPath(JobConf, Path)}. Also, the + framework doesn't sort the map-outputs before writing it out to HDFS.

    + + @param n the number of reduce tasks for this job.]]> +     +     + + + mapreduce.map.maxattempts + property. If this property is not already set, the default is 4 attempts. + + @return the max number of attempts per map task.]]> + + + + + + + + + + + mapreduce.reduce.maxattempts + property. If this property is not already set, the default is 4 attempts. + + @return the max number of attempts per reduce task.]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + noFailures, the + tasktracker is blacklisted for this job. + + @param noFailures maximum no. of failures of a given job per tasktracker.]]> + + + + + blacklisted for this job. + + @return the maximum no. of failures of a given job per tasktracker.]]> + + + + + failed. + + Defaults to zero, i.e. any failed map-task results in + the job being declared as {@link JobStatus#FAILED}. + + @return the maximum percentage of map tasks that can fail without + the job being aborted.]]> + + + + + + failed. + + @param percent the maximum percentage of map tasks that can fail without + the job being aborted.]]> + + + + + failed. + + Defaults to zero, i.e. any failed reduce-task results + in the job being declared as {@link JobStatus#FAILED}. + + @return the maximum percentage of reduce tasks that can fail without + the job being aborted.]]> + + + + + + failed. + + @param percent the maximum percentage of reduce tasks that can fail without + the job being aborted.]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + The debug script can aid debugging of failed map tasks. The script is + given task's stdout, stderr, syslog, jobconf files as arguments.

    + +

    The debug command, run on the node where the map failed, is:

    + 

    + $script $stdout $stderr $syslog $jobconf.
+
    + +

    The script file is distributed through {@link DistributedCache} + APIs. The script needs to be symlinked.

    + +

    Here is an example on how to submit a script + 

    + job.setMapDebugScript("./myscript");
+ DistributedCache.createSymlink(job);
+ DistributedCache.addCacheFile("/debug/scripts/myscript#myscript");
+
    + + @param mDbgScript the script name]]> +     +     + + + + + + + + + The debug script can aid debugging of failed reduce tasks. The script + is given task's stdout, stderr, syslog, jobconf files as arguments.

    + +

    The debug command, run on the node where the map failed, is:

    + 

    + $script $stdout $stderr $syslog $jobconf.
+
    + +

    The script file is distributed through {@link DistributedCache} + APIs. The script file needs to be symlinked

    + +

    Here is an example on how to submit a script + 

    + job.setReduceDebugScript("./myscript");
+ DistributedCache.createSymlink(job);
+ DistributedCache.addCacheFile("/debug/scripts/myscript#myscript");
+
    + + @param rDbgScript the script name]]> +     +     + + + + + + + + null if it hasn't + been set. + @see #setJobEndNotificationURI(String)]]> + + + + + + The uri can contain 2 special parameters: $jobId and + $jobStatus. Those, if present, are replaced by the job's + identifier and completion-status respectively.

    + +

    This is typically used by application-writers to implement chaining of + Map-Reduce jobs in an asynchronous manner.

    + + @param uri the job end notification uri + @see JobStatus]]> +     +     + + + + + + + + + + + + + + + When a job starts, a shared directory is created at location + + ${mapreduce.cluster.local.dir}/taskTracker/$user/jobcache/$jobid/work/ . + This directory is exposed to the users through + mapreduce.job.local.dir . + So, the tasks can use this space + as scratch space and share files among them.

    + This value is available as System property also. + + @return The localized job specific shared directory]]> +     +     + + + + For backward compatibility, if the job configuration sets the + key {@link #MAPRED_TASK_MAXVMEM_PROPERTY} to a value different + from {@link #DISABLED_MEMORY_LIMIT}, that value will be used + after converting it from bytes to MB. + @return memory required to run a map task of the job, in MB,]]> + + + + + + + + + For backward compatibility, if the job configuration sets the + key {@link #MAPRED_TASK_MAXVMEM_PROPERTY} to a value different + from {@link #DISABLED_MEMORY_LIMIT}, that value will be used + after converting it from bytes to MB. + @return memory required to run a reduce task of the job, in MB.]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + This method is deprecated. Now, different memory limits can be + set for map and reduce tasks of a job, in MB. +

    + For backward compatibility, if the job configuration sets the + key {@link #MAPRED_TASK_MAXVMEM_PROPERTY}, that value is returned. + Otherwise, this method will return the larger of the values returned by + {@link #getMemoryForMapTask()} and {@link #getMemoryForReduceTask()} + after converting them into bytes. + + @return Memory required to run a task of this job, in bytes. + @see #setMaxVirtualMemoryForTask(long) + @deprecated Use {@link #getMemoryForMapTask()} and + {@link #getMemoryForReduceTask()}]]> + + + + + + + mapred.task.maxvmem is split into + mapreduce.map.memory.mb + and mapreduce.map.memory.mb,mapred + each of the new key are set + as mapred.task.maxvmem / 1024 + as new values are in MB + + @param vmem Maximum amount of virtual memory in bytes any task of this job + can use. + @see #getMaxVirtualMemoryForTask() + @deprecated + Use {@link #setMemoryForMapTask(long mem)} and + Use {@link #setMemoryForReduceTask(long mem)}]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + k1=v1,k2=v2. Further it can + reference existing environment variables via $key on + Linux or %key% on Windows. + + Example: +

      +
    • A=foo - This will set the env variable A to foo.
      • +
    + + @deprecated Use {@link #MAPRED_MAP_TASK_ENV} or + {@link #MAPRED_REDUCE_TASK_ENV}]]> +     + + + + k1=v1,k2=v2. Further it can + reference existing environment variables via $key on + Linux or %key% on Windows. + + Example: +
      +
    • A=foo - This will set the env variable A to foo.
      • +
    + + You can also add environment variables individually by appending + .VARNAME to this configuration key, where VARNAME is + the name of the environment variable. + + Example: +
      +
    • mapreduce.map.env.VARNAME=value
      • +
    ]]> +     +     + + + k1=v1,k2=v2. Further it can + reference existing environment variables via $key on + Linux or %key% on Windows. + + Example: +
      +
    • A=foo - This will set the env variable A to foo.
      • +
    + + You can also add environment variables individually by appending + .VARNAME to this configuration key, where VARNAME is + the name of the environment variable. + + Example: +
      +
    • mapreduce.reduce.env.VARNAME=value
      • +
    ]]> +     +     + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + JobConf is the primary interface for a user to describe a + map-reduce job to the Hadoop framework for execution. The framework tries to + faithfully execute the job as-is described by JobConf, however: +
      +
    1. + Some configuration parameters might have been marked as + + final by administrators and hence cannot be altered. +
      2. +
    2. + While some job parameters are straight-forward to set + (e.g. {@link #setNumReduceTasks(int)}), some parameters interact subtly + with the rest of the framework and/or job-configuration and is relatively + more complex for the user to control finely + (e.g. {@link #setNumMapTasks(int)}). +
      3. +
    + +

    JobConf typically specifies the {@link Mapper}, combiner + (if any), {@link Partitioner}, {@link Reducer}, {@link InputFormat} and + {@link OutputFormat} implementations to be used etc. + +

    Optionally JobConf is used to specify other advanced facets + of the job such as Comparators to be used, files to be put in + the {@link DistributedCache}, whether or not intermediate and/or job outputs + are to be compressed (and how), debugability via user-provided scripts + ( {@link #setMapDebugScript(String)}/{@link #setReduceDebugScript(String)}), + for doing post-processing on task logs, task's stdout, stderr, syslog. + and etc.

    + +

    Here is an example on how to configure a job via JobConf:

    + 

    +     // Create a new JobConf
+     JobConf job = new JobConf(new Configuration(), MyJob.class);
+     
+     // Specify various job-specific parameters     
+     job.setJobName("myjob");
+     
+     FileInputFormat.setInputPaths(job, new Path("in"));
+     FileOutputFormat.setOutputPath(job, new Path("out"));
+     
+     job.setMapperClass(MyJob.MyMapper.class);
+     job.setCombinerClass(MyJob.MyReducer.class);
+     job.setReducerClass(MyJob.MyReducer.class);
+     
+     job.setInputFormat(SequenceFileInputFormat.class);
+     job.setOutputFormat(SequenceFileOutputFormat.class);
+
    + + @see JobClient + @see ClusterStatus + @see Tool + @see DistributedCache]]> +     +     + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + any job + run on the jobtracker started at 200707121733, we would use : + 
     
+ JobID.getTaskIDsPattern("200707121733", null);
+
    + which will return : + 
     "job_200707121733_[0-9]*"
    + @param jtIdentifier jobTracker identifier, or null + @param jobId job number, or null + @return a regex pattern matching JobIDs]]> +     +     + + + An example JobID is : + job_200707121733_0003 , which represents the third job + running at the jobtracker started at 200707121733. +

    + Applications should never construct or parse JobID strings, but rather + use appropriate constructors or {@link #forName(String)} method. + + @see TaskID + @see TaskAttemptID]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Output pairs need not be of the same types as input pairs. A given + input pair may map to zero or many output pairs. Output pairs are + collected with calls to + {@link OutputCollector#collect(Object,Object)}.

    + +

    Applications can use the {@link Reporter} provided to report progress + or just indicate that they are alive. In scenarios where the application + takes significant amount of time to process individual key/value + pairs, this is crucial since the framework might assume that the task has + timed-out and kill that task. The other way of avoiding this is to set + + mapreduce.task.timeout to a high-enough value (or even zero for no + time-outs).

    + + @param key the input key. + @param value the input value. + @param output collects mapped keys and values. + @param reporter facility to report progress.]]> +     + + + Maps are the individual tasks which transform input records into a + intermediate records. The transformed intermediate records need not be of + the same type as the input records. A given input pair may map to zero or + many output pairs.

    + +

    The Hadoop Map-Reduce framework spawns one map task for each + {@link InputSplit} generated by the {@link InputFormat} for the job. + Mapper implementations can access the {@link JobConf} for the + job via the {@link JobConfigurable#configure(JobConf)} and initialize + themselves. Similarly they can use the {@link Closeable#close()} method for + de-initialization.

    + +

    The framework then calls + {@link #map(Object, Object, OutputCollector, Reporter)} + for each key/value pair in the InputSplit for that task.

    + +

    All intermediate values associated with a given output key are + subsequently grouped by the framework, and passed to a {@link Reducer} to + determine the final output. Users can control the grouping by specifying + a Comparator via + {@link JobConf#setOutputKeyComparatorClass(Class)}.

    + +

    The grouped Mapper outputs are partitioned per + Reducer. Users can control which keys (and hence records) go to + which Reducer by implementing a custom {@link Partitioner}. + +

    Users can optionally specify a combiner, via + {@link JobConf#setCombinerClass(Class)}, to perform local aggregation of the + intermediate outputs, which helps to cut down the amount of data transferred + from the Mapper to the Reducer. + +

    The intermediate, grouped outputs are always stored in + {@link SequenceFile}s. Applications can specify if and how the intermediate + outputs are to be compressed and which {@link CompressionCodec}s are to be + used via the JobConf.

    + +

    If the job has + zero + reduces then the output of the Mapper is directly written + to the {@link FileSystem} without grouping by keys.

    + +

    Example:

    + 

    +     public class MyMapper<K extends WritableComparable, V extends Writable> 
+     extends MapReduceBase implements Mapper<K, V, K, V> {
+     
+       static enum MyCounters { NUM_RECORDS }
+       
+       private String mapTaskId;
+       private String inputFile;
+       private int noRecords = 0;
+       
+       public void configure(JobConf job) {
+         mapTaskId = job.get(JobContext.TASK_ATTEMPT_ID);
+         inputFile = job.get(JobContext.MAP_INPUT_FILE);
+       }
+       
+       public void map(K key, V val,
+                       OutputCollector<K, V> output, Reporter reporter)
+       throws IOException {
+         // Process the <key, value> pair (assume this takes a while)
+         // ...
+         // ...
+         
+         // Let the framework know that we are alive, and kicking!
+         // reporter.progress();
+         
+         // Process some more
+         // ...
+         // ...
+         
+         // Increment the no. of <key, value> pairs processed
+         ++noRecords;
+
+         // Increment counters
+         reporter.incrCounter(NUM_RECORDS, 1);
+        
+         // Every 100 records update application-level status
+         if ((noRecords%100) == 0) {
+           reporter.setStatus(mapTaskId + " processed " + noRecords + 
+                              " from input-file: " + inputFile); 
+         }
+         
+         // Output the result
+         output.collect(key, val);
+       }
+     }
+
    + +

    Applications may write a custom {@link MapRunnable} to exert greater + control on map processing e.g. multi-threaded Mappers etc.

    + + @see JobConf + @see InputFormat + @see Partitioner + @see Reducer + @see MapReduceBase + @see MapRunnable + @see SequenceFile]]> +     +     + + + + + + + + + + + + + + + + + + + + + Provides default no-op implementations for a few methods, most non-trivial + applications need to override some of them.

    ]]> +     +     + + + + + + + + + + + <key, value> pairs. + +

    Mapping of input records to output records is complete when this method + returns.

    + + @param input the {@link RecordReader} to read the input records. + @param output the {@link OutputCollector} to collect the outputrecords. + @param reporter {@link Reporter} to report progress, status-updates etc. + @throws IOException]]> +     +     + + Custom implementations of MapRunnable can exert greater + control on map processing e.g. multi-threaded, asynchronous mappers etc.

    + + @see Mapper]]> +     +     + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + nearly + equal content length.
    + Subclasses implement {@link #getRecordReader(InputSplit, JobConf, Reporter)} + to construct RecordReader's for MultiFileSplit's. + @see MultiFileSplit]]> +     +     + + + + + + + + + + + + + MultiFileSplit can be used to implement {@link RecordReader}'s, with + reading one record per file. + @see FileSplit + @see MultiFileInputFormat]]> + + + + + + + + + + + + + + + <key, value> pairs output by {@link Mapper}s + and {@link Reducer}s. + +

    OutputCollector is the generalization of the facility + provided by the Map-Reduce framework to collect data output by either the + Mapper or the Reducer i.e. intermediate outputs + or the output of the job.

    ]]> +     +     + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + true if task output recovery is supported, + false otherwise + @throws IOException + @see #recoverTask(TaskAttemptContext)]]> + + + + + + + true repeatable job commit is supported, + false otherwise + @throws IOException]]> + + + + + + + + + + + OutputCommitter. This is called from the application master + process, but it is called individually for each task. + + If an exception is thrown the task will be attempted again. + + @param taskContext Context of the task whose output is being recovered + @throws IOException]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + OutputCommitter describes the commit of task output for a + Map-Reduce job. + +

    The Map-Reduce framework relies on the OutputCommitter of + the job to:

    +

      +
    1. + Setup the job during initialization. For example, create the temporary + output directory for the job during the initialization of the job. +
      2. +
    2. + Cleanup the job after the job completion. For example, remove the + temporary output directory after the job completion. +
      3. +
    3. + Setup the task temporary output. +
      4. +
    4. + Check whether a task needs a commit. This is to avoid the commit + procedure if a task does not need commit. +
      5. +
    5. + Commit of the task output. +
      6. +
    6. + Discard the task commit. +
      7. +
    + The methods in this class can be called from several different processes and + from several different contexts. It is important to know which process and + which context each is called from. Each method should be marked accordingly + in its documentation. It is also important to note that not all methods are + guaranteed to be called once and only once. If a method is not guaranteed to + have this property the output committer needs to handle this appropriately. + Also note it will only be in rare situations where they may be called + multiple times for the same task. + + @see FileOutputCommitter + @see JobContext + @see TaskAttemptContext]]> +     +     + + + + + + + + + + + + + + + + + + + This is to validate the output specification for the job when it is + a job is submitted. Typically checks that it does not already exist, + throwing an exception when it already exists, so that output is not + overwritten.

    + + @param ignored + @param job job configuration. + @throws IOException when output should not be attempted]]> +     +     + + OutputFormat describes the output-specification for a + Map-Reduce job. + +

    The Map-Reduce framework relies on the OutputFormat of the + job to:

    +

      +
    1. + Validate the output-specification of the job. For e.g. check that the + output directory doesn't already exist. +
    2. + Provide the {@link RecordWriter} implementation to be used to write out + the output files of the job. Output files are stored in a + {@link FileSystem}. +
      3. +
    + + @see RecordWriter + @see JobConf]]> +     +     + + + + + + + + + + + + + + + + + + + + + + + Typically a hash function on a all or a subset of the key.

    + + @param key the key to be paritioned. + @param value the entry value. + @param numPartitions the total number of partitions. + @return the partition number for the key.]]> +     +     + + Partitioner controls the partitioning of the keys of the + intermediate map-outputs. The key (or a subset of the key) is used to derive + the partition, typically by a hash function. The total number of partitions + is the same as the number of reduce tasks for the job. Hence this controls + which of the m reduce tasks the intermediate key (and hence the + record) is sent for reduction.

    + +

    Note: A Partitioner is created only when there are multiple + reducers.

    + + @see Reducer]]> +     +     + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + 0.0 to 1.0. + @throws IOException]]> + + + + RecordReader reads <key, value> pairs from an + {@link InputSplit}. + +

    RecordReader, typically, converts the byte-oriented view of + the input, provided by the InputSplit, and presents a + record-oriented view for the {@link Mapper} and {@link Reducer} tasks for + processing. It thus assumes the responsibility of processing record + boundaries and presenting the tasks with keys and values.

    + + @see InputSplit + @see InputFormat]]> +     +     + + + + + + + + + + + + + + + + RecordWriter to future operations. + + @param reporter facility to report progress. + @throws IOException]]> + + + + RecordWriter writes the output <key, value> pairs + to an output file. + +

    RecordWriter implementations write the job outputs to the + {@link FileSystem}. + + @see OutputFormat]]> + + + + + + + + + + + + + + + Reduces values for a given key. + +

    The framework calls this method for each + <key, (list of values)> pair in the grouped inputs. + Output values must be of the same type as input values. Input keys must + not be altered. The framework will reuse the key and value objects + that are passed into the reduce, therefore the application should clone + the objects they want to keep a copy of. In many cases, all values are + combined into zero or one value. +

    + +

    Output pairs are collected with calls to + {@link OutputCollector#collect(Object,Object)}.

    + +

    Applications can use the {@link Reporter} provided to report progress + or just indicate that they are alive. In scenarios where the application + takes a significant amount of time to process individual key/value + pairs, this is crucial since the framework might assume that the task has + timed-out and kill that task. The other way of avoiding this is to set + + mapreduce.task.timeout to a high-enough value (or even zero for no + time-outs).

    + + @param key the key. + @param values the list of values to reduce. + @param output to collect keys and combined values. + @param reporter facility to report progress.]]> +     + + + The number of Reducers for the job is set by the user via + {@link JobConf#setNumReduceTasks(int)}. Reducer implementations + can access the {@link JobConf} for the job via the + {@link JobConfigurable#configure(JobConf)} method and initialize themselves. + Similarly they can use the {@link Closeable#close()} method for + de-initialization.

    + +

    Reducer has 3 primary phases:

    +
      +
    1. + + Shuffle + +

      Reducer is input the grouped output of a {@link Mapper}. + In the phase the framework, for each Reducer, fetches the + relevant partition of the output of all the Mappers, via HTTP. +

      +
      2. + +
    2. + Sort + +

      The framework groups Reducer inputs by keys + (since different Mappers may have output the same key) in this + stage.

      + +

      The shuffle and sort phases occur simultaneously i.e. while outputs are + being fetched they are merged.

      + + SecondarySort + +

      If equivalence rules for keys while grouping the intermediates are + different from those for grouping keys before reduction, then one may + specify a Comparator via + {@link JobConf#setOutputValueGroupingComparator(Class)}.Since + {@link JobConf#setOutputKeyComparatorClass(Class)} can be used to + control how intermediate keys are grouped, these can be used in conjunction + to simulate secondary sort on values.

      + + + For example, say that you want to find duplicate web pages and tag them + all with the url of the "best" known example. You would set up the job + like: +
        +
      • Map Input Key: url
        • +
      • Map Input Value: document
        • +
      • Map Output Key: document checksum, url pagerank
        • +
      • Map Output Value: url
        • +
      • Partitioner: by checksum
        • +
      • OutputKeyComparator: by checksum and then decreasing pagerank
        • +
      • OutputValueGroupingComparator: by checksum
        • +
      +
      3. + +
    3. + Reduce + +

      In this phase the + {@link #reduce(Object, Iterator, OutputCollector, Reporter)} + method is called for each <key, (list of values)> pair in + the grouped inputs.

      +

      The output of the reduce task is typically written to the + {@link FileSystem} via + {@link OutputCollector#collect(Object, Object)}.

      +
      4. +
    + +

    The output of the Reducer is not re-sorted.

    + +

    Example:

    + 

    +     public class MyReducer<K extends WritableComparable, V extends Writable> 
+     extends MapReduceBase implements Reducer<K, V, K, V> {
+     
+       static enum MyCounters { NUM_RECORDS }
+        
+       private String reduceTaskId;
+       private int noKeys = 0;
+       
+       public void configure(JobConf job) {
+         reduceTaskId = job.get(JobContext.TASK_ATTEMPT_ID);
+       }
+       
+       public void reduce(K key, Iterator<V> values,
+                          OutputCollector<K, V> output, 
+                          Reporter reporter)
+       throws IOException {
+       
+         // Process
+         int noValues = 0;
+         while (values.hasNext()) {
+           V value = values.next();
+           
+           // Increment the no. of values for this key
+           ++noValues;
+           
+           // Process the <key, value> pair (assume this takes a while)
+           // ...
+           // ...
+           
+           // Let the framework know that we are alive, and kicking!
+           if ((noValues%10) == 0) {
+             reporter.progress();
+           }
+         
+           // Process some more
+           // ...
+           // ...
+           
+           // Output the <key, value> 
+           output.collect(key, value);
+         }
+         
+         // Increment the no. of <key, list of values> pairs processed
+         ++noKeys;
+         
+         // Increment counters
+         reporter.incrCounter(NUM_RECORDS, 1);
+         
+         // Every 100 keys update application-level status
+         if ((noKeys%100) == 0) {
+           reporter.setStatus(reduceTaskId + " processed " + noKeys);
+         }
+       }
+     }
+
    + + @see Mapper + @see Partitioner + @see Reporter + @see MapReduceBase]]> +     +     + + + + + + + + + + + + + + Counter of the given group/name.]]> + + + + + + + Counter of the given group/name.]]> + + + + + + + Enum. + @param amount A non-negative amount by which the counter is to + be incremented.]]> + + + + + + + + + + + + + + InputSplit that the map is reading from. + @throws UnsupportedOperationException if called outside a mapper]]> + + + + + + + + + + + + + + {@link Mapper} and {@link Reducer} can use the Reporter + provided to report progress or just indicate that they are alive. In + scenarios where the application takes significant amount of time to + process individual key/value pairs, this is crucial since the framework + might assume that the task has timed-out and kill that task. + +

    Applications can also update {@link Counters} via the provided + Reporter .

    + + @see Progressable + @see Counters]]> +     +     + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + progress of the job's map-tasks, as a float between 0.0 + and 1.0. When all map tasks have completed, the function returns 1.0. + + @return the progress of the job's map-tasks. + @throws IOException]]> + + + + + + progress of the job's reduce-tasks, as a float between 0.0 + and 1.0. When all reduce tasks have completed, the function returns 1.0. + + @return the progress of the job's reduce-tasks. + @throws IOException]]> + + + + + + progress of the job's cleanup-tasks, as a float between 0.0 + and 1.0. When all cleanup tasks have completed, the function returns 1.0. + + @return the progress of the job's cleanup-tasks. + @throws IOException]]> + + + + + + progress of the job's setup-tasks, as a float between 0.0 + and 1.0. When all setup tasks have completed, the function returns 1.0. + + @return the progress of the job's setup-tasks. + @throws IOException]]> + + + + + + true if the job is complete, else false. + @throws IOException]]> + + + + + + true if the job succeeded, else false. + @throws IOException]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + true if the job retired, else false. + @throws IOException]]> + + + + + + + + + + RunningJob is the user-interface to query for details on a + running Map-Reduce job. + +

    Clients can get hold of RunningJob via the {@link JobClient} + and then query the running-job for details such as name, configuration, + progress etc.

    + + @see JobClient]]> +     +     + + + + + + + + + + + + + + + + + + + + + + + + + This allows the user to specify the key class to be different + from the actual class ({@link BytesWritable}) used for writing

    + + @param conf the {@link JobConf} to modify + @param theClass the SequenceFile output key class.]]> +     +     + + + + + This allows the user to specify the value class to be different + from the actual class ({@link BytesWritable}) used for writing

    + + @param conf the {@link JobConf} to modify + @param theClass the SequenceFile output key class.]]> +     +     + + + + + + + + + + + + + + + + + + + + + + + + + + + +     + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + true if auto increment + {@link SkipBadRecords#COUNTER_MAP_PROCESSED_RECORDS}. + false otherwise.]]> + + + + + + + + + + + + + true if auto increment + {@link SkipBadRecords#COUNTER_REDUCE_PROCESSED_GROUPS}. + false otherwise.]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Hadoop provides an optional mode of execution in which the bad records + are detected and skipped in further attempts. + +

    This feature can be used when map/reduce tasks crashes deterministically on + certain input. This happens due to bugs in the map/reduce function. The usual + course would be to fix these bugs. But sometimes this is not possible; + perhaps the bug is in third party libraries for which the source code is + not available. Due to this, the task never reaches to completion even with + multiple attempts and complete data for that task is lost.

    + +

    With this feature, only a small portion of data is lost surrounding + the bad record, which may be acceptable for some user applications. + see {@link SkipBadRecords#setMapperMaxSkipRecords(Configuration, long)}

    + +

    The skipping mode gets kicked off after certain no of failures + see {@link SkipBadRecords#setAttemptsToStartSkipping(Configuration, int)}

    + +

    In the skipping mode, the map/reduce task maintains the record range which + is getting processed at all times. Before giving the input to the + map/reduce function, it sends this record range to the Task tracker. + If task crashes, the Task tracker knows which one was the last reported + range. On further attempts that range get skipped.

    ]]> +     +     + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + all task attempt IDs + of any jobtracker, in any job, of the first + map task, we would use : + 
     
+ TaskAttemptID.getTaskAttemptIDsPattern(null, null, true, 1, null);
+
    + which will return : + 
     "attempt_[^_]*_[0-9]*_m_000001_[0-9]*"
    + @param jtIdentifier jobTracker identifier, or null + @param jobId job number, or null + @param isMap whether the tip is a map, or null + @param taskId taskId number, or null + @param attemptId the task attempt number, or null + @return a regex pattern matching TaskAttemptIDs]]> +     +     + + + + + + + + all task attempt IDs + of any jobtracker, in any job, of the first + map task, we would use : + 
     
+ TaskAttemptID.getTaskAttemptIDsPattern(null, null, TaskType.MAP, 1, null);
+
    + which will return : + 
     "attempt_[^_]*_[0-9]*_m_000001_[0-9]*"
    + @param jtIdentifier jobTracker identifier, or null + @param jobId job number, or null + @param type the {@link TaskType} + @param taskId taskId number, or null + @param attemptId the task attempt number, or null + @return a regex pattern matching TaskAttemptIDs]]> +     +     + + + An example TaskAttemptID is : + attempt_200707121733_0003_m_000005_0 , which represents the + zeroth task attempt for the fifth map task in the third job + running at the jobtracker started at 200707121733. +

    + Applications should never construct or parse TaskAttemptID strings + , but rather use appropriate constructors or {@link #forName(String)} + method. + + @see JobID + @see TaskID]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + the first map task + of any jobtracker, of any job, we would use : + 

     
+ TaskID.getTaskIDsPattern(null, null, true, 1);
+
    + which will return : + 
     "task_[^_]*_[0-9]*_m_000001*"
    + @param jtIdentifier jobTracker identifier, or null + @param jobId job number, or null + @param isMap whether the tip is a map, or null + @param taskId taskId number, or null + @return a regex pattern matching TaskIDs + @deprecated Use {@link TaskID#getTaskIDsPattern(String, Integer, TaskType, + Integer)}]]> +     + + + + + + + + the first map task + of any jobtracker, of any job, we would use : + 
     
+ TaskID.getTaskIDsPattern(null, null, true, 1);
+
    + which will return : + 
     "task_[^_]*_[0-9]*_m_000001*"
    + @param jtIdentifier jobTracker identifier, or null + @param jobId job number, or null + @param type the {@link TaskType}, or null + @param taskId taskId number, or null + @return a regex pattern matching TaskIDs]]> +     +     + + + + + + + An example TaskID is : + task_200707121733_0003_m_000005 , which represents the + fifth map task in the third job running at the jobtracker + started at 200707121733. +

    + Applications should never construct or parse TaskID strings + , but rather use appropriate constructors or {@link #forName(String)} + method. + + @see JobID + @see TaskAttemptID]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + true if the Job was added.]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + ([,]*) + func ::= tbl(,"") + class ::= @see java.lang.Class#forName(java.lang.String) + path ::= @see org.apache.hadoop.fs.Path#Path(java.lang.String) + } + Reads expression from the mapred.join.expr property and + user-supplied join types from mapred.join.define.<ident> + types. Paths supplied to tbl are given as input paths to the + InputFormat class listed. + @see #compose(java.lang.String, java.lang.Class, java.lang.String...)]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + ,

    ) }]]> + + + + + + + + (tbl(,),tbl(,),...,tbl(,)) }]]> + + + + + + + + (tbl(,),tbl(,),...,tbl(,)) }]]> + + + + mapred.join.define.<ident> to a classname. In the expression + mapred.join.expr, the identifier will be assumed to be a + ComposableRecordReader. + mapred.join.keycomparator can be a classname used to compare keys + in the join. + @see #setFormat + @see JoinRecordReader + @see MultiFilterRecordReader]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + ...... + }]]> + + + + + + + + + + + + + + + + + + + + + capacity children to position + id in the parent reader. + The id of a root CompositeRecordReader is -1 by convention, but relying + on this is not recommended.]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + override(S1,S2,S3) will prefer values + from S3 over S2, and values from S2 over S1 for all keys + emitted from all sources.]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + It has to be specified how key and values are passed from one element of + the chain to the next, by value or by reference. If a Mapper leverages the + assumed semantics that the key and values are not modified by the collector + 'by value' must be used. If the Mapper does not expect this semantics, as + an optimization to avoid serialization and deserialization 'by reference' + can be used. +

    + For the added Mapper the configuration given for it, + mapperConf, have precedence over the job's JobConf. This + precedence is in effect when the task is running. +

    + IMPORTANT: There is no need to specify the output key/value classes for the + ChainMapper, this is done by the addMapper for the last mapper in the chain +

    + + @param job job's JobConf to add the Mapper class. + @param klass the Mapper class to add. + @param inputKeyClass mapper input key class. + @param inputValueClass mapper input value class. + @param outputKeyClass mapper output key class. + @param outputValueClass mapper output value class. + @param byValue indicates if key/values should be passed by value + to the next Mapper in the chain, if any. + @param mapperConf a JobConf with the configuration for the Mapper + class. It is recommended to use a JobConf without default values using the + JobConf(boolean loadDefaults) constructor with FALSE.]]> + + + + + + + If this method is overriden super.configure(...) should be + invoked at the beginning of the overwriter method.]]> + + + + + + + + + + map(...) methods of the Mappers in the chain.]]> + + + + + + + If this method is overriden super.close() should be + invoked at the end of the overwriter method.]]> + + + + + The Mapper classes are invoked in a chained (or piped) fashion, the output of + the first becomes the input of the second, and so on until the last Mapper, + the output of the last Mapper will be written to the task's output. +

    + The key functionality of this feature is that the Mappers in the chain do not + need to be aware that they are executed in a chain. This enables having + reusable specialized Mappers that can be combined to perform composite + operations within a single task. +

    + Special care has to be taken when creating chains that the key/values output + by a Mapper are valid for the following Mapper in the chain. It is assumed + all Mappers and the Reduce in the chain use maching output and input key and + value classes as no conversion is done by the chaining code. +

    + Using the ChainMapper and the ChainReducer classes is possible to compose + Map/Reduce jobs that look like [MAP+ / REDUCE MAP*]. And + immediate benefit of this pattern is a dramatic reduction in disk IO. +

    + IMPORTANT: There is no need to specify the output key/value classes for the + ChainMapper, this is done by the addMapper for the last mapper in the chain. +

    + ChainMapper usage pattern: +

    + 

    + ...
+ conf.setJobName("chain");
+ conf.setInputFormat(TextInputFormat.class);
+ conf.setOutputFormat(TextOutputFormat.class);
+
+ JobConf mapAConf = new JobConf(false);
+ ...
+ ChainMapper.addMapper(conf, AMap.class, LongWritable.class, Text.class,
+   Text.class, Text.class, true, mapAConf);
+
+ JobConf mapBConf = new JobConf(false);
+ ...
+ ChainMapper.addMapper(conf, BMap.class, Text.class, Text.class,
+   LongWritable.class, Text.class, false, mapBConf);
+
+ JobConf reduceConf = new JobConf(false);
+ ...
+ ChainReducer.setReducer(conf, XReduce.class, LongWritable.class, Text.class,
+   Text.class, Text.class, true, reduceConf);
+
+ ChainReducer.addMapper(conf, CMap.class, Text.class, Text.class,
+   LongWritable.class, Text.class, false, null);
+
+ ChainReducer.addMapper(conf, DMap.class, LongWritable.class, Text.class,
+   LongWritable.class, LongWritable.class, true, null);
+
+ FileInputFormat.setInputPaths(conf, inDir);
+ FileOutputFormat.setOutputPath(conf, outDir);
+ ...
+
+ JobClient jc = new JobClient(conf);
+ RunningJob job = jc.submitJob(conf);
+ ...
+
    ]]> +     +     + + + + + + + + + + + + + + + + + + + + + It has to be specified how key and values are passed from one element of + the chain to the next, by value or by reference. If a Reducer leverages the + assumed semantics that the key and values are not modified by the collector + 'by value' must be used. If the Reducer does not expect this semantics, as + an optimization to avoid serialization and deserialization 'by reference' + can be used. +

    + For the added Reducer the configuration given for it, + reducerConf, have precedence over the job's JobConf. This + precedence is in effect when the task is running. +

    + IMPORTANT: There is no need to specify the output key/value classes for the + ChainReducer, this is done by the setReducer or the addMapper for the last + element in the chain. + + @param job job's JobConf to add the Reducer class. + @param klass the Reducer class to add. + @param inputKeyClass reducer input key class. + @param inputValueClass reducer input value class. + @param outputKeyClass reducer output key class. + @param outputValueClass reducer output value class. + @param byValue indicates if key/values should be passed by value + to the next Mapper in the chain, if any. + @param reducerConf a JobConf with the configuration for the Reducer + class. It is recommended to use a JobConf without default values using the + JobConf(boolean loadDefaults) constructor with FALSE.]]> + + + + + + + + + + + + + + It has to be specified how key and values are passed from one element of + the chain to the next, by value or by reference. If a Mapper leverages the + assumed semantics that the key and values are not modified by the collector + 'by value' must be used. If the Mapper does not expect this semantics, as + an optimization to avoid serialization and deserialization 'by reference' + can be used. +

    + For the added Mapper the configuration given for it, + mapperConf, have precedence over the job's JobConf. This + precedence is in effect when the task is running. +

    + IMPORTANT: There is no need to specify the output key/value classes for the + ChainMapper, this is done by the addMapper for the last mapper in the chain + . + + @param job chain job's JobConf to add the Mapper class. + @param klass the Mapper class to add. + @param inputKeyClass mapper input key class. + @param inputValueClass mapper input value class. + @param outputKeyClass mapper output key class. + @param outputValueClass mapper output value class. + @param byValue indicates if key/values should be passed by value + to the next Mapper in the chain, if any. + @param mapperConf a JobConf with the configuration for the Mapper + class. It is recommended to use a JobConf without default values using the + JobConf(boolean loadDefaults) constructor with FALSE.]]> + + + + + + + If this method is overriden super.configure(...) should be + invoked at the beginning of the overwriter method.]]> + + + + + + + + + + reduce(...) method of the Reducer with the + map(...) methods of the Mappers in the chain.]]> + + + + + + + If this method is overriden super.close() should be + invoked at the end of the overwriter method.]]> + + + + + For each record output by the Reducer, the Mapper classes are invoked in a + chained (or piped) fashion, the output of the first becomes the input of the + second, and so on until the last Mapper, the output of the last Mapper will + be written to the task's output. +

    + The key functionality of this feature is that the Mappers in the chain do not + need to be aware that they are executed after the Reducer or in a chain. + This enables having reusable specialized Mappers that can be combined to + perform composite operations within a single task. +

    + Special care has to be taken when creating chains that the key/values output + by a Mapper are valid for the following Mapper in the chain. It is assumed + all Mappers and the Reduce in the chain use maching output and input key and + value classes as no conversion is done by the chaining code. +

    + Using the ChainMapper and the ChainReducer classes is possible to compose + Map/Reduce jobs that look like [MAP+ / REDUCE MAP*]. And + immediate benefit of this pattern is a dramatic reduction in disk IO. +

    + IMPORTANT: There is no need to specify the output key/value classes for the + ChainReducer, this is done by the setReducer or the addMapper for the last + element in the chain. +

    + ChainReducer usage pattern: +

    + 

    + ...
+ conf.setJobName("chain");
+ conf.setInputFormat(TextInputFormat.class);
+ conf.setOutputFormat(TextOutputFormat.class);
+
+ JobConf mapAConf = new JobConf(false);
+ ...
+ ChainMapper.addMapper(conf, AMap.class, LongWritable.class, Text.class,
+   Text.class, Text.class, true, mapAConf);
+
+ JobConf mapBConf = new JobConf(false);
+ ...
+ ChainMapper.addMapper(conf, BMap.class, Text.class, Text.class,
+   LongWritable.class, Text.class, false, mapBConf);
+
+ JobConf reduceConf = new JobConf(false);
+ ...
+ ChainReducer.setReducer(conf, XReduce.class, LongWritable.class, Text.class,
+   Text.class, Text.class, true, reduceConf);
+
+ ChainReducer.addMapper(conf, CMap.class, Text.class, Text.class,
+   LongWritable.class, Text.class, false, null);
+
+ ChainReducer.addMapper(conf, DMap.class, LongWritable.class, Text.class,
+   LongWritable.class, LongWritable.class, true, null);
+
+ FileInputFormat.setInputPaths(conf, inDir);
+ FileOutputFormat.setOutputPath(conf, outDir);
+ ...
+
+ JobClient jc = new JobClient(conf);
+ RunningJob job = jc.submitJob(conf);
+ ...
+
    ]]> +     +     + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + RecordReader's for CombineFileSplit's. + @see CombineFileSplit]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + CombineFileRecordReader. + + Subclassing is needed to get a concrete record reader wrapper because of the + constructor requirement. + + @see CombineFileRecordReader + @see CombineFileInputFormat]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + CombineFileInputFormat-equivalent for + SequenceFileInputFormat. + + @see CombineFileInputFormat]]> + + + + + + + + + + + + + + + CombineFileInputFormat-equivalent for + TextInputFormat. + + @see CombineFileInputFormat]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + true if the name output is multi, false + if it is single. If the name output is not defined it returns + false]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + By default these counters are disabled. +

    + MultipleOutputs supports counters, by default the are disabled. + The counters group is the {@link MultipleOutputs} class name. +

    + The names of the counters are the same as the named outputs. For multi + named outputs the name of the counter is the concatenation of the named + output, and underscore '_' and the multiname. + + @param conf job conf to enableadd the named output. + @param enabled indicates if the counters will be enabled or not.]]> +     +     + + + + + By default these counters are disabled. +

    + MultipleOutputs supports counters, by default the are disabled. + The counters group is the {@link MultipleOutputs} class name. +

    + The names of the counters are the same as the named outputs. For multi + named outputs the name of the counter is the concatenation of the named + output, and underscore '_' and the multiname. + + + @param conf job conf to enableadd the named output. + @return TRUE if the counters are enabled, FALSE if they are disabled.]]> +     +     + + + + + + + + + + + + + + + + + + + + + + + + + + + If overriden subclasses must invoke super.close() at the + end of their close() + + @throws java.io.IOException thrown if any of the MultipleOutput files + could not be closed properly.]]> + + + + OutputCollector passed to + the map() and reduce() methods of the + Mapper and Reducer implementations. +

    + Each additional output, or named output, may be configured with its own + OutputFormat, with its own key class and with its own value + class. +

    + A named output can be a single file or a multi file. The later is referred as + a multi named output. +

    + A multi named output is an unbound set of files all sharing the same + OutputFormat, key class and value class configuration. +

    + When named outputs are used within a Mapper implementation, + key/values written to a name output are not part of the reduce phase, only + key/values written to the job OutputCollector are part of the + reduce phase. +

    + MultipleOutputs supports counters, by default the are disabled. The counters + group is the {@link MultipleOutputs} class name. +

    + The names of the counters are the same as the named outputs. For multi + named outputs the name of the counter is the concatenation of the named + output, and underscore '_' and the multiname. +

    + Job configuration usage pattern is: + 

    +
+ JobConf conf = new JobConf();
+
+ conf.setInputPath(inDir);
+ FileOutputFormat.setOutputPath(conf, outDir);
+
+ conf.setMapperClass(MOMap.class);
+ conf.setReducerClass(MOReduce.class);
+ ...
+
+ // Defines additional single text based output 'text' for the job
+ MultipleOutputs.addNamedOutput(conf, "text", TextOutputFormat.class,
+ LongWritable.class, Text.class);
+
+ // Defines additional multi sequencefile based output 'sequence' for the
+ // job
+ MultipleOutputs.addMultiNamedOutput(conf, "seq",
+   SequenceFileOutputFormat.class,
+   LongWritable.class, Text.class);
+ ...
+
+ JobClient jc = new JobClient();
+ RunningJob job = jc.submitJob(conf);
+
+ ...
+
    +

    + Job configuration usage pattern is: + 

    +
+ public class MOReduce implements
+   Reducer<WritableComparable, Writable> {
+ private MultipleOutputs mos;
+
+ public void configure(JobConf conf) {
+ ...
+ mos = new MultipleOutputs(conf);
+ }
+
+ public void reduce(WritableComparable key, Iterator<Writable> values,
+ OutputCollector output, Reporter reporter)
+ throws IOException {
+ ...
+ mos.getCollector("text", reporter).collect(key, new Text("Hello"));
+ mos.getCollector("seq", "A", reporter).collect(key, new Text("Bye"));
+ mos.getCollector("seq", "B", reporter).collect(key, new Text("Chau"));
+ ...
+ }
+
+ public void close() throws IOException {
+ mos.close();
+ ...
+ }
+
+ }
+
    ]]> +     +     + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + It can be used instead of the default implementation, + of {@link org.apache.hadoop.mapred.MapRunner}, when the Map + operation is not CPU bound in order to improve throughput. +

    + Map implementations using this MapRunnable must be thread-safe. +

    + The Map-Reduce job has to be configured to use this MapRunnable class (using + the JobConf.setMapRunnerClass method) and + the number of threads the thread-pool can use with the + mapred.map.multithreadedrunner.threads property, its default + value is 10 threads. +

    ]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + R reduces, there are R-1 + keys in the SequenceFile. + @deprecated Use + {@link #setPartitionFile(Configuration, Path)} + instead]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Cluster. + @throws IOException]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + ClusterMetrics provides clients with information such as: +

      +
    1. + Size of the cluster. +
      2. +
    2. + Number of blacklisted and decommissioned trackers. +
      3. +
    3. + Slot capacity of the cluster. +
      4. +
    4. + The number of currently occupied/reserved map and reduce slots. +
      5. +
    5. + The number of currently running map and reduce tasks. +
      6. +
    6. + The number of job submissions. +
      7. +
    + +

    Clients can query for the latest ClusterMetrics, via + {@link Cluster#getClusterStatus()}.

    + + @see Cluster]]> +     +     + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Counters represent global counters, defined either by the + Map-Reduce framework or applications. Each Counter is named by + an {@link Enum} and has a long for the value.

    + +

    Counters are bunched into Groups, each comprising of + counters from a particular Enum class.]]> + + + + + + + + + + + + + + + + + + + + + the type of counter + @param the type of counter group + @param counters the old counters object]]> + + + + Counters holds per job/task counters, defined either by the + Map-Reduce framework or applications. Each Counter can be of + any {@link Enum} type.

    + +

    Counters are bunched into {@link CounterGroup}s, each + comprising of counters from a particular Enum class.]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Each {@link InputSplit} is then assigned to an individual {@link Mapper} + for processing.

    + +

    Note: The split is a logical split of the inputs and the + input files are not physically split into chunks. For e.g. a split could + be <input-file-path, start, offset> tuple. The InputFormat + also creates the {@link RecordReader} to read the {@link InputSplit}. + + @param context job configuration. + @return an array of {@link InputSplit}s for the job.]]> + + + + + + + + + + + + + InputFormat describes the input-specification for a + Map-Reduce job. + +

    The Map-Reduce framework relies on the InputFormat of the + job to:

    +

      +
    1. + Validate the input-specification of the job. +
    2. + Split-up the input file(s) into logical {@link InputSplit}s, each of + which is then assigned to an individual {@link Mapper}. +
      3. +
    3. + Provide the {@link RecordReader} implementation to be used to glean + input records from the logical InputSplit for processing by + the {@link Mapper}. +
      4. +
    + +

    The default behavior of file-based {@link InputFormat}s, typically + sub-classes of {@link FileInputFormat}, is to split the + input into logical {@link InputSplit}s based on the total size, in + bytes, of the input files. However, the {@link FileSystem} blocksize of + the input files is treated as an upper bound for input splits. A lower bound + on the split size can be set via + + mapreduce.input.fileinputformat.split.minsize.

    + +

    Clearly, logical splits based on input-size is insufficient for many + applications since record boundaries are to respected. In such cases, the + application has to also implement a {@link RecordReader} on whom lies the + responsibility to respect record-boundaries and present a record-oriented + view of the logical InputSplit to the individual task. + + @see InputSplit + @see RecordReader + @see FileInputFormat]]> + + + + + + + + + + + + + + + + + + + + + + + + + SplitLocationInfos describing how the split + data is stored at each location. A null value indicates that all the + locations have the data stored on disk. + @throws IOException]]> + + + + InputSplit represents the data to be processed by an + individual {@link Mapper}. + +

    Typically, it presents a byte-oriented view on the input and is the + responsibility of {@link RecordReader} of the job to process this and present + a record-oriented view. + + @see InputFormat + @see RecordReader]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Job makes a copy of the Configuration so + that any necessary internal modifications do not reflect on the incoming + parameter. + + A Cluster will be created from the conf parameter only when it's needed. + + @param conf the configuration + @return the {@link Job} , with no connection to a cluster yet. + @throws IOException]]> + + + + + + + + Job makes a copy of the Configuration so + that any necessary internal modifications do not reflect on the incoming + parameter. + + @param conf the configuration + @return the {@link Job} , with no connection to a cluster yet. + @throws IOException]]> + + + + + + + + Job makes a copy of the Configuration so + that any necessary internal modifications do not reflect on the incoming + parameter. + + @param status job status + @param conf job configuration + @return the {@link Job} , with no connection to a cluster yet. + @throws IOException]]> + + + + + + + Job makes a copy of the Configuration so + that any necessary internal modifications do not reflect on the incoming + parameter. + + @param ignored + @return the {@link Job} , with no connection to a cluster yet. + @throws IOException + @deprecated Use {@link #getInstance()}]]> + + + + + + + + Job makes a copy of the Configuration so + that any necessary internal modifications do not reflect on the incoming + parameter. + + @param ignored + @param conf job configuration + @return the {@link Job} , with no connection to a cluster yet. + @throws IOException + @deprecated Use {@link #getInstance(Configuration)}]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + progress of the job's map-tasks, as a float between 0.0 + and 1.0. When all map tasks have completed, the function returns 1.0. + + @return the progress of the job's map-tasks. + @throws IOException]]> + + + + + + progress of the job's reduce-tasks, as a float between 0.0 + and 1.0. When all reduce tasks have completed, the function returns 1.0. + + @return the progress of the job's reduce-tasks. + @throws IOException]]> + + + + + + + progress of the job's cleanup-tasks, as a float between 0.0 + and 1.0. When all cleanup tasks have completed, the function returns 1.0. + + @return the progress of the job's cleanup-tasks. + @throws IOException]]> + + + + + + progress of the job's setup-tasks, as a float between 0.0 + and 1.0. When all setup tasks have completed, the function returns 1.0. + + @return the progress of the job's setup-tasks. + @throws IOException]]> + + + + + + true if the job is complete, else false. + @throws IOException]]> + + + + + + true if the job succeeded, else false. + @throws IOException]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + InputFormat to use + @throws IllegalStateException if the job is submitted]]> + + + + + + + OutputFormat to use + @throws IllegalStateException if the job is submitted]]> + + + + + + + Mapper to use + @throws IllegalStateException if the job is submitted]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Reducer to use + @throws IllegalStateException if the job is submitted]]> + + + + + + + Partitioner to use + @throws IllegalStateException if the job is submitted]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + true if speculative execution + should be turned on, else false.]]> + + + + + + true if speculative execution + should be turned on for map tasks, + else false.]]> + + + + + + true if speculative execution + should be turned on for reduce tasks, + else false.]]> + + + + + + true, job-setup and job-cleanup will be + considered from {@link OutputCommitter} + else ignored.]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + JobTracker is lost]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Job. + @throws IOException if fail to close.]]> + + + + + + + + + + + + + + + + + + + + + + + + + + It allows the user to configure the + job, submit it, control its execution, and query the state. The set methods + only work until the job is submitted, afterwards they will throw an + IllegalStateException.

    + +

    + Normally the user creates the application, describes various facets of the + job via {@link Job} and then submits the job and monitor its progress.

    + +

    Here is an example on how to submit a job:

    + 

    +     // Create a new Job
+     Job job = Job.getInstance();
+     job.setJarByClass(MyJob.class);
+     
+     // Specify various job-specific parameters     
+     job.setJobName("myjob");
+     
+     job.setInputPath(new Path("in"));
+     job.setOutputPath(new Path("out"));
+     
+     job.setMapperClass(MyJob.MyMapper.class);
+     job.setReducerClass(MyJob.MyReducer.class);
+
+     // Submit the job, then poll for progress until the job is complete
+     job.waitForCompletion(true);
+
    ]]> +     + + + + + + + + + + + + + + + + + + + + + + + 1. + @return the number of reduce tasks for this job.]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + mapred.map.max.attempts + property. If this property is not already set, the default is 4 attempts. + + @return the max number of attempts per map task.]]> + + + + + mapred.reduce.max.attempts + property. If this property is not already set, the default is 4 attempts. + + @return the max number of attempts per reduce task.]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + An example JobID is : + job_200707121733_0003 , which represents the third job + running at the jobtracker started at 200707121733. +

    + Applications should never construct or parse JobID strings, but rather + use appropriate constructors or {@link #forName(String)} method. + + @see TaskID + @see TaskAttemptID]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + the key input type to the Mapper + @param the value input type to the Mapper + @param the key output type from the Mapper + @param the value output type from the Mapper]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Maps are the individual tasks which transform input records into a + intermediate records. The transformed intermediate records need not be of + the same type as the input records. A given input pair may map to zero or + many output pairs.

    + +

    The Hadoop Map-Reduce framework spawns one map task for each + {@link InputSplit} generated by the {@link InputFormat} for the job. + Mapper implementations can access the {@link Configuration} for + the job via the {@link JobContext#getConfiguration()}. + +

    The framework first calls + {@link #setup(org.apache.hadoop.mapreduce.Mapper.Context)}, followed by + {@link #map(Object, Object, org.apache.hadoop.mapreduce.Mapper.Context)} + for each key/value pair in the InputSplit. Finally + {@link #cleanup(org.apache.hadoop.mapreduce.Mapper.Context)} is called.

    + +

    All intermediate values associated with a given output key are + subsequently grouped by the framework, and passed to a {@link Reducer} to + determine the final output. Users can control the sorting and grouping by + specifying two key {@link RawComparator} classes.

    + +

    The Mapper outputs are partitioned per + Reducer. Users can control which keys (and hence records) go to + which Reducer by implementing a custom {@link Partitioner}. + +

    Users can optionally specify a combiner, via + {@link Job#setCombinerClass(Class)}, to perform local aggregation of the + intermediate outputs, which helps to cut down the amount of data transferred + from the Mapper to the Reducer. + +

    Applications can specify if and how the intermediate + outputs are to be compressed and which {@link CompressionCodec}s are to be + used via the Configuration.

    + +

    If the job has zero + reduces then the output of the Mapper is directly written + to the {@link OutputFormat} without sorting by keys.

    + +

    Example:

    + 

    + public class TokenCounterMapper 
+     extends Mapper<Object, Text, Text, IntWritable>{
+    
+   private final static IntWritable one = new IntWritable(1);
+   private Text word = new Text();
+   
+   public void map(Object key, Text value, Context context) throws IOException, InterruptedException {
+     StringTokenizer itr = new StringTokenizer(value.toString());
+     while (itr.hasMoreTokens()) {
+       word.set(itr.nextToken());
+       context.write(word, one);
+     }
+   }
+ }
+
    + +

    Applications may override the + {@link #run(org.apache.hadoop.mapreduce.Mapper.Context)} method to exert + greater control on map processing e.g. multi-threaded Mappers + etc.

    + + @see InputFormat + @see JobContext + @see Partitioner + @see Reducer]]> +     +     + + + + + + + + + + + + + + + + + + + + + + + + + + MarkableIterator is a wrapper iterator class that + implements the {@link MarkableIteratorInterface}.]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + true if task output recovery is supported, + false otherwise + @see #recoverTask(TaskAttemptContext) + @deprecated Use {@link #isRecoverySupported(JobContext)} instead.]]> + + + + + + + true repeatable job commit is supported, + false otherwise + @throws IOException]]> + + + + + + + true if task output recovery is supported, + false otherwise + @throws IOException + @see #recoverTask(TaskAttemptContext)]]> + + + + + + + OutputCommitter. This is called from the application master + process, but it is called individually for each task. + + If an exception is thrown the task will be attempted again. + + This may be called multiple times for the same task. But from different + application attempts. + + @param taskContext Context of the task whose output is being recovered + @throws IOException]]> + + + + OutputCommitter describes the commit of task output for a + Map-Reduce job. + +

    The Map-Reduce framework relies on the OutputCommitter of + the job to:

    +

      +
    1. + Setup the job during initialization. For example, create the temporary + output directory for the job during the initialization of the job. +
      2. +
    2. + Cleanup the job after the job completion. For example, remove the + temporary output directory after the job completion. +
      3. +
    3. + Setup the task temporary output. +
      4. +
    4. + Check whether a task needs a commit. This is to avoid the commit + procedure if a task does not need commit. +
      5. +
    5. + Commit of the task output. +
      6. +
    6. + Discard the task commit. +
      7. +
    + The methods in this class can be called from several different processes and + from several different contexts. It is important to know which process and + which context each is called from. Each method should be marked accordingly + in its documentation. It is also important to note that not all methods are + guaranteed to be called once and only once. If a method is not guaranteed to + have this property the output committer needs to handle this appropriately. + Also note it will only be in rare situations where they may be called + multiple times for the same task. + + @see org.apache.hadoop.mapreduce.lib.output.FileOutputCommitter + @see JobContext + @see TaskAttemptContext]]> +     +     + + + + + + + + + + + + + + + + + + + This is to validate the output specification for the job when it is + a job is submitted. Typically checks that it does not already exist, + throwing an exception when it already exists, so that output is not + overwritten.

    + + @param context information about the job + @throws IOException when output should not be attempted]]> +     +     + + + + + + + + + + OutputFormat describes the output-specification for a + Map-Reduce job. + +

    The Map-Reduce framework relies on the OutputFormat of the + job to:

    +

      +
    1. + Validate the output-specification of the job. For e.g. check that the + output directory doesn't already exist. +
    2. + Provide the {@link RecordWriter} implementation to be used to write out + the output files of the job. Output files are stored in a + {@link FileSystem}. +
      3. +
    + + @see RecordWriter]]> +     +     + + + + + + + + + + + Typically a hash function on a all or a subset of the key.

    + + @param key the key to be partioned. + @param value the entry value. + @param numPartitions the total number of partitions. + @return the partition number for the key.]]> +     +     + + Partitioner controls the partitioning of the keys of the + intermediate map-outputs. The key (or a subset of the key) is used to derive + the partition, typically by a hash function. The total number of partitions + is the same as the number of reduce tasks for the job. Hence this controls + which of the m reduce tasks the intermediate key (and hence the + record) is sent for reduction.

    + +

    Note: A Partitioner is created only when there are multiple + reducers.

    + +

    Note: If you require your Partitioner class to obtain the Job's + configuration object, implement the {@link Configurable} interface.

    + + @see Reducer]]> +     +     + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + "N/A" + + @return Scheduling information associated to particular Job Queue]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + @param ]]> + + + + + + + + + + + + + + + + + + + + + + RecordWriter to future operations. + + @param context the context of the task + @throws IOException]]> + + + + RecordWriter writes the output <key, value> pairs + to an output file. + +

    RecordWriter implementations write the job outputs to the + {@link FileSystem}. + + @see OutputFormat]]> + + + + + + + + + + + + + + + + + + + + + + the class of the input keys + @param the class of the input values + @param the class of the output keys + @param the class of the output values]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Reducer implementations + can access the {@link Configuration} for the job via the + {@link JobContext#getConfiguration()} method.

    + +

    Reducer has 3 primary phases:

    +
      +
    1. + + Shuffle + +

      The Reducer copies the sorted output from each + {@link Mapper} using HTTP across the network.

      +
      2. + +
    2. + Sort + +

      The framework merge sorts Reducer inputs by + keys + (since different Mappers may have output the same key).

      + +

      The shuffle and sort phases occur simultaneously i.e. while outputs are + being fetched they are merged.

      + + SecondarySort + +

      To achieve a secondary sort on the values returned by the value + iterator, the application should extend the key with the secondary + key and define a grouping comparator. The keys will be sorted using the + entire key, but will be grouped using the grouping comparator to decide + which keys and values are sent in the same call to reduce.The grouping + comparator is specified via + {@link Job#setGroupingComparatorClass(Class)}. The sort order is + controlled by + {@link Job#setSortComparatorClass(Class)}.

      + + + For example, say that you want to find duplicate web pages and tag them + all with the url of the "best" known example. You would set up the job + like: +
        +
      • Map Input Key: url
        • +
      • Map Input Value: document
        • +
      • Map Output Key: document checksum, url pagerank
        • +
      • Map Output Value: url
        • +
      • Partitioner: by checksum
        • +
      • OutputKeyComparator: by checksum and then decreasing pagerank
        • +
      • OutputValueGroupingComparator: by checksum
        • +
      +
      3. + +
    3. + Reduce + +

      In this phase the + {@link #reduce(Object, Iterable, org.apache.hadoop.mapreduce.Reducer.Context)} + method is called for each <key, (collection of values)> in + the sorted inputs.

      +

      The output of the reduce task is typically written to a + {@link RecordWriter} via + {@link Context#write(Object, Object)}.

      +
      4. +
    + +

    The output of the Reducer is not re-sorted.

    + +

    Example:

    + 

    + public class IntSumReducer<Key> extends Reducer<Key,IntWritable,
+                                                 Key,IntWritable> {
+   private IntWritable result = new IntWritable();
+ 
+   public void reduce(Key key, Iterable<IntWritable> values,
+                      Context context) throws IOException, InterruptedException {
+     int sum = 0;
+     for (IntWritable val : values) {
+       sum += val.get();
+     }
+     result.set(sum);
+     context.write(key, result);
+   }
+ }
+
    + + @see Mapper + @see Partitioner]]> +     +     + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + counterName. + @param counterName counter name + @return the Counter for the given counterName]]> + + + + + + + groupName and + counterName. + @param counterName counter name + @return the Counter for the given groupName and + counterName]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + An example TaskAttemptID is : + attempt_200707121733_0003_m_000005_0 , which represents the + zeroth task attempt for the fifth map task in the third job + running at the jobtracker started at 200707121733. +

    + Applications should never construct or parse TaskAttemptID strings + , but rather use appropriate constructors or {@link #forName(String)} + method. + + @see JobID + @see TaskID]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + An example TaskID is : + task_200707121733_0003_m_000005 , which represents the + fifth map task in the third job running at the jobtracker + started at 200707121733. +

    + Applications should never construct or parse TaskID strings + , but rather use appropriate constructors or {@link #forName(String)} + method. + + @see JobID + @see TaskAttemptID]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + OutputCommitter for the task-attempt]]> + + + + the input key type for the task + @param the input value type for the task + @param the output key type for the task + @param the output value type for the task]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + type of the other counter + @param type of the other counter group + @param counters the counters object to copy + @param groupFactory the factory for new groups]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + type of counter inside the counters + @param type of group inside the counters]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + type of the counter for the group]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + The key and values are passed from one element of the chain to the next, by + value. For the added Mapper the configuration given for it, + mapperConf, have precedence over the job's Configuration. This + precedence is in effect when the task is running. +

    +

    + IMPORTANT: There is no need to specify the output key/value classes for the + ChainMapper, this is done by the addMapper for the last mapper in the chain +

    + + @param job + The job. + @param klass + the Mapper class to add. + @param inputKeyClass + mapper input key class. + @param inputValueClass + mapper input value class. + @param outputKeyClass + mapper output key class. + @param outputValueClass + mapper output value class. + @param mapperConf + a configuration for the Mapper class. It is recommended to use a + Configuration without default values using the + Configuration(boolean loadDefaults) constructor with + FALSE.]]> +     + + + + + + + + + + + + The Mapper classes are invoked in a chained (or piped) fashion, the output of + the first becomes the input of the second, and so on until the last Mapper, + the output of the last Mapper will be written to the task's output. +

    +

    + The key functionality of this feature is that the Mappers in the chain do not + need to be aware that they are executed in a chain. This enables having + reusable specialized Mappers that can be combined to perform composite + operations within a single task. +

    +

    + Special care has to be taken when creating chains that the key/values output + by a Mapper are valid for the following Mapper in the chain. It is assumed + all Mappers and the Reduce in the chain use matching output and input key and + value classes as no conversion is done by the chaining code. +

    +

    + Using the ChainMapper and the ChainReducer classes is possible to compose + Map/Reduce jobs that look like [MAP+ / REDUCE MAP*]. And + immediate benefit of this pattern is a dramatic reduction in disk IO. +

    +

    + IMPORTANT: There is no need to specify the output key/value classes for the + ChainMapper, this is done by the addMapper for the last mapper in the chain. +

    + ChainMapper usage pattern: +

    + + 

    + ...
+ Job = new Job(conf);
+
+ Configuration mapAConf = new Configuration(false);
+ ...
+ ChainMapper.addMapper(job, AMap.class, LongWritable.class, Text.class,
+   Text.class, Text.class, true, mapAConf);
+
+ Configuration mapBConf = new Configuration(false);
+ ...
+ ChainMapper.addMapper(job, BMap.class, Text.class, Text.class,
+   LongWritable.class, Text.class, false, mapBConf);
+
+ ...
+
+ job.waitForComplettion(true);
+ ...
+
    ]]> +     +     + + + + + + + + + + + + + + + + The key and values are passed from one element of the chain to the next, by + value. For the added Reducer the configuration given for it, + reducerConf, have precedence over the job's Configuration. + This precedence is in effect when the task is running. +

    +

    + IMPORTANT: There is no need to specify the output key/value classes for the + ChainReducer, this is done by the setReducer or the addMapper for the last + element in the chain. +

    + + @param job + the job + @param klass + the Reducer class to add. + @param inputKeyClass + reducer input key class. + @param inputValueClass + reducer input value class. + @param outputKeyClass + reducer output key class. + @param outputValueClass + reducer output value class. + @param reducerConf + a configuration for the Reducer class. It is recommended to use a + Configuration without default values using the + Configuration(boolean loadDefaults) constructor with + FALSE.]]> +     +     + + + + + + + + + + + + The key and values are passed from one element of the chain to the next, by + value For the added Mapper the configuration given for it, + mapperConf, have precedence over the job's Configuration. This + precedence is in effect when the task is running. +

    +

    + IMPORTANT: There is no need to specify the output key/value classes for the + ChainMapper, this is done by the addMapper for the last mapper in the + chain. +

    + + @param job + The job. + @param klass + the Mapper class to add. + @param inputKeyClass + mapper input key class. + @param inputValueClass + mapper input value class. + @param outputKeyClass + mapper output key class. + @param outputValueClass + mapper output value class. + @param mapperConf + a configuration for the Mapper class. It is recommended to use a + Configuration without default values using the + Configuration(boolean loadDefaults) constructor with + FALSE.]]> +     +     + + + + + + + + + + + For each record output by the Reducer, the Mapper classes are invoked in a + chained (or piped) fashion. The output of the reducer becomes the input of + the first mapper and output of first becomes the input of the second, and so + on until the last Mapper, the output of the last Mapper will be written to + the task's output. +

    +

    + The key functionality of this feature is that the Mappers in the chain do not + need to be aware that they are executed after the Reducer or in a chain. This + enables having reusable specialized Mappers that can be combined to perform + composite operations within a single task. +

    +

    + Special care has to be taken when creating chains that the key/values output + by a Mapper are valid for the following Mapper in the chain. It is assumed + all Mappers and the Reduce in the chain use matching output and input key and + value classes as no conversion is done by the chaining code. +

    +

    Using the ChainMapper and the ChainReducer classes is possible to + compose Map/Reduce jobs that look like [MAP+ / REDUCE MAP*]. And + immediate benefit of this pattern is a dramatic reduction in disk IO.

    +

    + IMPORTANT: There is no need to specify the output key/value classes for the + ChainReducer, this is done by the setReducer or the addMapper for the last + element in the chain. +

    + ChainReducer usage pattern: +

    + + 

    + ...
+ Job = new Job(conf);
+ ....
+
+ Configuration reduceConf = new Configuration(false);
+ ...
+ ChainReducer.setReducer(job, XReduce.class, LongWritable.class, Text.class,
+   Text.class, Text.class, true, reduceConf);
+
+ ChainReducer.addMapper(job, CMap.class, Text.class, Text.class,
+   LongWritable.class, Text.class, false, null);
+
+ ChainReducer.addMapper(job, DMap.class, LongWritable.class, Text.class,
+   LongWritable.class, LongWritable.class, true, null);
+
+ ...
+
+ job.waitForCompletion(true);
+ ...
+
    ]]> +     +     + +     + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + DBInputFormat emits LongWritables containing the record number as + key and DBWritables as value. + + The SQL query, and input class can be using one of the two + setInput methods.]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + {@link DBOutputFormat} accepts <key,value> pairs, where + key has a type extending DBWritable. Returned {@link RecordWriter} + writes only the key to the database with a batch SQL query.]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + DBWritable. DBWritable, is similar to {@link Writable} + except that the {@link #write(PreparedStatement)} method takes a + {@link PreparedStatement}, and {@link #readFields(ResultSet)} + takes a {@link ResultSet}. +

    + Implementations are responsible for writing the fields of the object + to PreparedStatement, and reading the fields of the object from the + ResultSet. + +

    Example:

    + If we have the following table in the database : + 
    + CREATE TABLE MyTable (
+   counter        INTEGER NOT NULL,
+   timestamp      BIGINT  NOT NULL,
+ );
+
    + then we can read/write the tuples from/to the table with : + 

    + public class MyWritable implements Writable, DBWritable {
+   // Some data     
+   private int counter;
+   private long timestamp;
+       
+   //Writable#write() implementation
+   public void write(DataOutput out) throws IOException {
+     out.writeInt(counter);
+     out.writeLong(timestamp);
+   }
+       
+   //Writable#readFields() implementation
+   public void readFields(DataInput in) throws IOException {
+     counter = in.readInt();
+     timestamp = in.readLong();
+   }
+       
+   public void write(PreparedStatement statement) throws SQLException {
+     statement.setInt(1, counter);
+     statement.setLong(2, timestamp);
+   }
+       
+   public void readFields(ResultSet resultSet) throws SQLException {
+     counter = resultSet.getInt(1);
+     timestamp = resultSet.getLong(2);
+   } 
+ }
+
    ]]> +     +     + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +     + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + RecordReader's for + CombineFileSplit's. + + @see CombineFileSplit]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + CombineFileRecordReader. + + Subclassing is needed to get a concrete record reader wrapper because of the + constructor requirement. + + @see CombineFileRecordReader + @see CombineFileInputFormat]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + th Path]]> + + + + + + th Path]]> + + + + + + + + + + + th Path]]> + + + + + + + + + + + + + + + + + + + + + + + + + + CombineFileSplit can be used to implement {@link RecordReader}'s, + with reading one record per file. + + @see FileSplit + @see CombineFileInputFormat]]> + + + + + + + + + + + + + + CombineFileInputFormat-equivalent for + SequenceFileInputFormat. + + @see CombineFileInputFormat]]> + + + + + + + + + + + + + + CombineFileInputFormat-equivalent for + TextInputFormat. + + @see CombineFileInputFormat]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + FileInputFormat always returns + true. Implementations that may deal with non-splittable files must + override this method. + + FileInputFormat implementations can override this and return + false to ensure that individual input files are never split-up + so that {@link Mapper}s process entire files. + + @param context the job context + @param filename the file name to check + @return is this file splitable?]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + FileInputFormat is the base class for all file-based + InputFormats. This provides a generic implementation of + {@link #getSplits(JobContext)}. + + Implementations of FileInputFormat can also override the + {@link #isSplitable(JobContext, Path)} method to prevent input files + from being split-up in certain situations. Implementations that may + deal with non-splittable files must override this method, since + the default implementation assumes splitting is always possible.]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
    or + conf.setInt(FixedLengthInputFormat.FIXED_RECORD_LENGTH, recordLength); +

    + @see FixedLengthRecordReader]]> +     +     + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +     + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + true if the Job was added.]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + ([,]*) + func ::= tbl(,"") + class ::= @see java.lang.Class#forName(java.lang.String) + path ::= @see org.apache.hadoop.fs.Path#Path(java.lang.String) + } + Reads expression from the mapreduce.join.expr property and + user-supplied join types from mapreduce.join.define.<ident> + types. Paths supplied to tbl are given as input paths to the + InputFormat class listed. + @see #compose(java.lang.String, java.lang.Class, java.lang.String...)]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + ,

    ) }]]> + + + + + + + + (tbl(,),tbl(,),...,tbl(,)) }]]> + + + + + + + + (tbl(,),tbl(,),...,tbl(,)) }]]> + + + + + + + + mapreduce.join.define.<ident> to a classname. + In the expression mapreduce.join.expr, the identifier will be + assumed to be a ComposableRecordReader. + mapreduce.join.keycomparator can be a classname used to compare + keys in the join. + @see #setFormat + @see JoinRecordReader + @see MultiFilterRecordReader]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + ...... + }]]> + + + + + + + + + + + + + + + + + + + + + capacity children to position + id in the parent reader. + The id of a root CompositeRecordReader is -1 by convention, but relying + on this is not recommended.]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + override(S1,S2,S3) will prefer values + from S3 over S2, and values from S2 over S1 for all keys + emitted from all sources.]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + [<child1>,<child2>,...,<childn>]]]> + + + + + + + out. + TupleWritable format: + {@code + ...... + }]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + the map's input key type + @param the map's input value type + @param the map's output key type + @param the map's output value type + @param job the job + @return the mapper class to run]]> + + + + + + + the map input key type + @param the map input value type + @param the map output key type + @param the map output value type + @param job the job to modify + @param cls the class to use as the mapper]]> + + + + + + + + + + + + + + + + + It can be used instead of the default implementation, + {@link org.apache.hadoop.mapred.MapRunner}, when the Map operation is not CPU + bound in order to improve throughput. +

    + Mapper implementations using this MapRunnable must be thread-safe. +

    + The Map-Reduce job has to be configured with the mapper to use via + {@link #setMapperClass(Job, Class)} and + the number of thread the thread-pool can use with the + {@link #getNumberOfThreads(JobContext)} method. The default + value is 10 threads. +

    ]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + MapContext to be wrapped + @return a wrapped Mapper.Context for custom implementations]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +

  • + In applications which take a classname of committer in + a configuration option, set it to the canonical name of this class + (see {@link #NAME}). When this class is instantiated, it will + use the factory mechanism to locate the configured committer for the + destination. +
    • +
  • + In code, explicitly create an instance of this committer through + its constructor, then invoke commit lifecycle operations on it. + The dynamically configured committer will be created in the constructor + and have the lifecycle operations relayed to it. +
    • + ]]> +     +     + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + true if the job output should be compressed, + false otherwise]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Tasks' Side-Effect Files + +

    Some applications need to create/write-to side-files, which differ from + the actual job-outputs. + +

    In such cases there could be issues with 2 instances of the same TIP + (running simultaneously e.g. speculative tasks) trying to open/write-to the + same file (path) on HDFS. Hence the application-writer will have to pick + unique names per task-attempt (e.g. using the attemptid, say + attempt_200709221812_0001_m_000000_0), not just per TIP.

    + +

    To get around this the Map-Reduce framework helps the application-writer + out by maintaining a special + ${mapreduce.output.fileoutputformat.outputdir}/_temporary/_${taskid} + sub-directory for each task-attempt on HDFS where the output of the + task-attempt goes. On successful completion of the task-attempt the files + in the ${mapreduce.output.fileoutputformat.outputdir}/_temporary/_${taskid} (only) + are promoted to ${mapreduce.output.fileoutputformat.outputdir}. Of course, the + framework discards the sub-directory of unsuccessful task-attempts. This + is completely transparent to the application.

    + +

    The application-writer can take advantage of this by creating any + side-files required in a work directory during execution + of his task i.e. via + {@link #getWorkOutputPath(TaskInputOutputContext)}, and + the framework will move them out similarly - thus she doesn't have to pick + unique paths per task-attempt.

    + +

    The entire discussion holds true for maps of jobs with + reducer=NONE (i.e. 0 reduces) since output of the map, in that case, + goes directly to HDFS.

    + + @return the {@link Path} to the task's temporary output directory + for the map-reduce job.]]> +     +     + + + + + + + + The path can be used to create custom files from within the map and + reduce tasks. The path name will be unique for each task. The path parent + will be the job output directory.

    ls + +

    This method uses the {@link #getUniqueFile} method to make the file name + unique for the task.

    + + @param context the context for the task. + @param name the name for the file. + @param extension the extension for the file + @return a unique path accross all tasks of the job.]]> +     +     + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +     + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Warning: when the baseOutputPath is a path that resolves + outside of the final job output directory, the directory is created + immediately and then persists through subsequent task retries, breaking + the concept of output committing.]]> + + + + + + + + + + Warning: when the baseOutputPath is a path that resolves + outside of the final job output directory, the directory is created + immediately and then persists through subsequent task retries, breaking + the concept of output committing.]]> + + + + + + + super.close() at the + end of their close()]]> + + + + + Case one: writing to additional outputs other than the job default output. + + Each additional output, or named output, may be configured with its own + OutputFormat, with its own key class and with its own value + class. +

    + +

    + Case two: to write data to different files provided by user +

    + +

    + MultipleOutputs supports counters, by default they are disabled. The + counters group is the {@link MultipleOutputs} class name. The names of the + counters are the same as the output name. These count the number records + written to each output name. +

    + + Usage pattern for job submission: + 
    +
+ Job job = new Job();
+
+ FileInputFormat.setInputPath(job, inDir);
+ FileOutputFormat.setOutputPath(job, outDir);
+
+ job.setMapperClass(MOMap.class);
+ job.setReducerClass(MOReduce.class);
+ ...
+
+ // Defines additional single text based output 'text' for the job
+ MultipleOutputs.addNamedOutput(job, "text", TextOutputFormat.class,
+ LongWritable.class, Text.class);
+
+ // Defines additional sequence-file based output 'sequence' for the job
+ MultipleOutputs.addNamedOutput(job, "seq",
+   SequenceFileOutputFormat.class,
+   LongWritable.class, Text.class);
+ ...
+
+ job.waitForCompletion(true);
+ ...
+
    +

    + Usage in Reducer: + 

    + <K, V> String generateFileName(K k, V v) {
+   return k.toString() + "_" + v.toString();
+ }
+ 
+ public class MOReduce extends
+   Reducer<WritableComparable, Writable,WritableComparable, Writable> {
+ private MultipleOutputs mos;
+ public void setup(Context context) {
+ ...
+ mos = new MultipleOutputs(context);
+ }
+
+ public void reduce(WritableComparable key, Iterator<Writable> values,
+ Context context)
+ throws IOException {
+ ...
+ mos.write("text", , key, new Text("Hello"));
+ mos.write("seq", LongWritable(1), new Text("Bye"), "seq_a");
+ mos.write("seq", LongWritable(2), key, new Text("Chau"), "seq_b");
+ mos.write(key, new Text("value"), generateFileName(key, new Text("value")));
+ ...
+ }
+
+ public void cleanup(Context) throws IOException {
+ mos.close();
+ ...
+ }
+
+ }
+
    + +

    + When used in conjuction with org.apache.hadoop.mapreduce.lib.output.LazyOutputFormat, + MultipleOutputs can mimic the behaviour of MultipleTextOutputFormat and MultipleSequenceFileOutputFormat + from the old Hadoop API - ie, output can be written from the Reducer to more than one location. +

    + +

    + Use MultipleOutputs.write(KEYOUT key, VALUEOUT value, String baseOutputPath) to write key and + value to a path specified by baseOutputPath, with no need to specify a named output. + Warning: when the baseOutputPath passed to MultipleOutputs.write + is a path that resolves outside of the final job output directory, the + directory is created immediately and then persists through subsequent + task retries, breaking the concept of output committing: +

    + + 
    + private MultipleOutputs<Text, Text> out;
+ 
+ public void setup(Context context) {
+   out = new MultipleOutputs<Text, Text>(context);
+   ...
+ }
+ 
+ public void reduce(Text key, Iterable<Text> values, Context context) throws IOException, InterruptedException {
+ for (Text t : values) {
+   out.write(key, t, generateFileName(<parameter list...>));
+   }
+ }
+ 
+ protected void cleanup(Context context) throws IOException, InterruptedException {
+   out.close();
+ }
+
    + +

    + Use your own code in generateFileName() to create a custom path to your results. + '/' characters in baseOutputPath will be translated into directory levels in your file system. + Also, append your custom-generated path with "part" or similar, otherwise your output will be -00000, -00001 etc. + No call to context.write() is necessary. See example generateFileName() code below. +

    + + 
    + private String generateFileName(Text k) {
+   // expect Text k in format "Surname|Forename"
+   String[] kStr = k.toString().split("\\|");
+   
+   String sName = kStr[0];
+   String fName = kStr[1];
+
+   // example for k = Smith|John
+   // output written to /user/hadoop/path/to/output/Smith/John-r-00000 (etc)
+   return sName + "/" + fName;
+ }
+
    + +

    + Using MultipleOutputs in this way will still create zero-sized default output, eg part-00000. + To prevent this use LazyOutputFormat.setOutputFormatClass(job, TextOutputFormat.class); + instead of job.setOutputFormatClass(TextOutputFormat.class); in your Hadoop job configuration. +

    ]]> +     +     + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + This allows the user to specify the key class to be different + from the actual class ({@link BytesWritable}) used for writing

    + + @param job the {@link Job} to modify + @param theClass the SequenceFile output key class.]]> +     +     + + + + + This allows the user to specify the value class to be different + from the actual class ({@link BytesWritable}) used for writing

    + + @param job the {@link Job} to modify + @param theClass the SequenceFile output key class.]]> +     +     + + + + + + + + + + + + + + + + + + + + + + + + + + + +     + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +     + + + + + + + + + + + + bytes[left:(right+1)] in Python syntax. + + @param conf configuration object + @param left left Python-style offset + @param right right Python-style offset]]> + + + + + + + bytes[offset:] in Python syntax. + + @param conf configuration object + @param offset left Python-style offset]]> + + + + + + + bytes[:(offset+1)] in Python syntax. + + @param conf configuration object + @param offset right Python-style offset]]> + + + + + + + + + + + + + + + + + + + + + Partition {@link BinaryComparable} keys using a configurable part of + the bytes array returned by {@link BinaryComparable#getBytes()}.

    + +

    The subarray to be used for the partitioning can be defined by means + of the following properties: +

      +
    • + mapreduce.partition.binarypartitioner.left.offset: + left offset in array (0 by default) +
      • +
    • + mapreduce.partition.binarypartitioner.right.offset: + right offset in array (-1 by default) +
      • +
    + Like in Python, both negative and positive offsets are allowed, but + the meaning is slightly different. In case of an array of length 5, + for instance, the possible offsets are: + 
    
+  +---+---+---+---+---+
+  | B | B | B | B | B |
+  +---+---+---+---+---+
+    0   1   2   3   4
+   -5  -4  -3  -2  -1
+
    + The first row of numbers gives the position of the offsets 0...5 in + the array; the second row gives the corresponding negative offsets. + Contrary to Python, the specified subarray has byte i + and j as first and last element, repectively, when + i and j are the left and right offset. + +

    For Hadoop programs written in Java, it is advisable to use one of + the following static convenience methods for setting the offsets: +

      +
    • {@link #setOffsets}
      • +
    • {@link #setLeftOffset}
      • +
    • {@link #setRightOffset}
      • +
    ]]> +     +     + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + total.order.partitioner.natural.order is not false, a trie + of the first total.order.partitioner.max.trie.depth(2) + 1 bytes + will be built. Otherwise, keys will be located using a binary search of + the partition keyset using the {@link org.apache.hadoop.io.RawComparator} + defined for this job. The input file must be sorted with the same + comparator and contain {@link Job#getNumReduceTasks()} - 1 keys.]]> + + + + + + + + + + + + + + R reduces, there are R-1 + keys in the SequenceFile.]]> + + + + + + + + + + + + + + + + + + + + + +     + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + ReduceContext to be wrapped + @return a wrapped Reducer.Context for custom implementations]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +     diff --git a/hadoop-mapreduce-project/dev-support/jdiff/Apache_Hadoop_MapReduce_Core_3.3.4.xml b/hadoop-mapreduce-project/dev-support/jdiff/Apache_Hadoop_MapReduce_Core_3.3.4.xml new file mode 100644 index 0000000000000..dc0a88deac948 --- /dev/null +++ b/hadoop-mapreduce-project/dev-support/jdiff/Apache_Hadoop_MapReduce_Core_3.3.4.xml @@ -0,0 +1,28087 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + FileStatus of a given cache file on hdfs + @throws IOException]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + DistributedCache is a facility provided by the Map-Reduce + framework to cache files (text, archives, jars etc.) needed by applications. +

    + +

    Applications specify the files, via urls (hdfs:// or http://) to be cached + via the {@link org.apache.hadoop.mapred.JobConf}. The + DistributedCache assumes that the files specified via urls are + already present on the {@link FileSystem} at the path specified by the url + and are accessible by every machine in the cluster.

    + +

    The framework will copy the necessary files on to the worker node before + any tasks for the job are executed on that node. Its efficiency stems from + the fact that the files are only copied once per job and the ability to + cache archives which are un-archived on the workers.

    + +

    DistributedCache can be used to distribute simple, read-only + data/text files and/or more complex types such as archives, jars etc. + Archives (zip, tar and tgz/tar.gz files) are un-archived at the worker nodes. + Jars may be optionally added to the classpath of the tasks, a rudimentary + software distribution mechanism. Files have execution permissions. + In older version of Hadoop Map/Reduce users could optionally ask for symlinks + to be created in the working directory of the child task. In the current + version symlinks are always created. If the URL does not have a fragment + the name of the file or directory will be used. If multiple files or + directories map to the same link name, the last one added, will be used. All + others will not even be downloaded.

    + +

    DistributedCache tracks modification timestamps of the cache + files. Clearly the cache files should not be modified by the application + or externally while the job is executing.

    + +

    Here is an illustrative example on how to use the + DistributedCache:

    + 

    +     // Setting up the cache for the application
+
+     1. Copy the requisite files to the FileSystem:
+
+     $ bin/hadoop fs -copyFromLocal lookup.dat /myapp/lookup.dat
+     $ bin/hadoop fs -copyFromLocal map.zip /myapp/map.zip
+     $ bin/hadoop fs -copyFromLocal mylib.jar /myapp/mylib.jar
+     $ bin/hadoop fs -copyFromLocal mytar.tar /myapp/mytar.tar
+     $ bin/hadoop fs -copyFromLocal mytgz.tgz /myapp/mytgz.tgz
+     $ bin/hadoop fs -copyFromLocal mytargz.tar.gz /myapp/mytargz.tar.gz
+
+     2. Setup the application's JobConf:
+
+     JobConf job = new JobConf();
+     DistributedCache.addCacheFile(new URI("/myapp/lookup.dat#lookup.dat"),
+                                   job);
+     DistributedCache.addCacheArchive(new URI("/myapp/map.zip"), job);
+     DistributedCache.addFileToClassPath(new Path("/myapp/mylib.jar"), job);
+     DistributedCache.addCacheArchive(new URI("/myapp/mytar.tar"), job);
+     DistributedCache.addCacheArchive(new URI("/myapp/mytgz.tgz"), job);
+     DistributedCache.addCacheArchive(new URI("/myapp/mytargz.tar.gz"), job);
+
+     3. Use the cached files in the {@link org.apache.hadoop.mapred.Mapper}
+     or {@link org.apache.hadoop.mapred.Reducer}:
+
+     public static class MapClass extends MapReduceBase
+     implements Mapper<K, V, K, V> {
+
+       private Path[] localArchives;
+       private Path[] localFiles;
+
+       public void configure(JobConf job) {
+         // Get the cached archives/files
+         File f = new File("./map.zip/some/file/in/zip.txt");
+       }
+
+       public void map(K key, V value,
+                       OutputCollector<K, V> output, Reporter reporter)
+       throws IOException {
+         // Use data from the cached archives/files here
+         // ...
+         // ...
+         output.collect(k, v);
+       }
+     }
+
+
    + + It is also very common to use the DistributedCache by using + {@link org.apache.hadoop.util.GenericOptionsParser}. + + This class includes methods that should be used by users + (specifically those mentioned in the example above, as well + as {@link DistributedCache#addArchiveToClassPath(Path, Configuration)}), + as well as methods intended for use by the MapReduce framework + (e.g., {@link org.apache.hadoop.mapred.JobClient}). + + @see org.apache.hadoop.mapred.JobConf + @see org.apache.hadoop.mapred.JobClient + @see org.apache.hadoop.mapreduce.Job]]> +     +     + +     + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + JobTracker, + as {@link JobTracker.State} + + {@link JobTracker.State} should no longer be used on M/R 2.x. The function + is kept to be compatible with M/R 1.x applications. + + @return the invalid state of the JobTracker.]]> + + + + + + + + + + + + + + ClusterStatus provides clients with information such as: +
      +
    1. + Size of the cluster. +
      2. +
    2. + Name of the trackers. +
      3. +
    3. + Task capacity of the cluster. +
      4. +
    4. + The number of currently running map and reduce tasks. +
      5. +
    5. + State of the JobTracker. +
      6. +
    6. + Details regarding black listed trackers. +
      7. +
    + +

    Clients can query for the latest ClusterStatus, via + {@link JobClient#getClusterStatus()}.

    + + @see JobClient]]> +     +     + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Counters represent global counters, defined either by the + Map-Reduce framework or applications. Each Counter can be of + any {@link Enum} type.

    + +

    Counters are bunched into {@link Group}s, each comprising of + counters from a particular Enum class.]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Group of counters, comprising of counters from a particular + counter {@link Enum} class. + +

    Grouphandles localization of the class name and the + counter names.

    ]]> +     +     + + + + + + + + + + + + + + + + + + + + + + + + + FileInputFormat always returns + true. Implementations that may deal with non-splittable files must + override this method. + + FileInputFormat implementations can override this and return + false to ensure that individual input files are never split-up + so that {@link Mapper}s process entire files. + + @param fs the file system that the file is on + @param filename the file name to check + @return is this file splitable?]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + FileInputFormat is the base class for all file-based + InputFormats. This provides a generic implementation of + {@link #getSplits(JobConf, int)}. + + Implementations of FileInputFormat can also override the + {@link #isSplitable(FileSystem, Path)} method to prevent input files + from being split-up in certain situations. Implementations that may + deal with non-splittable files must override this method, since + the default implementation assumes splitting is always possible.]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + true if the job output should be compressed, + false otherwise]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Tasks' Side-Effect Files + +

    Note: The following is valid only if the {@link OutputCommitter} + is {@link FileOutputCommitter}. If OutputCommitter is not + a FileOutputCommitter, the task's temporary output + directory is same as {@link #getOutputPath(JobConf)} i.e. + ${mapreduce.output.fileoutputformat.outputdir}$

    + +

    Some applications need to create/write-to side-files, which differ from + the actual job-outputs. + +

    In such cases there could be issues with 2 instances of the same TIP + (running simultaneously e.g. speculative tasks) trying to open/write-to the + same file (path) on HDFS. Hence the application-writer will have to pick + unique names per task-attempt (e.g. using the attemptid, say + attempt_200709221812_0001_m_000000_0), not just per TIP.

    + +

    To get around this the Map-Reduce framework helps the application-writer + out by maintaining a special + ${mapreduce.output.fileoutputformat.outputdir}/_temporary/_${taskid} + sub-directory for each task-attempt on HDFS where the output of the + task-attempt goes. On successful completion of the task-attempt the files + in the ${mapreduce.output.fileoutputformat.outputdir}/_temporary/_${taskid} (only) + are promoted to ${mapreduce.output.fileoutputformat.outputdir}. Of course, the + framework discards the sub-directory of unsuccessful task-attempts. This + is completely transparent to the application.

    + +

    The application-writer can take advantage of this by creating any + side-files required in ${mapreduce.task.output.dir} during execution + of his reduce-task i.e. via {@link #getWorkOutputPath(JobConf)}, and the + framework will move them out similarly - thus she doesn't have to pick + unique paths per task-attempt.

    + +

    Note: the value of ${mapreduce.task.output.dir} during + execution of a particular task-attempt is actually + ${mapreduce.output.fileoutputformat.outputdir}/_temporary/_{$taskid}, and this value is + set by the map-reduce framework. So, just create any side-files in the + path returned by {@link #getWorkOutputPath(JobConf)} from map/reduce + task to take advantage of this feature.

    + +

    The entire discussion holds true for maps of jobs with + reducer=NONE (i.e. 0 reduces) since output of the map, in that case, + goes directly to HDFS.

    + + @return the {@link Path} to the task's temporary output directory + for the map-reduce job.]]> +     +     + + + + + + + + + + + + + The generated name can be used to create custom files from within the + different tasks for the job, the names for different tasks will not collide + with each other.

    + +

    The given name is postfixed with the task type, 'm' for maps, 'r' for + reduces and the task partition number. For example, give a name 'test' + running on the first map o the job the generated name will be + 'test-m-00000'.

    + + @param conf the configuration for the job. + @param name the name to make unique. + @return a unique name accross all tasks of the job.]]> +     +     + + + + + The path can be used to create custom files from within the map and + reduce tasks. The path name will be unique for each task. The path parent + will be the job output directory.

    ls + +

    This method uses the {@link #getUniqueName} method to make the file name + unique for the task.

    + + @param conf the configuration for the job. + @param name the name for the file. + @return a unique path accross all tasks of the job.]]> +     +     + + + +     + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
    or + conf.setInt(FixedLengthInputFormat.FIXED_RECORD_LENGTH, recordLength); +

    + @see FixedLengthRecordReader]]> +     +     + + + + + + + + + + + + + + + + + + + + + + + Each {@link InputSplit} is then assigned to an individual {@link Mapper} + for processing.

    + +

    Note: The split is a logical split of the inputs and the + input files are not physically split into chunks. For e.g. a split could + be <input-file-path, start, offset> tuple. + + @param job job configuration. + @param numSplits the desired number of splits, a hint. + @return an array of {@link InputSplit}s for the job.]]> + + + + + + + + + It is the responsibility of the RecordReader to respect + record boundaries while processing the logical split to present a + record-oriented view to the individual task.

    + + @param split the {@link InputSplit} + @param job the job that this split belongs to + @return a {@link RecordReader}]]> +     +     + + InputFormat describes the input-specification for a + Map-Reduce job. + +

    The Map-Reduce framework relies on the InputFormat of the + job to:

    +

      +
    1. + Validate the input-specification of the job. +
    2. + Split-up the input file(s) into logical {@link InputSplit}s, each of + which is then assigned to an individual {@link Mapper}. +
      3. +
    3. + Provide the {@link RecordReader} implementation to be used to glean + input records from the logical InputSplit for processing by + the {@link Mapper}. +
      4. +
    + +

    The default behavior of file-based {@link InputFormat}s, typically + sub-classes of {@link FileInputFormat}, is to split the + input into logical {@link InputSplit}s based on the total size, in + bytes, of the input files. However, the {@link FileSystem} blocksize of + the input files is treated as an upper bound for input splits. A lower bound + on the split size can be set via + + mapreduce.input.fileinputformat.split.minsize.

    + +

    Clearly, logical splits based on input-size is insufficient for many + applications since record boundaries are to be respected. In such cases, the + application has to also implement a {@link RecordReader} on whom lies the + responsibilty to respect record-boundaries and present a record-oriented + view of the logical InputSplit to the individual task. + + @see InputSplit + @see RecordReader + @see JobClient + @see FileInputFormat]]> + + + + + + + + + + InputSplit. + + @return the number of bytes in the input split. + @throws IOException]]> + + + + + + InputSplit is + located as an array of Strings. + @throws IOException]]> + + + + InputSplit represents the data to be processed by an + individual {@link Mapper}. + +

    Typically, it presents a byte-oriented view on the input and is the + responsibility of {@link RecordReader} of the job to process this and present + a record-oriented view. + + @see InputFormat + @see RecordReader]]> + + + + + + + + + + SplitLocationInfos describing how the split + data is stored at each location. A null value indicates that all the + locations have the data stored on disk. + @throws IOException]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + JobClient.]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + jobid doesn't correspond to any known job. + @throws IOException]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + JobClient is the primary interface for the user-job to interact + with the cluster. + + JobClient provides facilities to submit jobs, track their + progress, access component-tasks' reports/logs, get the Map-Reduce cluster + status information etc. + +

    The job submission process involves: +

      +
    1. + Checking the input and output specifications of the job. +
      2. +
    2. + Computing the {@link InputSplit}s for the job. +
      3. +
    3. + Setup the requisite accounting information for the {@link DistributedCache} + of the job, if necessary. +
      4. +
    4. + Copying the job's jar and configuration to the map-reduce system directory + on the distributed file-system. +
      5. +
    5. + Submitting the job to the cluster and optionally monitoring + it's status. +
      6. +
    + + Normally the user creates the application, describes various facets of the + job via {@link JobConf} and then uses the JobClient to submit + the job and monitor its progress. + +

    Here is an example on how to use JobClient:

    + 

    +     // Create a new JobConf
+     JobConf job = new JobConf(new Configuration(), MyJob.class);
+     
+     // Specify various job-specific parameters     
+     job.setJobName("myjob");
+     
+     job.setInputPath(new Path("in"));
+     job.setOutputPath(new Path("out"));
+     
+     job.setMapperClass(MyJob.MyMapper.class);
+     job.setReducerClass(MyJob.MyReducer.class);
+
+     // Submit the job, then poll for progress until the job is complete
+     JobClient.runJob(job);
+
    + + Job Control + +

    At times clients would chain map-reduce jobs to accomplish complex tasks + which cannot be done via a single map-reduce job. This is fairly easy since + the output of the job, typically, goes to distributed file-system and that + can be used as the input for the next job.

    + +

    However, this also means that the onus on ensuring jobs are complete + (success/failure) lies squarely on the clients. In such situations the + various job-control options are: +

      +
    1. + {@link #runJob(JobConf)} : submits the job and returns only after + the job has completed. +
      2. +
    2. + {@link #submitJob(JobConf)} : only submits the job, then poll the + returned handle to the {@link RunningJob} to query status and make + scheduling decisions. +
      3. +
    3. + {@link JobConf#setJobEndNotificationURI(String)} : setup a notification + on job-completion, thus avoiding polling. +
      4. +
    + + @see JobConf + @see ClusterStatus + @see Tool + @see DistributedCache]]> +     + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + If the parameter {@code loadDefaults} is false, the new instance + will not load resources from the default files. + + @param loadDefaults specifies whether to load from the default files]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + true if framework should keep the intermediate files + for failed tasks, false otherwise.]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + true if the outputs of the maps are to be compressed, + false otherwise.]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + This comparator should be provided if the equivalence rules for keys + for sorting the intermediates are different from those for grouping keys + before each call to + {@link Reducer#reduce(Object, java.util.Iterator, OutputCollector, Reporter)}.

    + +

    For key-value pairs (K1,V1) and (K2,V2), the values (V1, V2) are passed + in a single call to the reduce function if K1 and K2 compare as equal.

    + +

    Since {@link #setOutputKeyComparatorClass(Class)} can be used to control + how keys are sorted, this can be used in conjunction to simulate + secondary sort on values.

    + +

    Note: This is not a guarantee of the combiner sort being + stable in any sense. (In any case, with the order of available + map-outputs to the combiner being non-deterministic, it wouldn't make + that much sense.)

    + + @param theClass the comparator class to be used for grouping keys for the + combiner. It should implement RawComparator. + @see #setOutputKeyComparatorClass(Class)]]> +     +     + + + + This comparator should be provided if the equivalence rules for keys + for sorting the intermediates are different from those for grouping keys + before each call to + {@link Reducer#reduce(Object, java.util.Iterator, OutputCollector, Reporter)}.

    + +

    For key-value pairs (K1,V1) and (K2,V2), the values (V1, V2) are passed + in a single call to the reduce function if K1 and K2 compare as equal.

    + +

    Since {@link #setOutputKeyComparatorClass(Class)} can be used to control + how keys are sorted, this can be used in conjunction to simulate + secondary sort on values.

    + +

    Note: This is not a guarantee of the reduce sort being + stable in any sense. (In any case, with the order of available + map-outputs to the reduce being non-deterministic, it wouldn't make + that much sense.)

    + + @param theClass the comparator class to be used for grouping keys. + It should implement RawComparator. + @see #setOutputKeyComparatorClass(Class) + @see #setCombinerKeyGroupingComparator(Class)]]> +     +     + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + combiner class used to combine map-outputs + before being sent to the reducers. Typically the combiner is same as the + the {@link Reducer} for the job i.e. {@link #getReducerClass()}. + + @return the user-defined combiner class used to combine map-outputs.]]> + + + + + + combiner class used to combine map-outputs + before being sent to the reducers. + +

    The combiner is an application-specified aggregation operation, which + can help cut down the amount of data transferred between the + {@link Mapper} and the {@link Reducer}, leading to better performance.

    + +

    The framework may invoke the combiner 0, 1, or multiple times, in both + the mapper and reducer tasks. In general, the combiner is called as the + sort/merge result is written to disk. The combiner must: +

      +
    • be side-effect free
      • +
    • have the same input and output key types and the same input and + output value types
      • +
    + +

    Typically the combiner is same as the Reducer for the + job i.e. {@link #setReducerClass(Class)}.

    + + @param theClass the user-defined combiner class used to combine + map-outputs.]]> +     +     + + + true. + + @return true if speculative execution be used for this job, + false otherwise.]]> + + + + + + true if speculative execution + should be turned on, else false.]]> + + + + + true. + + @return true if speculative execution be + used for this job for map tasks, + false otherwise.]]> + + + + + + true if speculative execution + should be turned on for map tasks, + else false.]]> + + + + + true. + + @return true if speculative execution be used + for reduce tasks for this job, + false otherwise.]]> + + + + + + true if speculative execution + should be turned on for reduce tasks, + else false.]]> + + + + + 1. + + @return the number of map tasks for this job.]]> + + + + + + Note: This is only a hint to the framework. The actual + number of spawned map tasks depends on the number of {@link InputSplit}s + generated by the job's {@link InputFormat#getSplits(JobConf, int)}. + + A custom {@link InputFormat} is typically used to accurately control + the number of map tasks for the job.

    + + How many maps? + +

    The number of maps is usually driven by the total size of the inputs + i.e. total number of blocks of the input files.

    + +

    The right level of parallelism for maps seems to be around 10-100 maps + per-node, although it has been set up to 300 or so for very cpu-light map + tasks. Task setup takes awhile, so it is best if the maps take at least a + minute to execute.

    + +

    The default behavior of file-based {@link InputFormat}s is to split the + input into logical {@link InputSplit}s based on the total size, in + bytes, of input files. However, the {@link FileSystem} blocksize of the + input files is treated as an upper bound for input splits. A lower bound + on the split size can be set via + + mapreduce.input.fileinputformat.split.minsize.

    + +

    Thus, if you expect 10TB of input data and have a blocksize of 128MB, + you'll end up with 82,000 maps, unless {@link #setNumMapTasks(int)} is + used to set it even higher.

    + + @param n the number of map tasks for this job. + @see InputFormat#getSplits(JobConf, int) + @see FileInputFormat + @see FileSystem#getDefaultBlockSize() + @see FileStatus#getBlockSize()]]> +     +     + + + 1. + + @return the number of reduce tasks for this job.]]> + + + + + + How many reduces? + +

    The right number of reduces seems to be 0.95 or + 1.75 multiplied by ( + available memory for reduce tasks + (The value of this should be smaller than + numNodes * yarn.nodemanager.resource.memory-mb + since the resource of memory is shared by map tasks and other + applications) / + + mapreduce.reduce.memory.mb). +

    + +

    With 0.95 all of the reduces can launch immediately and + start transfering map outputs as the maps finish. With 1.75 + the faster nodes will finish their first round of reduces and launch a + second wave of reduces doing a much better job of load balancing.

    + +

    Increasing the number of reduces increases the framework overhead, but + increases load balancing and lowers the cost of failures.

    + +

    The scaling factors above are slightly less than whole numbers to + reserve a few reduce slots in the framework for speculative-tasks, failures + etc.

    + + Reducer NONE + +

    It is legal to set the number of reduce-tasks to zero.

    + +

    In this case the output of the map-tasks directly go to distributed + file-system, to the path set by + {@link FileOutputFormat#setOutputPath(JobConf, Path)}. Also, the + framework doesn't sort the map-outputs before writing it out to HDFS.

    + + @param n the number of reduce tasks for this job.]]> +     +     + + + mapreduce.map.maxattempts + property. If this property is not already set, the default is 4 attempts. + + @return the max number of attempts per map task.]]> + + + + + + + + + + + mapreduce.reduce.maxattempts + property. If this property is not already set, the default is 4 attempts. + + @return the max number of attempts per reduce task.]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + noFailures, the + tasktracker is blacklisted for this job. + + @param noFailures maximum no. of failures of a given job per tasktracker.]]> + + + + + blacklisted for this job. + + @return the maximum no. of failures of a given job per tasktracker.]]> + + + + + failed. + + Defaults to zero, i.e. any failed map-task results in + the job being declared as {@link JobStatus#FAILED}. + + @return the maximum percentage of map tasks that can fail without + the job being aborted.]]> + + + + + + failed. + + @param percent the maximum percentage of map tasks that can fail without + the job being aborted.]]> + + + + + failed. + + Defaults to zero, i.e. any failed reduce-task results + in the job being declared as {@link JobStatus#FAILED}. + + @return the maximum percentage of reduce tasks that can fail without + the job being aborted.]]> + + + + + + failed. + + @param percent the maximum percentage of reduce tasks that can fail without + the job being aborted.]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + The debug script can aid debugging of failed map tasks. The script is + given task's stdout, stderr, syslog, jobconf files as arguments.

    + +

    The debug command, run on the node where the map failed, is:

    + 

    + $script $stdout $stderr $syslog $jobconf.
+
    + +

    The script file is distributed through {@link DistributedCache} + APIs. The script needs to be symlinked.

    + +

    Here is an example on how to submit a script + 

    + job.setMapDebugScript("./myscript");
+ DistributedCache.createSymlink(job);
+ DistributedCache.addCacheFile("/debug/scripts/myscript#myscript");
+
    + + @param mDbgScript the script name]]> +     +     + + + + + + + + + The debug script can aid debugging of failed reduce tasks. The script + is given task's stdout, stderr, syslog, jobconf files as arguments.

    + +

    The debug command, run on the node where the map failed, is:

    + 

    + $script $stdout $stderr $syslog $jobconf.
+
    + +

    The script file is distributed through {@link DistributedCache} + APIs. The script file needs to be symlinked

    + +

    Here is an example on how to submit a script + 

    + job.setReduceDebugScript("./myscript");
+ DistributedCache.createSymlink(job);
+ DistributedCache.addCacheFile("/debug/scripts/myscript#myscript");
+
    + + @param rDbgScript the script name]]> +     +     + + + + + + + + null if it hasn't + been set. + @see #setJobEndNotificationURI(String)]]> + + + + + + The uri can contain 2 special parameters: $jobId and + $jobStatus. Those, if present, are replaced by the job's + identifier and completion-status respectively.

    + +

    This is typically used by application-writers to implement chaining of + Map-Reduce jobs in an asynchronous manner.

    + + @param uri the job end notification uri + @see JobStatus]]> +     +     + + + + + + + + + + + + + + + When a job starts, a shared directory is created at location + + ${mapreduce.cluster.local.dir}/taskTracker/$user/jobcache/$jobid/work/ . + This directory is exposed to the users through + mapreduce.job.local.dir . + So, the tasks can use this space + as scratch space and share files among them.

    + This value is available as System property also. + + @return The localized job specific shared directory]]> +     +     + + + + For backward compatibility, if the job configuration sets the + key {@link #MAPRED_TASK_MAXVMEM_PROPERTY} to a value different + from {@link #DISABLED_MEMORY_LIMIT}, that value will be used + after converting it from bytes to MB. + @return memory required to run a map task of the job, in MB,]]> + + + + + + + + + For backward compatibility, if the job configuration sets the + key {@link #MAPRED_TASK_MAXVMEM_PROPERTY} to a value different + from {@link #DISABLED_MEMORY_LIMIT}, that value will be used + after converting it from bytes to MB. + @return memory required to run a reduce task of the job, in MB.]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + This method is deprecated. Now, different memory limits can be + set for map and reduce tasks of a job, in MB. +

    + For backward compatibility, if the job configuration sets the + key {@link #MAPRED_TASK_MAXVMEM_PROPERTY}, that value is returned. + Otherwise, this method will return the larger of the values returned by + {@link #getMemoryForMapTask()} and {@link #getMemoryForReduceTask()} + after converting them into bytes. + + @return Memory required to run a task of this job, in bytes. + @see #setMaxVirtualMemoryForTask(long) + @deprecated Use {@link #getMemoryForMapTask()} and + {@link #getMemoryForReduceTask()}]]> + + + + + + + mapred.task.maxvmem is split into + mapreduce.map.memory.mb + and mapreduce.map.memory.mb,mapred + each of the new key are set + as mapred.task.maxvmem / 1024 + as new values are in MB + + @param vmem Maximum amount of virtual memory in bytes any task of this job + can use. + @see #getMaxVirtualMemoryForTask() + @deprecated + Use {@link #setMemoryForMapTask(long mem)} and + Use {@link #setMemoryForReduceTask(long mem)}]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + k1=v1,k2=v2. Further it can + reference existing environment variables via $key on + Linux or %key% on Windows. + + Example: +

      +
    • A=foo - This will set the env variable A to foo.
      • +
    + + @deprecated Use {@link #MAPRED_MAP_TASK_ENV} or + {@link #MAPRED_REDUCE_TASK_ENV}]]> +     + + + + k1=v1,k2=v2. Further it can + reference existing environment variables via $key on + Linux or %key% on Windows. + + Example: +
      +
    • A=foo - This will set the env variable A to foo.
      • +
    + + You can also add environment variables individually by appending + .VARNAME to this configuration key, where VARNAME is + the name of the environment variable. + + Example: +
      +
    • mapreduce.map.env.VARNAME=value
      • +
    ]]> +     +     + + + k1=v1,k2=v2. Further it can + reference existing environment variables via $key on + Linux or %key% on Windows. + + Example: +
      +
    • A=foo - This will set the env variable A to foo.
      • +
    + + You can also add environment variables individually by appending + .VARNAME to this configuration key, where VARNAME is + the name of the environment variable. + + Example: +
      +
    • mapreduce.reduce.env.VARNAME=value
      • +
    ]]> +     +     + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + JobConf is the primary interface for a user to describe a + map-reduce job to the Hadoop framework for execution. The framework tries to + faithfully execute the job as-is described by JobConf, however: +
      +
    1. + Some configuration parameters might have been marked as + + final by administrators and hence cannot be altered. +
      2. +
    2. + While some job parameters are straight-forward to set + (e.g. {@link #setNumReduceTasks(int)}), some parameters interact subtly + with the rest of the framework and/or job-configuration and is relatively + more complex for the user to control finely + (e.g. {@link #setNumMapTasks(int)}). +
      3. +
    + +

    JobConf typically specifies the {@link Mapper}, combiner + (if any), {@link Partitioner}, {@link Reducer}, {@link InputFormat} and + {@link OutputFormat} implementations to be used etc. + +

    Optionally JobConf is used to specify other advanced facets + of the job such as Comparators to be used, files to be put in + the {@link DistributedCache}, whether or not intermediate and/or job outputs + are to be compressed (and how), debugability via user-provided scripts + ( {@link #setMapDebugScript(String)}/{@link #setReduceDebugScript(String)}), + for doing post-processing on task logs, task's stdout, stderr, syslog. + and etc.

    + +

    Here is an example on how to configure a job via JobConf:

    + 

    +     // Create a new JobConf
+     JobConf job = new JobConf(new Configuration(), MyJob.class);
+     
+     // Specify various job-specific parameters     
+     job.setJobName("myjob");
+     
+     FileInputFormat.setInputPaths(job, new Path("in"));
+     FileOutputFormat.setOutputPath(job, new Path("out"));
+     
+     job.setMapperClass(MyJob.MyMapper.class);
+     job.setCombinerClass(MyJob.MyReducer.class);
+     job.setReducerClass(MyJob.MyReducer.class);
+     
+     job.setInputFormat(SequenceFileInputFormat.class);
+     job.setOutputFormat(SequenceFileOutputFormat.class);
+
    + + @see JobClient + @see ClusterStatus + @see Tool + @see DistributedCache]]> +     +     + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + any job + run on the jobtracker started at 200707121733, we would use : + 
     
+ JobID.getTaskIDsPattern("200707121733", null);
+
    + which will return : + 
     "job_200707121733_[0-9]*"
    + @param jtIdentifier jobTracker identifier, or null + @param jobId job number, or null + @return a regex pattern matching JobIDs]]> +     +     + + + An example JobID is : + job_200707121733_0003 , which represents the third job + running at the jobtracker started at 200707121733. +

    + Applications should never construct or parse JobID strings, but rather + use appropriate constructors or {@link #forName(String)} method. + + @see TaskID + @see TaskAttemptID]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Output pairs need not be of the same types as input pairs. A given + input pair may map to zero or many output pairs. Output pairs are + collected with calls to + {@link OutputCollector#collect(Object,Object)}.

    + +

    Applications can use the {@link Reporter} provided to report progress + or just indicate that they are alive. In scenarios where the application + takes significant amount of time to process individual key/value + pairs, this is crucial since the framework might assume that the task has + timed-out and kill that task. The other way of avoiding this is to set + + mapreduce.task.timeout to a high-enough value (or even zero for no + time-outs).

    + + @param key the input key. + @param value the input value. + @param output collects mapped keys and values. + @param reporter facility to report progress.]]> +     + + + Maps are the individual tasks which transform input records into a + intermediate records. The transformed intermediate records need not be of + the same type as the input records. A given input pair may map to zero or + many output pairs.

    + +

    The Hadoop Map-Reduce framework spawns one map task for each + {@link InputSplit} generated by the {@link InputFormat} for the job. + Mapper implementations can access the {@link JobConf} for the + job via the {@link JobConfigurable#configure(JobConf)} and initialize + themselves. Similarly they can use the {@link Closeable#close()} method for + de-initialization.

    + +

    The framework then calls + {@link #map(Object, Object, OutputCollector, Reporter)} + for each key/value pair in the InputSplit for that task.

    + +

    All intermediate values associated with a given output key are + subsequently grouped by the framework, and passed to a {@link Reducer} to + determine the final output. Users can control the grouping by specifying + a Comparator via + {@link JobConf#setOutputKeyComparatorClass(Class)}.

    + +

    The grouped Mapper outputs are partitioned per + Reducer. Users can control which keys (and hence records) go to + which Reducer by implementing a custom {@link Partitioner}. + +

    Users can optionally specify a combiner, via + {@link JobConf#setCombinerClass(Class)}, to perform local aggregation of the + intermediate outputs, which helps to cut down the amount of data transferred + from the Mapper to the Reducer. + +

    The intermediate, grouped outputs are always stored in + {@link SequenceFile}s. Applications can specify if and how the intermediate + outputs are to be compressed and which {@link CompressionCodec}s are to be + used via the JobConf.

    + +

    If the job has + zero + reduces then the output of the Mapper is directly written + to the {@link FileSystem} without grouping by keys.

    + +

    Example:

    + 

    +     public class MyMapper<K extends WritableComparable, V extends Writable> 
+     extends MapReduceBase implements Mapper<K, V, K, V> {
+     
+       static enum MyCounters { NUM_RECORDS }
+       
+       private String mapTaskId;
+       private String inputFile;
+       private int noRecords = 0;
+       
+       public void configure(JobConf job) {
+         mapTaskId = job.get(JobContext.TASK_ATTEMPT_ID);
+         inputFile = job.get(JobContext.MAP_INPUT_FILE);
+       }
+       
+       public void map(K key, V val,
+                       OutputCollector<K, V> output, Reporter reporter)
+       throws IOException {
+         // Process the <key, value> pair (assume this takes a while)
+         // ...
+         // ...
+         
+         // Let the framework know that we are alive, and kicking!
+         // reporter.progress();
+         
+         // Process some more
+         // ...
+         // ...
+         
+         // Increment the no. of <key, value> pairs processed
+         ++noRecords;
+
+         // Increment counters
+         reporter.incrCounter(NUM_RECORDS, 1);
+        
+         // Every 100 records update application-level status
+         if ((noRecords%100) == 0) {
+           reporter.setStatus(mapTaskId + " processed " + noRecords + 
+                              " from input-file: " + inputFile); 
+         }
+         
+         // Output the result
+         output.collect(key, val);
+       }
+     }
+
    + +

    Applications may write a custom {@link MapRunnable} to exert greater + control on map processing e.g. multi-threaded Mappers etc.

    + + @see JobConf + @see InputFormat + @see Partitioner + @see Reducer + @see MapReduceBase + @see MapRunnable + @see SequenceFile]]> +     +     + + + + + + + + + + + + + + + + + + + + + Provides default no-op implementations for a few methods, most non-trivial + applications need to override some of them.

    ]]> +     +     + + + + + + + + + + + <key, value> pairs. + +

    Mapping of input records to output records is complete when this method + returns.

    + + @param input the {@link RecordReader} to read the input records. + @param output the {@link OutputCollector} to collect the outputrecords. + @param reporter {@link Reporter} to report progress, status-updates etc. + @throws IOException]]> +     +     + + Custom implementations of MapRunnable can exert greater + control on map processing e.g. multi-threaded, asynchronous mappers etc.

    + + @see Mapper]]> +     +     + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + nearly + equal content length.
    + Subclasses implement {@link #getRecordReader(InputSplit, JobConf, Reporter)} + to construct RecordReader's for MultiFileSplit's. + @see MultiFileSplit]]> +     +     + + + + + + + + + + + + + MultiFileSplit can be used to implement {@link RecordReader}'s, with + reading one record per file. + @see FileSplit + @see MultiFileInputFormat]]> + + + + + + + + + + + + + + + <key, value> pairs output by {@link Mapper}s + and {@link Reducer}s. + +

    OutputCollector is the generalization of the facility + provided by the Map-Reduce framework to collect data output by either the + Mapper or the Reducer i.e. intermediate outputs + or the output of the job.

    ]]> +     +     + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + true if task output recovery is supported, + false otherwise + @throws IOException + @see #recoverTask(TaskAttemptContext)]]> + + + + + + + true repeatable job commit is supported, + false otherwise + @throws IOException]]> + + + + + + + + + + + OutputCommitter. This is called from the application master + process, but it is called individually for each task. + + If an exception is thrown the task will be attempted again. + + @param taskContext Context of the task whose output is being recovered + @throws IOException]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + OutputCommitter describes the commit of task output for a + Map-Reduce job. + +

    The Map-Reduce framework relies on the OutputCommitter of + the job to:

    +

      +
    1. + Setup the job during initialization. For example, create the temporary + output directory for the job during the initialization of the job. +
      2. +
    2. + Cleanup the job after the job completion. For example, remove the + temporary output directory after the job completion. +
      3. +
    3. + Setup the task temporary output. +
      4. +
    4. + Check whether a task needs a commit. This is to avoid the commit + procedure if a task does not need commit. +
      5. +
    5. + Commit of the task output. +
      6. +
    6. + Discard the task commit. +
      7. +
    + The methods in this class can be called from several different processes and + from several different contexts. It is important to know which process and + which context each is called from. Each method should be marked accordingly + in its documentation. It is also important to note that not all methods are + guaranteed to be called once and only once. If a method is not guaranteed to + have this property the output committer needs to handle this appropriately. + Also note it will only be in rare situations where they may be called + multiple times for the same task. + + @see FileOutputCommitter + @see JobContext + @see TaskAttemptContext]]> +     +     + + + + + + + + + + + + + + + + + + + This is to validate the output specification for the job when it is + a job is submitted. Typically checks that it does not already exist, + throwing an exception when it already exists, so that output is not + overwritten.

    + + Implementations which write to filesystems which support delegation + tokens usually collect the tokens for the destination path(s) + and attach them to the job configuration. + @param ignored + @param job job configuration. + @throws IOException when output should not be attempted]]> +     +     + + OutputFormat describes the output-specification for a + Map-Reduce job. + +

    The Map-Reduce framework relies on the OutputFormat of the + job to:

    +

      +
    1. + Validate the output-specification of the job. For e.g. check that the + output directory doesn't already exist. +
    2. + Provide the {@link RecordWriter} implementation to be used to write out + the output files of the job. Output files are stored in a + {@link FileSystem}. +
      3. +
    + + @see RecordWriter + @see JobConf]]> +     +     + + + + + + + + + + + + + + + + + + + + + + + Typically a hash function on a all or a subset of the key.

    + + @param key the key to be paritioned. + @param value the entry value. + @param numPartitions the total number of partitions. + @return the partition number for the key.]]> +     +     + + Partitioner controls the partitioning of the keys of the + intermediate map-outputs. The key (or a subset of the key) is used to derive + the partition, typically by a hash function. The total number of partitions + is the same as the number of reduce tasks for the job. Hence this controls + which of the m reduce tasks the intermediate key (and hence the + record) is sent for reduction.

    + +

    Note: A Partitioner is created only when there are multiple + reducers.

    + + @see Reducer]]> +     +     + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + 0.0 to 1.0. + @throws IOException]]> + + + + RecordReader reads <key, value> pairs from an + {@link InputSplit}. + +

    RecordReader, typically, converts the byte-oriented view of + the input, provided by the InputSplit, and presents a + record-oriented view for the {@link Mapper} and {@link Reducer} tasks for + processing. It thus assumes the responsibility of processing record + boundaries and presenting the tasks with keys and values.

    + + @see InputSplit + @see InputFormat]]> +     +     + + + + + + + + + + + + + + + + RecordWriter to future operations. + + @param reporter facility to report progress. + @throws IOException]]> + + + + RecordWriter writes the output <key, value> pairs + to an output file. + +

    RecordWriter implementations write the job outputs to the + {@link FileSystem}. + + @see OutputFormat]]> + + + + + + + + + + + + + + + Reduces values for a given key. + +

    The framework calls this method for each + <key, (list of values)> pair in the grouped inputs. + Output values must be of the same type as input values. Input keys must + not be altered. The framework will reuse the key and value objects + that are passed into the reduce, therefore the application should clone + the objects they want to keep a copy of. In many cases, all values are + combined into zero or one value. +

    + +

    Output pairs are collected with calls to + {@link OutputCollector#collect(Object,Object)}.

    + +

    Applications can use the {@link Reporter} provided to report progress + or just indicate that they are alive. In scenarios where the application + takes a significant amount of time to process individual key/value + pairs, this is crucial since the framework might assume that the task has + timed-out and kill that task. The other way of avoiding this is to set + + mapreduce.task.timeout to a high-enough value (or even zero for no + time-outs).

    + + @param key the key. + @param values the list of values to reduce. + @param output to collect keys and combined values. + @param reporter facility to report progress.]]> +     + + + The number of Reducers for the job is set by the user via + {@link JobConf#setNumReduceTasks(int)}. Reducer implementations + can access the {@link JobConf} for the job via the + {@link JobConfigurable#configure(JobConf)} method and initialize themselves. + Similarly they can use the {@link Closeable#close()} method for + de-initialization.

    + +

    Reducer has 3 primary phases:

    +
      +
    1. + + Shuffle + +

      Reducer is input the grouped output of a {@link Mapper}. + In the phase the framework, for each Reducer, fetches the + relevant partition of the output of all the Mappers, via HTTP. +

      +
      2. + +
    2. + Sort + +

      The framework groups Reducer inputs by keys + (since different Mappers may have output the same key) in this + stage.

      + +

      The shuffle and sort phases occur simultaneously i.e. while outputs are + being fetched they are merged.

      + + SecondarySort + +

      If equivalence rules for keys while grouping the intermediates are + different from those for grouping keys before reduction, then one may + specify a Comparator via + {@link JobConf#setOutputValueGroupingComparator(Class)}.Since + {@link JobConf#setOutputKeyComparatorClass(Class)} can be used to + control how intermediate keys are grouped, these can be used in conjunction + to simulate secondary sort on values.

      + + + For example, say that you want to find duplicate web pages and tag them + all with the url of the "best" known example. You would set up the job + like: +
        +
      • Map Input Key: url
        • +
      • Map Input Value: document
        • +
      • Map Output Key: document checksum, url pagerank
        • +
      • Map Output Value: url
        • +
      • Partitioner: by checksum
        • +
      • OutputKeyComparator: by checksum and then decreasing pagerank
        • +
      • OutputValueGroupingComparator: by checksum
        • +
      +
      3. + +
    3. + Reduce + +

      In this phase the + {@link #reduce(Object, Iterator, OutputCollector, Reporter)} + method is called for each <key, (list of values)> pair in + the grouped inputs.

      +

      The output of the reduce task is typically written to the + {@link FileSystem} via + {@link OutputCollector#collect(Object, Object)}.

      +
      4. +
    + +

    The output of the Reducer is not re-sorted.

    + +

    Example:

    + 

    +     public class MyReducer<K extends WritableComparable, V extends Writable> 
+     extends MapReduceBase implements Reducer<K, V, K, V> {
+     
+       static enum MyCounters { NUM_RECORDS }
+        
+       private String reduceTaskId;
+       private int noKeys = 0;
+       
+       public void configure(JobConf job) {
+         reduceTaskId = job.get(JobContext.TASK_ATTEMPT_ID);
+       }
+       
+       public void reduce(K key, Iterator<V> values,
+                          OutputCollector<K, V> output, 
+                          Reporter reporter)
+       throws IOException {
+       
+         // Process
+         int noValues = 0;
+         while (values.hasNext()) {
+           V value = values.next();
+           
+           // Increment the no. of values for this key
+           ++noValues;
+           
+           // Process the <key, value> pair (assume this takes a while)
+           // ...
+           // ...
+           
+           // Let the framework know that we are alive, and kicking!
+           if ((noValues%10) == 0) {
+             reporter.progress();
+           }
+         
+           // Process some more
+           // ...
+           // ...
+           
+           // Output the <key, value> 
+           output.collect(key, value);
+         }
+         
+         // Increment the no. of <key, list of values> pairs processed
+         ++noKeys;
+         
+         // Increment counters
+         reporter.incrCounter(NUM_RECORDS, 1);
+         
+         // Every 100 keys update application-level status
+         if ((noKeys%100) == 0) {
+           reporter.setStatus(reduceTaskId + " processed " + noKeys);
+         }
+       }
+     }
+
    + + @see Mapper + @see Partitioner + @see Reporter + @see MapReduceBase]]> +     +     + + + + + + + + + + + + + + Counter of the given group/name.]]> + + + + + + + Counter of the given group/name.]]> + + + + + + + Enum. + @param amount A non-negative amount by which the counter is to + be incremented.]]> + + + + + + + + + + + + + + InputSplit that the map is reading from. + @throws UnsupportedOperationException if called outside a mapper]]> + + + + + + + + + + + + + + {@link Mapper} and {@link Reducer} can use the Reporter + provided to report progress or just indicate that they are alive. In + scenarios where the application takes significant amount of time to + process individual key/value pairs, this is crucial since the framework + might assume that the task has timed-out and kill that task. + +

    Applications can also update {@link Counters} via the provided + Reporter .

    + + @see Progressable + @see Counters]]> +     +     + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + progress of the job's map-tasks, as a float between 0.0 + and 1.0. When all map tasks have completed, the function returns 1.0. + + @return the progress of the job's map-tasks. + @throws IOException]]> + + + + + + progress of the job's reduce-tasks, as a float between 0.0 + and 1.0. When all reduce tasks have completed, the function returns 1.0. + + @return the progress of the job's reduce-tasks. + @throws IOException]]> + + + + + + progress of the job's cleanup-tasks, as a float between 0.0 + and 1.0. When all cleanup tasks have completed, the function returns 1.0. + + @return the progress of the job's cleanup-tasks. + @throws IOException]]> + + + + + + progress of the job's setup-tasks, as a float between 0.0 + and 1.0. When all setup tasks have completed, the function returns 1.0. + + @return the progress of the job's setup-tasks. + @throws IOException]]> + + + + + + true if the job is complete, else false. + @throws IOException]]> + + + + + + true if the job succeeded, else false. + @throws IOException]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + true if the job retired, else false. + @throws IOException]]> + + + + + + + + + + RunningJob is the user-interface to query for details on a + running Map-Reduce job. + +

    Clients can get hold of RunningJob via the {@link JobClient} + and then query the running-job for details such as name, configuration, + progress etc.

    + + @see JobClient]]> +     +     + + + + + + + + + + + + + + + + + + + + + + + + + This allows the user to specify the key class to be different + from the actual class ({@link BytesWritable}) used for writing

    + + @param conf the {@link JobConf} to modify + @param theClass the SequenceFile output key class.]]> +     +     + + + + + This allows the user to specify the value class to be different + from the actual class ({@link BytesWritable}) used for writing

    + + @param conf the {@link JobConf} to modify + @param theClass the SequenceFile output key class.]]> +     +     + + + + + + + + + + + + + + + + + + + + + + + + + + + +     + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + true if auto increment + {@link SkipBadRecords#COUNTER_MAP_PROCESSED_RECORDS}. + false otherwise.]]> + + + + + + + + + + + + + true if auto increment + {@link SkipBadRecords#COUNTER_REDUCE_PROCESSED_GROUPS}. + false otherwise.]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Hadoop provides an optional mode of execution in which the bad records + are detected and skipped in further attempts. + +

    This feature can be used when map/reduce tasks crashes deterministically on + certain input. This happens due to bugs in the map/reduce function. The usual + course would be to fix these bugs. But sometimes this is not possible; + perhaps the bug is in third party libraries for which the source code is + not available. Due to this, the task never reaches to completion even with + multiple attempts and complete data for that task is lost.

    + +

    With this feature, only a small portion of data is lost surrounding + the bad record, which may be acceptable for some user applications. + see {@link SkipBadRecords#setMapperMaxSkipRecords(Configuration, long)}

    + +

    The skipping mode gets kicked off after certain no of failures + see {@link SkipBadRecords#setAttemptsToStartSkipping(Configuration, int)}

    + +

    In the skipping mode, the map/reduce task maintains the record range which + is getting processed at all times. Before giving the input to the + map/reduce function, it sends this record range to the Task tracker. + If task crashes, the Task tracker knows which one was the last reported + range. On further attempts that range get skipped.

    ]]> +     +     + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + all task attempt IDs + of any jobtracker, in any job, of the first + map task, we would use : + 
     
+ TaskAttemptID.getTaskAttemptIDsPattern(null, null, true, 1, null);
+
    + which will return : + 
     "attempt_[^_]*_[0-9]*_m_000001_[0-9]*"
    + @param jtIdentifier jobTracker identifier, or null + @param jobId job number, or null + @param isMap whether the tip is a map, or null + @param taskId taskId number, or null + @param attemptId the task attempt number, or null + @return a regex pattern matching TaskAttemptIDs]]> +     +     + + + + + + + + all task attempt IDs + of any jobtracker, in any job, of the first + map task, we would use : + 
     
+ TaskAttemptID.getTaskAttemptIDsPattern(null, null, TaskType.MAP, 1, null);
+
    + which will return : + 
     "attempt_[^_]*_[0-9]*_m_000001_[0-9]*"
    + @param jtIdentifier jobTracker identifier, or null + @param jobId job number, or null + @param type the {@link TaskType} + @param taskId taskId number, or null + @param attemptId the task attempt number, or null + @return a regex pattern matching TaskAttemptIDs]]> +     +     + + + An example TaskAttemptID is : + attempt_200707121733_0003_m_000005_0 , which represents the + zeroth task attempt for the fifth map task in the third job + running at the jobtracker started at 200707121733. +

    + Applications should never construct or parse TaskAttemptID strings + , but rather use appropriate constructors or {@link #forName(String)} + method. + + @see JobID + @see TaskID]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + the first map task + of any jobtracker, of any job, we would use : + 

     
+ TaskID.getTaskIDsPattern(null, null, true, 1);
+
    + which will return : + 
     "task_[^_]*_[0-9]*_m_000001*"
    + @param jtIdentifier jobTracker identifier, or null + @param jobId job number, or null + @param isMap whether the tip is a map, or null + @param taskId taskId number, or null + @return a regex pattern matching TaskIDs + @deprecated Use {@link TaskID#getTaskIDsPattern(String, Integer, TaskType, + Integer)}]]> +     + + + + + + + + the first map task + of any jobtracker, of any job, we would use : + 
     
+ TaskID.getTaskIDsPattern(null, null, true, 1);
+
    + which will return : + 
     "task_[^_]*_[0-9]*_m_000001*"
    + @param jtIdentifier jobTracker identifier, or null + @param jobId job number, or null + @param type the {@link TaskType}, or null + @param taskId taskId number, or null + @return a regex pattern matching TaskIDs]]> +     +     + + + + + + + An example TaskID is : + task_200707121733_0003_m_000005 , which represents the + fifth map task in the third job running at the jobtracker + started at 200707121733. +

    + Applications should never construct or parse TaskID strings + , but rather use appropriate constructors or {@link #forName(String)} + method. + + @see JobID + @see TaskAttemptID]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + true if the Job was added.]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + ([,]*) + func ::= tbl(,"") + class ::= @see java.lang.Class#forName(java.lang.String) + path ::= @see org.apache.hadoop.fs.Path#Path(java.lang.String) + } + Reads expression from the mapred.join.expr property and + user-supplied join types from mapred.join.define.<ident> + types. Paths supplied to tbl are given as input paths to the + InputFormat class listed. + @see #compose(java.lang.String, java.lang.Class, java.lang.String...)]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + ,

    ) }]]> + + + + + + + + (tbl(,),tbl(,),...,tbl(,)) }]]> + + + + + + + + (tbl(,),tbl(,),...,tbl(,)) }]]> + + + + mapred.join.define.<ident> to a classname. In the expression + mapred.join.expr, the identifier will be assumed to be a + ComposableRecordReader. + mapred.join.keycomparator can be a classname used to compare keys + in the join. + @see #setFormat + @see JoinRecordReader + @see MultiFilterRecordReader]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + ...... + }]]> + + + + + + + + + + + + + + + + + + + + + capacity children to position + id in the parent reader. + The id of a root CompositeRecordReader is -1 by convention, but relying + on this is not recommended.]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + override(S1,S2,S3) will prefer values + from S3 over S2, and values from S2 over S1 for all keys + emitted from all sources.]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + It has to be specified how key and values are passed from one element of + the chain to the next, by value or by reference. If a Mapper leverages the + assumed semantics that the key and values are not modified by the collector + 'by value' must be used. If the Mapper does not expect this semantics, as + an optimization to avoid serialization and deserialization 'by reference' + can be used. +

    + For the added Mapper the configuration given for it, + mapperConf, have precedence over the job's JobConf. This + precedence is in effect when the task is running. +

    + IMPORTANT: There is no need to specify the output key/value classes for the + ChainMapper, this is done by the addMapper for the last mapper in the chain +

    + + @param job job's JobConf to add the Mapper class. + @param klass the Mapper class to add. + @param inputKeyClass mapper input key class. + @param inputValueClass mapper input value class. + @param outputKeyClass mapper output key class. + @param outputValueClass mapper output value class. + @param byValue indicates if key/values should be passed by value + to the next Mapper in the chain, if any. + @param mapperConf a JobConf with the configuration for the Mapper + class. It is recommended to use a JobConf without default values using the + JobConf(boolean loadDefaults) constructor with FALSE.]]> + + + + + + + If this method is overriden super.configure(...) should be + invoked at the beginning of the overwriter method.]]> + + + + + + + + + + map(...) methods of the Mappers in the chain.]]> + + + + + + + If this method is overriden super.close() should be + invoked at the end of the overwriter method.]]> + + + + + The Mapper classes are invoked in a chained (or piped) fashion, the output of + the first becomes the input of the second, and so on until the last Mapper, + the output of the last Mapper will be written to the task's output. +

    + The key functionality of this feature is that the Mappers in the chain do not + need to be aware that they are executed in a chain. This enables having + reusable specialized Mappers that can be combined to perform composite + operations within a single task. +

    + Special care has to be taken when creating chains that the key/values output + by a Mapper are valid for the following Mapper in the chain. It is assumed + all Mappers and the Reduce in the chain use maching output and input key and + value classes as no conversion is done by the chaining code. +

    + Using the ChainMapper and the ChainReducer classes is possible to compose + Map/Reduce jobs that look like [MAP+ / REDUCE MAP*]. And + immediate benefit of this pattern is a dramatic reduction in disk IO. +

    + IMPORTANT: There is no need to specify the output key/value classes for the + ChainMapper, this is done by the addMapper for the last mapper in the chain. +

    + ChainMapper usage pattern: +

    + 

    + ...
+ conf.setJobName("chain");
+ conf.setInputFormat(TextInputFormat.class);
+ conf.setOutputFormat(TextOutputFormat.class);
+
+ JobConf mapAConf = new JobConf(false);
+ ...
+ ChainMapper.addMapper(conf, AMap.class, LongWritable.class, Text.class,
+   Text.class, Text.class, true, mapAConf);
+
+ JobConf mapBConf = new JobConf(false);
+ ...
+ ChainMapper.addMapper(conf, BMap.class, Text.class, Text.class,
+   LongWritable.class, Text.class, false, mapBConf);
+
+ JobConf reduceConf = new JobConf(false);
+ ...
+ ChainReducer.setReducer(conf, XReduce.class, LongWritable.class, Text.class,
+   Text.class, Text.class, true, reduceConf);
+
+ ChainReducer.addMapper(conf, CMap.class, Text.class, Text.class,
+   LongWritable.class, Text.class, false, null);
+
+ ChainReducer.addMapper(conf, DMap.class, LongWritable.class, Text.class,
+   LongWritable.class, LongWritable.class, true, null);
+
+ FileInputFormat.setInputPaths(conf, inDir);
+ FileOutputFormat.setOutputPath(conf, outDir);
+ ...
+
+ JobClient jc = new JobClient(conf);
+ RunningJob job = jc.submitJob(conf);
+ ...
+
    ]]> +     +     + + + + + + + + + + + + + + + + + + + + + It has to be specified how key and values are passed from one element of + the chain to the next, by value or by reference. If a Reducer leverages the + assumed semantics that the key and values are not modified by the collector + 'by value' must be used. If the Reducer does not expect this semantics, as + an optimization to avoid serialization and deserialization 'by reference' + can be used. +

    + For the added Reducer the configuration given for it, + reducerConf, have precedence over the job's JobConf. This + precedence is in effect when the task is running. +

    + IMPORTANT: There is no need to specify the output key/value classes for the + ChainReducer, this is done by the setReducer or the addMapper for the last + element in the chain. + + @param job job's JobConf to add the Reducer class. + @param klass the Reducer class to add. + @param inputKeyClass reducer input key class. + @param inputValueClass reducer input value class. + @param outputKeyClass reducer output key class. + @param outputValueClass reducer output value class. + @param byValue indicates if key/values should be passed by value + to the next Mapper in the chain, if any. + @param reducerConf a JobConf with the configuration for the Reducer + class. It is recommended to use a JobConf without default values using the + JobConf(boolean loadDefaults) constructor with FALSE.]]> + + + + + + + + + + + + + + It has to be specified how key and values are passed from one element of + the chain to the next, by value or by reference. If a Mapper leverages the + assumed semantics that the key and values are not modified by the collector + 'by value' must be used. If the Mapper does not expect this semantics, as + an optimization to avoid serialization and deserialization 'by reference' + can be used. +

    + For the added Mapper the configuration given for it, + mapperConf, have precedence over the job's JobConf. This + precedence is in effect when the task is running. +

    + IMPORTANT: There is no need to specify the output key/value classes for the + ChainMapper, this is done by the addMapper for the last mapper in the chain + . + + @param job chain job's JobConf to add the Mapper class. + @param klass the Mapper class to add. + @param inputKeyClass mapper input key class. + @param inputValueClass mapper input value class. + @param outputKeyClass mapper output key class. + @param outputValueClass mapper output value class. + @param byValue indicates if key/values should be passed by value + to the next Mapper in the chain, if any. + @param mapperConf a JobConf with the configuration for the Mapper + class. It is recommended to use a JobConf without default values using the + JobConf(boolean loadDefaults) constructor with FALSE.]]> + + + + + + + If this method is overriden super.configure(...) should be + invoked at the beginning of the overwriter method.]]> + + + + + + + + + + reduce(...) method of the Reducer with the + map(...) methods of the Mappers in the chain.]]> + + + + + + + If this method is overriden super.close() should be + invoked at the end of the overwriter method.]]> + + + + + For each record output by the Reducer, the Mapper classes are invoked in a + chained (or piped) fashion, the output of the first becomes the input of the + second, and so on until the last Mapper, the output of the last Mapper will + be written to the task's output. +

    + The key functionality of this feature is that the Mappers in the chain do not + need to be aware that they are executed after the Reducer or in a chain. + This enables having reusable specialized Mappers that can be combined to + perform composite operations within a single task. +

    + Special care has to be taken when creating chains that the key/values output + by a Mapper are valid for the following Mapper in the chain. It is assumed + all Mappers and the Reduce in the chain use maching output and input key and + value classes as no conversion is done by the chaining code. +

    + Using the ChainMapper and the ChainReducer classes is possible to compose + Map/Reduce jobs that look like [MAP+ / REDUCE MAP*]. And + immediate benefit of this pattern is a dramatic reduction in disk IO. +

    + IMPORTANT: There is no need to specify the output key/value classes for the + ChainReducer, this is done by the setReducer or the addMapper for the last + element in the chain. +

    + ChainReducer usage pattern: +

    + 

    + ...
+ conf.setJobName("chain");
+ conf.setInputFormat(TextInputFormat.class);
+ conf.setOutputFormat(TextOutputFormat.class);
+
+ JobConf mapAConf = new JobConf(false);
+ ...
+ ChainMapper.addMapper(conf, AMap.class, LongWritable.class, Text.class,
+   Text.class, Text.class, true, mapAConf);
+
+ JobConf mapBConf = new JobConf(false);
+ ...
+ ChainMapper.addMapper(conf, BMap.class, Text.class, Text.class,
+   LongWritable.class, Text.class, false, mapBConf);
+
+ JobConf reduceConf = new JobConf(false);
+ ...
+ ChainReducer.setReducer(conf, XReduce.class, LongWritable.class, Text.class,
+   Text.class, Text.class, true, reduceConf);
+
+ ChainReducer.addMapper(conf, CMap.class, Text.class, Text.class,
+   LongWritable.class, Text.class, false, null);
+
+ ChainReducer.addMapper(conf, DMap.class, LongWritable.class, Text.class,
+   LongWritable.class, LongWritable.class, true, null);
+
+ FileInputFormat.setInputPaths(conf, inDir);
+ FileOutputFormat.setOutputPath(conf, outDir);
+ ...
+
+ JobClient jc = new JobClient(conf);
+ RunningJob job = jc.submitJob(conf);
+ ...
+
    ]]> +     +     + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + RecordReader's for CombineFileSplit's. + @see CombineFileSplit]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + CombineFileRecordReader. + + Subclassing is needed to get a concrete record reader wrapper because of the + constructor requirement. + + @see CombineFileRecordReader + @see CombineFileInputFormat]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + CombineFileInputFormat-equivalent for + SequenceFileInputFormat. + + @see CombineFileInputFormat]]> + + + + + + + + + + + + + + + CombineFileInputFormat-equivalent for + TextInputFormat. + + @see CombineFileInputFormat]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + true if the name output is multi, false + if it is single. If the name output is not defined it returns + false]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + By default these counters are disabled. +

    + MultipleOutputs supports counters, by default the are disabled. + The counters group is the {@link MultipleOutputs} class name. +

    + The names of the counters are the same as the named outputs. For multi + named outputs the name of the counter is the concatenation of the named + output, and underscore '_' and the multiname. + + @param conf job conf to enableadd the named output. + @param enabled indicates if the counters will be enabled or not.]]> +     +     + + + + + By default these counters are disabled. +

    + MultipleOutputs supports counters, by default the are disabled. + The counters group is the {@link MultipleOutputs} class name. +

    + The names of the counters are the same as the named outputs. For multi + named outputs the name of the counter is the concatenation of the named + output, and underscore '_' and the multiname. + + + @param conf job conf to enableadd the named output. + @return TRUE if the counters are enabled, FALSE if they are disabled.]]> +     +     + + + + + + + + + + + + + + + + + + + + + + + + + + + If overriden subclasses must invoke super.close() at the + end of their close() + + @throws java.io.IOException thrown if any of the MultipleOutput files + could not be closed properly.]]> + + + + OutputCollector passed to + the map() and reduce() methods of the + Mapper and Reducer implementations. +

    + Each additional output, or named output, may be configured with its own + OutputFormat, with its own key class and with its own value + class. +

    + A named output can be a single file or a multi file. The later is referred as + a multi named output. +

    + A multi named output is an unbound set of files all sharing the same + OutputFormat, key class and value class configuration. +

    + When named outputs are used within a Mapper implementation, + key/values written to a name output are not part of the reduce phase, only + key/values written to the job OutputCollector are part of the + reduce phase. +

    + MultipleOutputs supports counters, by default the are disabled. The counters + group is the {@link MultipleOutputs} class name. +

    + The names of the counters are the same as the named outputs. For multi + named outputs the name of the counter is the concatenation of the named + output, and underscore '_' and the multiname. +

    + Job configuration usage pattern is: + 

    +
+ JobConf conf = new JobConf();
+
+ conf.setInputPath(inDir);
+ FileOutputFormat.setOutputPath(conf, outDir);
+
+ conf.setMapperClass(MOMap.class);
+ conf.setReducerClass(MOReduce.class);
+ ...
+
+ // Defines additional single text based output 'text' for the job
+ MultipleOutputs.addNamedOutput(conf, "text", TextOutputFormat.class,
+ LongWritable.class, Text.class);
+
+ // Defines additional multi sequencefile based output 'sequence' for the
+ // job
+ MultipleOutputs.addMultiNamedOutput(conf, "seq",
+   SequenceFileOutputFormat.class,
+   LongWritable.class, Text.class);
+ ...
+
+ JobClient jc = new JobClient();
+ RunningJob job = jc.submitJob(conf);
+
+ ...
+
    +

    + Job configuration usage pattern is: + 

    +
+ public class MOReduce implements
+   Reducer<WritableComparable, Writable> {
+ private MultipleOutputs mos;
+
+ public void configure(JobConf conf) {
+ ...
+ mos = new MultipleOutputs(conf);
+ }
+
+ public void reduce(WritableComparable key, Iterator<Writable> values,
+ OutputCollector output, Reporter reporter)
+ throws IOException {
+ ...
+ mos.getCollector("text", reporter).collect(key, new Text("Hello"));
+ mos.getCollector("seq", "A", reporter).collect(key, new Text("Bye"));
+ mos.getCollector("seq", "B", reporter).collect(key, new Text("Chau"));
+ ...
+ }
+
+ public void close() throws IOException {
+ mos.close();
+ ...
+ }
+
+ }
+
    ]]> +     +     + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + It can be used instead of the default implementation, + of {@link org.apache.hadoop.mapred.MapRunner}, when the Map + operation is not CPU bound in order to improve throughput. +

    + Map implementations using this MapRunnable must be thread-safe. +

    + The Map-Reduce job has to be configured to use this MapRunnable class (using + the JobConf.setMapRunnerClass method) and + the number of threads the thread-pool can use with the + mapred.map.multithreadedrunner.threads property, its default + value is 10 threads. +

    ]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + R reduces, there are R-1 + keys in the SequenceFile. + @deprecated Use + {@link #setPartitionFile(Configuration, Path)} + instead]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Cluster. + @throws IOException]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + ClusterMetrics provides clients with information such as: +

      +
    1. + Size of the cluster. +
      2. +
    2. + Number of blacklisted and decommissioned trackers. +
      3. +
    3. + Slot capacity of the cluster. +
      4. +
    4. + The number of currently occupied/reserved map and reduce slots. +
      5. +
    5. + The number of currently running map and reduce tasks. +
      6. +
    6. + The number of job submissions. +
      7. +
    + +

    Clients can query for the latest ClusterMetrics, via + {@link Cluster#getClusterStatus()}.

    + + @see Cluster]]> +     +     + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Counters represent global counters, defined either by the + Map-Reduce framework or applications. Each Counter is named by + an {@link Enum} and has a long for the value.

    + +

    Counters are bunched into Groups, each comprising of + counters from a particular Enum class.]]> + + + + + + + + + + + + + + + + + + + + + the type of counter + @param the type of counter group + @param counters the old counters object]]> + + + + Counters holds per job/task counters, defined either by the + Map-Reduce framework or applications. Each Counter can be of + any {@link Enum} type.

    + +

    Counters are bunched into {@link CounterGroup}s, each + comprising of counters from a particular Enum class.]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Each {@link InputSplit} is then assigned to an individual {@link Mapper} + for processing.

    + +

    Note: The split is a logical split of the inputs and the + input files are not physically split into chunks. For e.g. a split could + be <input-file-path, start, offset> tuple. The InputFormat + also creates the {@link RecordReader} to read the {@link InputSplit}. + + @param context job configuration. + @return an array of {@link InputSplit}s for the job.]]> + + + + + + + + + + + + + InputFormat describes the input-specification for a + Map-Reduce job. + +

    The Map-Reduce framework relies on the InputFormat of the + job to:

    +

      +
    1. + Validate the input-specification of the job. +
    2. + Split-up the input file(s) into logical {@link InputSplit}s, each of + which is then assigned to an individual {@link Mapper}. +
      3. +
    3. + Provide the {@link RecordReader} implementation to be used to glean + input records from the logical InputSplit for processing by + the {@link Mapper}. +
      4. +
    + +

    The default behavior of file-based {@link InputFormat}s, typically + sub-classes of {@link FileInputFormat}, is to split the + input into logical {@link InputSplit}s based on the total size, in + bytes, of the input files. However, the {@link FileSystem} blocksize of + the input files is treated as an upper bound for input splits. A lower bound + on the split size can be set via + + mapreduce.input.fileinputformat.split.minsize.

    + +

    Clearly, logical splits based on input-size is insufficient for many + applications since record boundaries are to respected. In such cases, the + application has to also implement a {@link RecordReader} on whom lies the + responsibility to respect record-boundaries and present a record-oriented + view of the logical InputSplit to the individual task. + + @see InputSplit + @see RecordReader + @see FileInputFormat]]> + + + + + + + + + + + + + + + + + + + + + + + + + SplitLocationInfos describing how the split + data is stored at each location. A null value indicates that all the + locations have the data stored on disk. + @throws IOException]]> + + + + InputSplit represents the data to be processed by an + individual {@link Mapper}. + +

    Typically, it presents a byte-oriented view on the input and is the + responsibility of {@link RecordReader} of the job to process this and present + a record-oriented view. + + @see InputFormat + @see RecordReader]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Job makes a copy of the Configuration so + that any necessary internal modifications do not reflect on the incoming + parameter. + + A Cluster will be created from the conf parameter only when it's needed. + + @param conf the configuration + @return the {@link Job} , with no connection to a cluster yet. + @throws IOException]]> + + + + + + + + Job makes a copy of the Configuration so + that any necessary internal modifications do not reflect on the incoming + parameter. + + @param conf the configuration + @return the {@link Job} , with no connection to a cluster yet. + @throws IOException]]> + + + + + + + + Job makes a copy of the Configuration so + that any necessary internal modifications do not reflect on the incoming + parameter. + + @param status job status + @param conf job configuration + @return the {@link Job} , with no connection to a cluster yet. + @throws IOException]]> + + + + + + + Job makes a copy of the Configuration so + that any necessary internal modifications do not reflect on the incoming + parameter. + + @param ignored + @return the {@link Job} , with no connection to a cluster yet. + @throws IOException + @deprecated Use {@link #getInstance()}]]> + + + + + + + + Job makes a copy of the Configuration so + that any necessary internal modifications do not reflect on the incoming + parameter. + + @param ignored + @param conf job configuration + @return the {@link Job} , with no connection to a cluster yet. + @throws IOException + @deprecated Use {@link #getInstance(Configuration)}]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + progress of the job's map-tasks, as a float between 0.0 + and 1.0. When all map tasks have completed, the function returns 1.0. + + @return the progress of the job's map-tasks. + @throws IOException]]> + + + + + + progress of the job's reduce-tasks, as a float between 0.0 + and 1.0. When all reduce tasks have completed, the function returns 1.0. + + @return the progress of the job's reduce-tasks. + @throws IOException]]> + + + + + + + progress of the job's cleanup-tasks, as a float between 0.0 + and 1.0. When all cleanup tasks have completed, the function returns 1.0. + + @return the progress of the job's cleanup-tasks. + @throws IOException]]> + + + + + + progress of the job's setup-tasks, as a float between 0.0 + and 1.0. When all setup tasks have completed, the function returns 1.0. + + @return the progress of the job's setup-tasks. + @throws IOException]]> + + + + + + true if the job is complete, else false. + @throws IOException]]> + + + + + + true if the job succeeded, else false. + @throws IOException]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + InputFormat to use + @throws IllegalStateException if the job is submitted]]> + + + + + + + OutputFormat to use + @throws IllegalStateException if the job is submitted]]> + + + + + + + Mapper to use + @throws IllegalStateException if the job is submitted]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Reducer to use + @throws IllegalStateException if the job is submitted]]> + + + + + + + Partitioner to use + @throws IllegalStateException if the job is submitted]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + true if speculative execution + should be turned on, else false.]]> + + + + + + true if speculative execution + should be turned on for map tasks, + else false.]]> + + + + + + true if speculative execution + should be turned on for reduce tasks, + else false.]]> + + + + + + true, job-setup and job-cleanup will be + considered from {@link OutputCommitter} + else ignored.]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + JobTracker is lost]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Job. + @throws IOException if fail to close.]]> + + + + + + + + + + + + + + + + + + + + + + + + + + It allows the user to configure the + job, submit it, control its execution, and query the state. The set methods + only work until the job is submitted, afterwards they will throw an + IllegalStateException.

    + +

    + Normally the user creates the application, describes various facets of the + job via {@link Job} and then submits the job and monitor its progress.

    + +

    Here is an example on how to submit a job:

    + 

    +     // Create a new Job
+     Job job = Job.getInstance();
+     job.setJarByClass(MyJob.class);
+     
+     // Specify various job-specific parameters     
+     job.setJobName("myjob");
+     
+     job.setInputPath(new Path("in"));
+     job.setOutputPath(new Path("out"));
+     
+     job.setMapperClass(MyJob.MyMapper.class);
+     job.setReducerClass(MyJob.MyReducer.class);
+
+     // Submit the job, then poll for progress until the job is complete
+     job.waitForCompletion(true);
+
    ]]> +     + + + + + + + + + + + + + + + + + + + + + + + 1. + @return the number of reduce tasks for this job.]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + mapred.map.max.attempts + property. If this property is not already set, the default is 4 attempts. + + @return the max number of attempts per map task.]]> + + + + + mapred.reduce.max.attempts + property. If this property is not already set, the default is 4 attempts. + + @return the max number of attempts per reduce task.]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + An example JobID is : + job_200707121733_0003 , which represents the third job + running at the jobtracker started at 200707121733. +

    + Applications should never construct or parse JobID strings, but rather + use appropriate constructors or {@link #forName(String)} method. + + @see TaskID + @see TaskAttemptID]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + the key input type to the Mapper + @param the value input type to the Mapper + @param the key output type from the Mapper + @param the value output type from the Mapper]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Maps are the individual tasks which transform input records into a + intermediate records. The transformed intermediate records need not be of + the same type as the input records. A given input pair may map to zero or + many output pairs.

    + +

    The Hadoop Map-Reduce framework spawns one map task for each + {@link InputSplit} generated by the {@link InputFormat} for the job. + Mapper implementations can access the {@link Configuration} for + the job via the {@link JobContext#getConfiguration()}. + +

    The framework first calls + {@link #setup(org.apache.hadoop.mapreduce.Mapper.Context)}, followed by + {@link #map(Object, Object, org.apache.hadoop.mapreduce.Mapper.Context)} + for each key/value pair in the InputSplit. Finally + {@link #cleanup(org.apache.hadoop.mapreduce.Mapper.Context)} is called.

    + +

    All intermediate values associated with a given output key are + subsequently grouped by the framework, and passed to a {@link Reducer} to + determine the final output. Users can control the sorting and grouping by + specifying two key {@link RawComparator} classes.

    + +

    The Mapper outputs are partitioned per + Reducer. Users can control which keys (and hence records) go to + which Reducer by implementing a custom {@link Partitioner}. + +

    Users can optionally specify a combiner, via + {@link Job#setCombinerClass(Class)}, to perform local aggregation of the + intermediate outputs, which helps to cut down the amount of data transferred + from the Mapper to the Reducer. + +

    Applications can specify if and how the intermediate + outputs are to be compressed and which {@link CompressionCodec}s are to be + used via the Configuration.

    + +

    If the job has zero + reduces then the output of the Mapper is directly written + to the {@link OutputFormat} without sorting by keys.

    + +

    Example:

    + 

    + public class TokenCounterMapper 
+     extends Mapper<Object, Text, Text, IntWritable>{
+    
+   private final static IntWritable one = new IntWritable(1);
+   private Text word = new Text();
+   
+   public void map(Object key, Text value, Context context) throws IOException, InterruptedException {
+     StringTokenizer itr = new StringTokenizer(value.toString());
+     while (itr.hasMoreTokens()) {
+       word.set(itr.nextToken());
+       context.write(word, one);
+     }
+   }
+ }
+
    + +

    Applications may override the + {@link #run(org.apache.hadoop.mapreduce.Mapper.Context)} method to exert + greater control on map processing e.g. multi-threaded Mappers + etc.

    + + @see InputFormat + @see JobContext + @see Partitioner + @see Reducer]]> +     +     + + + + + + + + + + + + + + + + + + + + + + + + + + MarkableIterator is a wrapper iterator class that + implements the {@link MarkableIteratorInterface}.]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + true if task output recovery is supported, + false otherwise + @see #recoverTask(TaskAttemptContext) + @deprecated Use {@link #isRecoverySupported(JobContext)} instead.]]> + + + + + + + true repeatable job commit is supported, + false otherwise + @throws IOException]]> + + + + + + + true if task output recovery is supported, + false otherwise + @throws IOException + @see #recoverTask(TaskAttemptContext)]]> + + + + + + + OutputCommitter. This is called from the application master + process, but it is called individually for each task. + + If an exception is thrown the task will be attempted again. + + This may be called multiple times for the same task. But from different + application attempts. + + @param taskContext Context of the task whose output is being recovered + @throws IOException]]> + + + + OutputCommitter describes the commit of task output for a + Map-Reduce job. + +

    The Map-Reduce framework relies on the OutputCommitter of + the job to:

    +

      +
    1. + Setup the job during initialization. For example, create the temporary + output directory for the job during the initialization of the job. +
      2. +
    2. + Cleanup the job after the job completion. For example, remove the + temporary output directory after the job completion. +
      3. +
    3. + Setup the task temporary output. +
      4. +
    4. + Check whether a task needs a commit. This is to avoid the commit + procedure if a task does not need commit. +
      5. +
    5. + Commit of the task output. +
      6. +
    6. + Discard the task commit. +
      7. +
    + The methods in this class can be called from several different processes and + from several different contexts. It is important to know which process and + which context each is called from. Each method should be marked accordingly + in its documentation. It is also important to note that not all methods are + guaranteed to be called once and only once. If a method is not guaranteed to + have this property the output committer needs to handle this appropriately. + Also note it will only be in rare situations where they may be called + multiple times for the same task. + + @see org.apache.hadoop.mapreduce.lib.output.FileOutputCommitter + @see JobContext + @see TaskAttemptContext]]> +     +     + + + + + + + + + + + + + + + + + + + This is to validate the output specification for the job when it is + a job is submitted. Typically checks that it does not already exist, + throwing an exception when it already exists, so that output is not + overwritten.

    + + Implementations which write to filesystems which support delegation + tokens usually collect the tokens for the destination path(s) + and attach them to the job context's JobConf. + @param context information about the job + @throws IOException when output should not be attempted]]> +     +     + + + + + + + + + + OutputFormat describes the output-specification for a + Map-Reduce job. + +

    The Map-Reduce framework relies on the OutputFormat of the + job to:

    +

      +
    1. + Validate the output-specification of the job. For e.g. check that the + output directory doesn't already exist. +
    2. + Provide the {@link RecordWriter} implementation to be used to write out + the output files of the job. Output files are stored in a + {@link FileSystem}. +
      3. +
    + + @see RecordWriter]]> +     +     + + + + + + + + + + + Typically a hash function on a all or a subset of the key.

    + + @param key the key to be partioned. + @param value the entry value. + @param numPartitions the total number of partitions. + @return the partition number for the key.]]> +     +     + + Partitioner controls the partitioning of the keys of the + intermediate map-outputs. The key (or a subset of the key) is used to derive + the partition, typically by a hash function. The total number of partitions + is the same as the number of reduce tasks for the job. Hence this controls + which of the m reduce tasks the intermediate key (and hence the + record) is sent for reduction.

    + +

    Note: A Partitioner is created only when there are multiple + reducers.

    + +

    Note: If you require your Partitioner class to obtain the Job's + configuration object, implement the {@link Configurable} interface.

    + + @see Reducer]]> +     +     + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + "N/A" + + @return Scheduling information associated to particular Job Queue]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + @param ]]> + + + + + + + + + + + + + + + + + + + + + + RecordWriter to future operations. + + @param context the context of the task + @throws IOException]]> + + + + RecordWriter writes the output <key, value> pairs + to an output file. + +

    RecordWriter implementations write the job outputs to the + {@link FileSystem}. + + @see OutputFormat]]> + + + + + + + + + + + + + + + + + + + + + + the class of the input keys + @param the class of the input values + @param the class of the output keys + @param the class of the output values]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Reducer implementations + can access the {@link Configuration} for the job via the + {@link JobContext#getConfiguration()} method.

    + +

    Reducer has 3 primary phases:

    +
      +
    1. + + Shuffle + +

      The Reducer copies the sorted output from each + {@link Mapper} using HTTP across the network.

      +
      2. + +
    2. + Sort + +

      The framework merge sorts Reducer inputs by + keys + (since different Mappers may have output the same key).

      + +

      The shuffle and sort phases occur simultaneously i.e. while outputs are + being fetched they are merged.

      + + SecondarySort + +

      To achieve a secondary sort on the values returned by the value + iterator, the application should extend the key with the secondary + key and define a grouping comparator. The keys will be sorted using the + entire key, but will be grouped using the grouping comparator to decide + which keys and values are sent in the same call to reduce.The grouping + comparator is specified via + {@link Job#setGroupingComparatorClass(Class)}. The sort order is + controlled by + {@link Job#setSortComparatorClass(Class)}.

      + + + For example, say that you want to find duplicate web pages and tag them + all with the url of the "best" known example. You would set up the job + like: +
        +
      • Map Input Key: url
        • +
      • Map Input Value: document
        • +
      • Map Output Key: document checksum, url pagerank
        • +
      • Map Output Value: url
        • +
      • Partitioner: by checksum
        • +
      • OutputKeyComparator: by checksum and then decreasing pagerank
        • +
      • OutputValueGroupingComparator: by checksum
        • +
      +
      3. + +
    3. + Reduce + +

      In this phase the + {@link #reduce(Object, Iterable, org.apache.hadoop.mapreduce.Reducer.Context)} + method is called for each <key, (collection of values)> in + the sorted inputs.

      +

      The output of the reduce task is typically written to a + {@link RecordWriter} via + {@link Context#write(Object, Object)}.

      +
      4. +
    + +

    The output of the Reducer is not re-sorted.

    + +

    Example:

    + 

    + public class IntSumReducer<Key> extends Reducer<Key,IntWritable,
+                                                 Key,IntWritable> {
+   private IntWritable result = new IntWritable();
+ 
+   public void reduce(Key key, Iterable<IntWritable> values,
+                      Context context) throws IOException, InterruptedException {
+     int sum = 0;
+     for (IntWritable val : values) {
+       sum += val.get();
+     }
+     result.set(sum);
+     context.write(key, result);
+   }
+ }
+
    + + @see Mapper + @see Partitioner]]> +     +     + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + counterName. + @param counterName counter name + @return the Counter for the given counterName]]> + + + + + + + groupName and + counterName. + @param counterName counter name + @return the Counter for the given groupName and + counterName]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + An example TaskAttemptID is : + attempt_200707121733_0003_m_000005_0 , which represents the + zeroth task attempt for the fifth map task in the third job + running at the jobtracker started at 200707121733. +

    + Applications should never construct or parse TaskAttemptID strings + , but rather use appropriate constructors or {@link #forName(String)} + method. + + @see JobID + @see TaskID]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + An example TaskID is : + task_200707121733_0003_m_000005 , which represents the + fifth map task in the third job running at the jobtracker + started at 200707121733. +

    + Applications should never construct or parse TaskID strings + , but rather use appropriate constructors or {@link #forName(String)} + method. + + @see JobID + @see TaskAttemptID]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + OutputCommitter for the task-attempt]]> + + + + the input key type for the task + @param the input value type for the task + @param the output key type for the task + @param the output value type for the task]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + type of the other counter + @param type of the other counter group + @param counters the counters object to copy + @param groupFactory the factory for new groups]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + type of counter inside the counters + @param type of group inside the counters]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + type of the counter for the group]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + The key and values are passed from one element of the chain to the next, by + value. For the added Mapper the configuration given for it, + mapperConf, have precedence over the job's Configuration. This + precedence is in effect when the task is running. +

    +

    + IMPORTANT: There is no need to specify the output key/value classes for the + ChainMapper, this is done by the addMapper for the last mapper in the chain +

    + + @param job + The job. + @param klass + the Mapper class to add. + @param inputKeyClass + mapper input key class. + @param inputValueClass + mapper input value class. + @param outputKeyClass + mapper output key class. + @param outputValueClass + mapper output value class. + @param mapperConf + a configuration for the Mapper class. It is recommended to use a + Configuration without default values using the + Configuration(boolean loadDefaults) constructor with + FALSE.]]> +     + + + + + + + + + + + + The Mapper classes are invoked in a chained (or piped) fashion, the output of + the first becomes the input of the second, and so on until the last Mapper, + the output of the last Mapper will be written to the task's output. +

    +

    + The key functionality of this feature is that the Mappers in the chain do not + need to be aware that they are executed in a chain. This enables having + reusable specialized Mappers that can be combined to perform composite + operations within a single task. +

    +

    + Special care has to be taken when creating chains that the key/values output + by a Mapper are valid for the following Mapper in the chain. It is assumed + all Mappers and the Reduce in the chain use matching output and input key and + value classes as no conversion is done by the chaining code. +

    +

    + Using the ChainMapper and the ChainReducer classes is possible to compose + Map/Reduce jobs that look like [MAP+ / REDUCE MAP*]. And + immediate benefit of this pattern is a dramatic reduction in disk IO. +

    +

    + IMPORTANT: There is no need to specify the output key/value classes for the + ChainMapper, this is done by the addMapper for the last mapper in the chain. +

    + ChainMapper usage pattern: +

    + + 

    + ...
+ Job = new Job(conf);
+
+ Configuration mapAConf = new Configuration(false);
+ ...
+ ChainMapper.addMapper(job, AMap.class, LongWritable.class, Text.class,
+   Text.class, Text.class, true, mapAConf);
+
+ Configuration mapBConf = new Configuration(false);
+ ...
+ ChainMapper.addMapper(job, BMap.class, Text.class, Text.class,
+   LongWritable.class, Text.class, false, mapBConf);
+
+ ...
+
+ job.waitForComplettion(true);
+ ...
+
    ]]> +     +     + + + + + + + + + + + + + + + + The key and values are passed from one element of the chain to the next, by + value. For the added Reducer the configuration given for it, + reducerConf, have precedence over the job's Configuration. + This precedence is in effect when the task is running. +

    +

    + IMPORTANT: There is no need to specify the output key/value classes for the + ChainReducer, this is done by the setReducer or the addMapper for the last + element in the chain. +

    + + @param job + the job + @param klass + the Reducer class to add. + @param inputKeyClass + reducer input key class. + @param inputValueClass + reducer input value class. + @param outputKeyClass + reducer output key class. + @param outputValueClass + reducer output value class. + @param reducerConf + a configuration for the Reducer class. It is recommended to use a + Configuration without default values using the + Configuration(boolean loadDefaults) constructor with + FALSE.]]> +     +     + + + + + + + + + + + + The key and values are passed from one element of the chain to the next, by + value For the added Mapper the configuration given for it, + mapperConf, have precedence over the job's Configuration. This + precedence is in effect when the task is running. +

    +

    + IMPORTANT: There is no need to specify the output key/value classes for the + ChainMapper, this is done by the addMapper for the last mapper in the + chain. +

    + + @param job + The job. + @param klass + the Mapper class to add. + @param inputKeyClass + mapper input key class. + @param inputValueClass + mapper input value class. + @param outputKeyClass + mapper output key class. + @param outputValueClass + mapper output value class. + @param mapperConf + a configuration for the Mapper class. It is recommended to use a + Configuration without default values using the + Configuration(boolean loadDefaults) constructor with + FALSE.]]> +     +     + + + + + + + + + + + For each record output by the Reducer, the Mapper classes are invoked in a + chained (or piped) fashion. The output of the reducer becomes the input of + the first mapper and output of first becomes the input of the second, and so + on until the last Mapper, the output of the last Mapper will be written to + the task's output. +

    +

    + The key functionality of this feature is that the Mappers in the chain do not + need to be aware that they are executed after the Reducer or in a chain. This + enables having reusable specialized Mappers that can be combined to perform + composite operations within a single task. +

    +

    + Special care has to be taken when creating chains that the key/values output + by a Mapper are valid for the following Mapper in the chain. It is assumed + all Mappers and the Reduce in the chain use matching output and input key and + value classes as no conversion is done by the chaining code. +

    +

    Using the ChainMapper and the ChainReducer classes is possible to + compose Map/Reduce jobs that look like [MAP+ / REDUCE MAP*]. And + immediate benefit of this pattern is a dramatic reduction in disk IO.

    +

    + IMPORTANT: There is no need to specify the output key/value classes for the + ChainReducer, this is done by the setReducer or the addMapper for the last + element in the chain. +

    + ChainReducer usage pattern: +

    + + 

    + ...
+ Job = new Job(conf);
+ ....
+
+ Configuration reduceConf = new Configuration(false);
+ ...
+ ChainReducer.setReducer(job, XReduce.class, LongWritable.class, Text.class,
+   Text.class, Text.class, true, reduceConf);
+
+ ChainReducer.addMapper(job, CMap.class, Text.class, Text.class,
+   LongWritable.class, Text.class, false, null);
+
+ ChainReducer.addMapper(job, DMap.class, LongWritable.class, Text.class,
+   LongWritable.class, LongWritable.class, true, null);
+
+ ...
+
+ job.waitForCompletion(true);
+ ...
+
    ]]> +     +     + +     + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + DBInputFormat emits LongWritables containing the record number as + key and DBWritables as value. + + The SQL query, and input class can be using one of the two + setInput methods.]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + {@link DBOutputFormat} accepts <key,value> pairs, where + key has a type extending DBWritable. Returned {@link RecordWriter} + writes only the key to the database with a batch SQL query.]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + DBWritable. DBWritable, is similar to {@link Writable} + except that the {@link #write(PreparedStatement)} method takes a + {@link PreparedStatement}, and {@link #readFields(ResultSet)} + takes a {@link ResultSet}. +

    + Implementations are responsible for writing the fields of the object + to PreparedStatement, and reading the fields of the object from the + ResultSet. + +

    Example:

    + If we have the following table in the database : + 
    + CREATE TABLE MyTable (
+   counter        INTEGER NOT NULL,
+   timestamp      BIGINT  NOT NULL,
+ );
+
    + then we can read/write the tuples from/to the table with : + 

    + public class MyWritable implements Writable, DBWritable {
+   // Some data     
+   private int counter;
+   private long timestamp;
+       
+   //Writable#write() implementation
+   public void write(DataOutput out) throws IOException {
+     out.writeInt(counter);
+     out.writeLong(timestamp);
+   }
+       
+   //Writable#readFields() implementation
+   public void readFields(DataInput in) throws IOException {
+     counter = in.readInt();
+     timestamp = in.readLong();
+   }
+       
+   public void write(PreparedStatement statement) throws SQLException {
+     statement.setInt(1, counter);
+     statement.setLong(2, timestamp);
+   }
+       
+   public void readFields(ResultSet resultSet) throws SQLException {
+     counter = resultSet.getInt(1);
+     timestamp = resultSet.getLong(2);
+   } 
+ }
+
    ]]> +     +     + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +     + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + RecordReader's for + CombineFileSplit's. + + @see CombineFileSplit]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + CombineFileRecordReader. + + Subclassing is needed to get a concrete record reader wrapper because of the + constructor requirement. + + @see CombineFileRecordReader + @see CombineFileInputFormat]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + th Path]]> + + + + + + th Path]]> + + + + + + + + + + + th Path]]> + + + + + + + + + + + + + + + + + + + + + + + + + + CombineFileSplit can be used to implement {@link RecordReader}'s, + with reading one record per file. + + @see FileSplit + @see CombineFileInputFormat]]> + + + + + + + + + + + + + + CombineFileInputFormat-equivalent for + SequenceFileInputFormat. + + @see CombineFileInputFormat]]> + + + + + + + + + + + + + + CombineFileInputFormat-equivalent for + TextInputFormat. + + @see CombineFileInputFormat]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + FileInputFormat always returns + true. Implementations that may deal with non-splittable files must + override this method. + + FileInputFormat implementations can override this and return + false to ensure that individual input files are never split-up + so that {@link Mapper}s process entire files. + + @param context the job context + @param filename the file name to check + @return is this file splitable?]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + FileInputFormat is the base class for all file-based + InputFormats. This provides a generic implementation of + {@link #getSplits(JobContext)}. + + Implementations of FileInputFormat can also override the + {@link #isSplitable(JobContext, Path)} method to prevent input files + from being split-up in certain situations. Implementations that may + deal with non-splittable files must override this method, since + the default implementation assumes splitting is always possible.]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
    or + conf.setInt(FixedLengthInputFormat.FIXED_RECORD_LENGTH, recordLength); +

    + @see FixedLengthRecordReader]]> +     +     + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +     + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + true if the Job was added.]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + ([,]*) + func ::= tbl(,"") + class ::= @see java.lang.Class#forName(java.lang.String) + path ::= @see org.apache.hadoop.fs.Path#Path(java.lang.String) + } + Reads expression from the mapreduce.join.expr property and + user-supplied join types from mapreduce.join.define.<ident> + types. Paths supplied to tbl are given as input paths to the + InputFormat class listed. + @see #compose(java.lang.String, java.lang.Class, java.lang.String...)]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + ,

    ) }]]> + + + + + + + + (tbl(,),tbl(,),...,tbl(,)) }]]> + + + + + + + + (tbl(,),tbl(,),...,tbl(,)) }]]> + + + + + + + + mapreduce.join.define.<ident> to a classname. + In the expression mapreduce.join.expr, the identifier will be + assumed to be a ComposableRecordReader. + mapreduce.join.keycomparator can be a classname used to compare + keys in the join. + @see #setFormat + @see JoinRecordReader + @see MultiFilterRecordReader]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + ...... + }]]> + + + + + + + + + + + + + + + + + + + + + capacity children to position + id in the parent reader. + The id of a root CompositeRecordReader is -1 by convention, but relying + on this is not recommended.]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + override(S1,S2,S3) will prefer values + from S3 over S2, and values from S2 over S1 for all keys + emitted from all sources.]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + [<child1>,<child2>,...,<childn>]]]> + + + + + + + out. + TupleWritable format: + {@code + ...... + }]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + the map's input key type + @param the map's input value type + @param the map's output key type + @param the map's output value type + @param job the job + @return the mapper class to run]]> + + + + + + + the map input key type + @param the map input value type + @param the map output key type + @param the map output value type + @param job the job to modify + @param cls the class to use as the mapper]]> + + + + + + + + + + + + + + + + + It can be used instead of the default implementation, + {@link org.apache.hadoop.mapred.MapRunner}, when the Map operation is not CPU + bound in order to improve throughput. +

    + Mapper implementations using this MapRunnable must be thread-safe. +

    + The Map-Reduce job has to be configured with the mapper to use via + {@link #setMapperClass(Job, Class)} and + the number of thread the thread-pool can use with the + {@link #getNumberOfThreads(JobContext)} method. The default + value is 10 threads. +

    ]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + MapContext to be wrapped + @return a wrapped Mapper.Context for custom implementations]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +

  • + In applications which take a classname of committer in + a configuration option, set it to the canonical name of this class + (see {@link #NAME}). When this class is instantiated, it will + use the factory mechanism to locate the configured committer for the + destination. +
    • +
  • + In code, explicitly create an instance of this committer through + its constructor, then invoke commit lifecycle operations on it. + The dynamically configured committer will be created in the constructor + and have the lifecycle operations relayed to it. +
    • + ]]> +     +     + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + true if the job output should be compressed, + false otherwise]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Tasks' Side-Effect Files + +

    Some applications need to create/write-to side-files, which differ from + the actual job-outputs. + +

    In such cases there could be issues with 2 instances of the same TIP + (running simultaneously e.g. speculative tasks) trying to open/write-to the + same file (path) on HDFS. Hence the application-writer will have to pick + unique names per task-attempt (e.g. using the attemptid, say + attempt_200709221812_0001_m_000000_0), not just per TIP.

    + +

    To get around this the Map-Reduce framework helps the application-writer + out by maintaining a special + ${mapreduce.output.fileoutputformat.outputdir}/_temporary/_${taskid} + sub-directory for each task-attempt on HDFS where the output of the + task-attempt goes. On successful completion of the task-attempt the files + in the ${mapreduce.output.fileoutputformat.outputdir}/_temporary/_${taskid} (only) + are promoted to ${mapreduce.output.fileoutputformat.outputdir}. Of course, the + framework discards the sub-directory of unsuccessful task-attempts. This + is completely transparent to the application.

    + +

    The application-writer can take advantage of this by creating any + side-files required in a work directory during execution + of his task i.e. via + {@link #getWorkOutputPath(TaskInputOutputContext)}, and + the framework will move them out similarly - thus she doesn't have to pick + unique paths per task-attempt.

    + +

    The entire discussion holds true for maps of jobs with + reducer=NONE (i.e. 0 reduces) since output of the map, in that case, + goes directly to HDFS.

    + + @return the {@link Path} to the task's temporary output directory + for the map-reduce job.]]> +     +     + + + + + + + + The path can be used to create custom files from within the map and + reduce tasks. The path name will be unique for each task. The path parent + will be the job output directory.

    ls + +

    This method uses the {@link #getUniqueFile} method to make the file name + unique for the task.

    + + @param context the context for the task. + @param name the name for the file. + @param extension the extension for the file + @return a unique path accross all tasks of the job.]]> +     +     + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +     + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Warning: when the baseOutputPath is a path that resolves + outside of the final job output directory, the directory is created + immediately and then persists through subsequent task retries, breaking + the concept of output committing.]]> + + + + + + + + + + Warning: when the baseOutputPath is a path that resolves + outside of the final job output directory, the directory is created + immediately and then persists through subsequent task retries, breaking + the concept of output committing.]]> + + + + + + + super.close() at the + end of their close()]]> + + + + + Case one: writing to additional outputs other than the job default output. + + Each additional output, or named output, may be configured with its own + OutputFormat, with its own key class and with its own value + class. +

    + +

    + Case two: to write data to different files provided by user +

    + +

    + MultipleOutputs supports counters, by default they are disabled. The + counters group is the {@link MultipleOutputs} class name. The names of the + counters are the same as the output name. These count the number records + written to each output name. +

    + + Usage pattern for job submission: + 
    +
+ Job job = new Job();
+
+ FileInputFormat.setInputPath(job, inDir);
+ FileOutputFormat.setOutputPath(job, outDir);
+
+ job.setMapperClass(MOMap.class);
+ job.setReducerClass(MOReduce.class);
+ ...
+
+ // Defines additional single text based output 'text' for the job
+ MultipleOutputs.addNamedOutput(job, "text", TextOutputFormat.class,
+ LongWritable.class, Text.class);
+
+ // Defines additional sequence-file based output 'sequence' for the job
+ MultipleOutputs.addNamedOutput(job, "seq",
+   SequenceFileOutputFormat.class,
+   LongWritable.class, Text.class);
+ ...
+
+ job.waitForCompletion(true);
+ ...
+
    +

    + Usage in Reducer: + 

    + <K, V> String generateFileName(K k, V v) {
+   return k.toString() + "_" + v.toString();
+ }
+ 
+ public class MOReduce extends
+   Reducer<WritableComparable, Writable,WritableComparable, Writable> {
+ private MultipleOutputs mos;
+ public void setup(Context context) {
+ ...
+ mos = new MultipleOutputs(context);
+ }
+
+ public void reduce(WritableComparable key, Iterator<Writable> values,
+ Context context)
+ throws IOException {
+ ...
+ mos.write("text", , key, new Text("Hello"));
+ mos.write("seq", LongWritable(1), new Text("Bye"), "seq_a");
+ mos.write("seq", LongWritable(2), key, new Text("Chau"), "seq_b");
+ mos.write(key, new Text("value"), generateFileName(key, new Text("value")));
+ ...
+ }
+
+ public void cleanup(Context) throws IOException {
+ mos.close();
+ ...
+ }
+
+ }
+
    + +

    + When used in conjuction with org.apache.hadoop.mapreduce.lib.output.LazyOutputFormat, + MultipleOutputs can mimic the behaviour of MultipleTextOutputFormat and MultipleSequenceFileOutputFormat + from the old Hadoop API - ie, output can be written from the Reducer to more than one location. +

    + +

    + Use MultipleOutputs.write(KEYOUT key, VALUEOUT value, String baseOutputPath) to write key and + value to a path specified by baseOutputPath, with no need to specify a named output. + Warning: when the baseOutputPath passed to MultipleOutputs.write + is a path that resolves outside of the final job output directory, the + directory is created immediately and then persists through subsequent + task retries, breaking the concept of output committing: +

    + + 
    + private MultipleOutputs<Text, Text> out;
+ 
+ public void setup(Context context) {
+   out = new MultipleOutputs<Text, Text>(context);
+   ...
+ }
+ 
+ public void reduce(Text key, Iterable<Text> values, Context context) throws IOException, InterruptedException {
+ for (Text t : values) {
+   out.write(key, t, generateFileName(<parameter list...>));
+   }
+ }
+ 
+ protected void cleanup(Context context) throws IOException, InterruptedException {
+   out.close();
+ }
+
    + +

    + Use your own code in generateFileName() to create a custom path to your results. + '/' characters in baseOutputPath will be translated into directory levels in your file system. + Also, append your custom-generated path with "part" or similar, otherwise your output will be -00000, -00001 etc. + No call to context.write() is necessary. See example generateFileName() code below. +

    + + 
    + private String generateFileName(Text k) {
+   // expect Text k in format "Surname|Forename"
+   String[] kStr = k.toString().split("\\|");
+   
+   String sName = kStr[0];
+   String fName = kStr[1];
+
+   // example for k = Smith|John
+   // output written to /user/hadoop/path/to/output/Smith/John-r-00000 (etc)
+   return sName + "/" + fName;
+ }
+
    + +

    + Using MultipleOutputs in this way will still create zero-sized default output, eg part-00000. + To prevent this use LazyOutputFormat.setOutputFormatClass(job, TextOutputFormat.class); + instead of job.setOutputFormatClass(TextOutputFormat.class); in your Hadoop job configuration. +

    ]]> +     +     + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + This allows the user to specify the key class to be different + from the actual class ({@link BytesWritable}) used for writing

    + + @param job the {@link Job} to modify + @param theClass the SequenceFile output key class.]]> +     +     + + + + + This allows the user to specify the value class to be different + from the actual class ({@link BytesWritable}) used for writing

    + + @param job the {@link Job} to modify + @param theClass the SequenceFile output key class.]]> +     +     + + + + + + + + + + + + + + + + + + + + + + + + + + + +     + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +     + + + + + + + + + + + + bytes[left:(right+1)] in Python syntax. + + @param conf configuration object + @param left left Python-style offset + @param right right Python-style offset]]> + + + + + + + bytes[offset:] in Python syntax. + + @param conf configuration object + @param offset left Python-style offset]]> + + + + + + + bytes[:(offset+1)] in Python syntax. + + @param conf configuration object + @param offset right Python-style offset]]> + + + + + + + + + + + + + + + + + + + + + Partition {@link BinaryComparable} keys using a configurable part of + the bytes array returned by {@link BinaryComparable#getBytes()}.

    + +

    The subarray to be used for the partitioning can be defined by means + of the following properties: +

      +
    • + mapreduce.partition.binarypartitioner.left.offset: + left offset in array (0 by default) +
      • +
    • + mapreduce.partition.binarypartitioner.right.offset: + right offset in array (-1 by default) +
      • +
    + Like in Python, both negative and positive offsets are allowed, but + the meaning is slightly different. In case of an array of length 5, + for instance, the possible offsets are: + 
    
+  +---+---+---+---+---+
+  | B | B | B | B | B |
+  +---+---+---+---+---+
+    0   1   2   3   4
+   -5  -4  -3  -2  -1
+
    + The first row of numbers gives the position of the offsets 0...5 in + the array; the second row gives the corresponding negative offsets. + Contrary to Python, the specified subarray has byte i + and j as first and last element, repectively, when + i and j are the left and right offset. + +

    For Hadoop programs written in Java, it is advisable to use one of + the following static convenience methods for setting the offsets: +

      +
    • {@link #setOffsets}
      • +
    • {@link #setLeftOffset}
      • +
    • {@link #setRightOffset}
      • +
    ]]> +     +     + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + total.order.partitioner.natural.order is not false, a trie + of the first total.order.partitioner.max.trie.depth(2) + 1 bytes + will be built. Otherwise, keys will be located using a binary search of + the partition keyset using the {@link org.apache.hadoop.io.RawComparator} + defined for this job. The input file must be sorted with the same + comparator and contain {@link Job#getNumReduceTasks()} - 1 keys.]]> + + + + + + + + + + + + + + R reduces, there are R-1 + keys in the SequenceFile.]]> + + + + + + + + + + + + + + + + + + + + + +     + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + ReduceContext to be wrapped + @return a wrapped Reducer.Context for custom implementations]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +     diff --git a/hadoop-mapreduce-project/dev-support/jdiff/Apache_Hadoop_MapReduce_JobClient_3.2.4.xml b/hadoop-mapreduce-project/dev-support/jdiff/Apache_Hadoop_MapReduce_JobClient_3.2.4.xml new file mode 100644 index 0000000000000..ed4979ac1fa14 --- /dev/null +++ b/hadoop-mapreduce-project/dev-support/jdiff/Apache_Hadoop_MapReduce_JobClient_3.2.4.xml @@ -0,0 +1,16 @@ + + + + + + + + + + + + diff --git a/hadoop-mapreduce-project/dev-support/jdiff/Apache_Hadoop_MapReduce_JobClient_3.3.4.xml b/hadoop-mapreduce-project/dev-support/jdiff/Apache_Hadoop_MapReduce_JobClient_3.3.4.xml new file mode 100644 index 0000000000000..dd96ac9059277 --- /dev/null +++ b/hadoop-mapreduce-project/dev-support/jdiff/Apache_Hadoop_MapReduce_JobClient_3.3.4.xml @@ -0,0 +1,16 @@ + + + + + + + + + + + + diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java index fc9254af97ab8..9023f6a11ad84 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java @@ -1481,7 +1481,7 @@ protected void processDoneFiles(JobId jobId) throws IOException { summaryFileOut.writeUTF(mi.getJobSummary().getJobSummaryString()); summaryFileOut.close(); doneDirFS.setPermission(qualifiedSummaryDoneFile, new FsPermission( - JobHistoryUtils.HISTORY_INTERMEDIATE_FILE_PERMISSIONS)); + JobHistoryUtils.getConfiguredHistoryIntermediateUserDoneDirPermissions(getConfig()))); } catch (IOException e) { LOG.info("Unable to write out JobSummaryInfo to [" + qualifiedSummaryDoneFile + "]", e); @@ -1738,8 +1738,9 @@ protected boolean moveToDoneNow(Path fromPath, Path toPath) boolean copied = FileUtil.copy(stagingDirFS, fromPath, doneDirFS, toPath, false, getConfig()); - doneDirFS.setPermission(toPath, new FsPermission( - JobHistoryUtils.HISTORY_INTERMEDIATE_FILE_PERMISSIONS)); + doneDirFS.setPermission(toPath, new FsPermission(JobHistoryUtils. + getConfiguredHistoryIntermediateUserDoneDirPermissions( + getConfig()))); if (copied) { LOG.info("Copied from: " + fromPath.toString() + " to done location: " + toPath.toString()); diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/launcher/ContainerLauncherImpl.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/launcher/ContainerLauncherImpl.java index d09b3cb1e56b3..d184d9be64bf8 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/launcher/ContainerLauncherImpl.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/launcher/ContainerLauncherImpl.java @@ -379,27 +379,38 @@ class EventProcessor implements Runnable { @Override public void run() { - LOG.info("Processing the event " + event.toString()); + LOG.info("Processing the event {}", event); // Load ContainerManager tokens before creating a connection. // TODO: Do it only once per NodeManager. ContainerId containerID = event.getContainerID(); - Container c = getContainer(event); switch(event.getType()) { case CONTAINER_REMOTE_LAUNCH: ContainerRemoteLaunchEvent launchEvent = (ContainerRemoteLaunchEvent) event; - c.launch(launchEvent); + getContainer(event).launch(launchEvent); break; case CONTAINER_REMOTE_CLEANUP: - c.kill(event.getDumpContainerThreads()); + // If the container failed to launch earlier (due to dead node for example), + // it has been marked as FAILED and removed from containers during + // CONTAINER_REMOTE_LAUNCH event handling. + // Skip kill() such container during CONTAINER_REMOTE_CLEANUP as + // it is not necessary and could cost 15 minutes delay if the node is dead. + if (!containers.containsKey(containerID)) { + LOG.info("Skip cleanup of already-removed container {}", containerID); + // send killed event to task attempt regardless like in kill(). + context.getEventHandler().handle(new TaskAttemptEvent(event.getTaskAttemptID(), + TaskAttemptEventType.TA_CONTAINER_CLEANED)); + return; + } + getContainer(event).kill(event.getDumpContainerThreads()); break; case CONTAINER_COMPLETED: - c.done(); + getContainer(event).done(); break; } diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/jobhistory/TestJobHistoryEventHandler.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/jobhistory/TestJobHistoryEventHandler.java index fbeb94a2a16ac..8159bc2456ca0 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/jobhistory/TestJobHistoryEventHandler.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/jobhistory/TestJobHistoryEventHandler.java @@ -19,6 +19,7 @@ package org.apache.hadoop.mapreduce.jobhistory; import static org.assertj.core.api.Assertions.assertThat; +import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; import static org.mockito.ArgumentMatchers.any; @@ -43,6 +44,7 @@ import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.LocalFileSystem; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.permission.FsPermission; import org.apache.hadoop.hdfs.HdfsConfiguration; import org.apache.hadoop.hdfs.MiniDFSCluster; import org.apache.hadoop.mapreduce.CounterGroup; @@ -1036,6 +1038,48 @@ public void testDontSetTrackingURLIfHistoryWriteThrows() throws Exception { jheh.stop(); } } + + @Test(timeout = 50000) + public void testJobHistoryFilePermissions() throws Exception { + TestParams t = new TestParams(true); + Configuration conf = new Configuration(); + String setFilePermission = "777"; + conf.set(JHAdminConfig.MR_HISTORY_INTERMEDIATE_USER_DONE_DIR_PERMISSIONS, setFilePermission); + + conf.set(CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY, dfsCluster.getURI().toString()); + + JHEvenHandlerForTest realJheh = new JHEvenHandlerForTest(t.mockAppContext, + 0, false); + JHEvenHandlerForTest jheh = spy(realJheh); + jheh.init(conf); + + try { + jheh.start(); + handleEvent(jheh, new JobHistoryEvent(t.jobId, + new AMStartedEvent(t.appAttemptId, 200, t.containerId, "nmhost", + 3000, 4000, -1))); + + // Job finishes and successfully writes history + handleEvent(jheh, new JobHistoryEvent(t.jobId, + new JobFinishedEvent(TypeConverter.fromYarn(t.jobId), 0, 0, + 0, 0, 0, 0, 0, + new Counters(), + new Counters(), new Counters()))); + + verify(jheh, times(1)).processDoneFiles(any(JobId.class)); + + String intermediateSummaryFileName = JobHistoryUtils.getIntermediateSummaryFileName(t.jobId); + String doneDir = JobHistoryUtils.getHistoryIntermediateDoneDirForUser(conf); + FileSystem fs = FileSystem.get(dfsCluster.getConfiguration(0)); + Path intermediateSummaryFileNamePath = new Path(doneDir, intermediateSummaryFileName); + FsPermission getIntermediateSummaryFilePermission = + fs.getFileStatus(intermediateSummaryFileNamePath).getPermission(); + assertEquals(setFilePermission, + String.valueOf(getIntermediateSummaryFilePermission.toOctal())); + } finally { + jheh.stop(); + } + } } class JHEvenHandlerForTest extends JobHistoryEventHandler { diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/launcher/TestContainerLauncherImpl.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/launcher/TestContainerLauncherImpl.java index 2057cc80ff01a..88ba8943ceb3f 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/launcher/TestContainerLauncherImpl.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/launcher/TestContainerLauncherImpl.java @@ -209,14 +209,11 @@ public void testHandle() throws Exception { ut.waitForPoolToIdle(); verify(mockCM).startContainers(any(StartContainersRequest.class)); - + LOG.info("inserting cleanup event"); - ContainerLauncherEvent mockCleanupEvent = - mock(ContainerLauncherEvent.class); - when(mockCleanupEvent.getType()) - .thenReturn(EventType.CONTAINER_REMOTE_CLEANUP); - when(mockCleanupEvent.getContainerID()) - .thenReturn(contId); + ContainerLauncherEvent mockCleanupEvent = mock(ContainerLauncherEvent.class); + when(mockCleanupEvent.getType()).thenReturn(EventType.CONTAINER_REMOTE_CLEANUP); + when(mockCleanupEvent.getContainerID()).thenReturn(contId); when(mockCleanupEvent.getTaskAttemptID()).thenReturn(taskAttemptId); when(mockCleanupEvent.getContainerMgrAddress()).thenReturn(cmAddress); ut.handle(mockCleanupEvent); @@ -283,8 +280,21 @@ public void testOutOfOrder() throws Exception { ut.handle(mockLaunchEvent); ut.waitForPoolToIdle(); - - verify(mockCM, never()).startContainers(any(StartContainersRequest.class)); + + verify(mockCM).startContainers(any(StartContainersRequest.class)); + + LOG.info("inserting cleanup event"); + ContainerLauncherEvent mockCleanupEvent2 = mock(ContainerLauncherEvent.class); + when(mockCleanupEvent2.getType()).thenReturn(EventType.CONTAINER_REMOTE_CLEANUP); + when(mockCleanupEvent2.getContainerID()).thenReturn(contId); + when(mockCleanupEvent2.getTaskAttemptID()).thenReturn(taskAttemptId); + when(mockCleanupEvent2.getContainerMgrAddress()).thenReturn(cmAddress); + ut.handle(mockCleanupEvent2); + + ut.waitForPoolToIdle(); + + // Verifies stopContainers is called on existing container + verify(mockCM).stopContainers(any(StopContainersRequest.class)); } finally { ut.stop(); } diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/jobhistory/JobHistoryUtils.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/jobhistory/JobHistoryUtils.java index e22726c303abf..1a1df010dd8c7 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/jobhistory/JobHistoryUtils.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/jobhistory/JobHistoryUtils.java @@ -91,9 +91,6 @@ public class JobHistoryUtils { public static final FsPermission HISTORY_INTERMEDIATE_DONE_DIR_PERMISSIONS = FsPermission.createImmutable((short) 01777); - public static final FsPermission HISTORY_INTERMEDIATE_FILE_PERMISSIONS = - FsPermission.createImmutable((short) 0770); // rwx------ - /** * Suffix for configuration files. */ @@ -206,7 +203,7 @@ public static String getConfiguredHistoryIntermediateDoneDirPrefix( /** * Gets the configured directory permissions for the user directories in the - * directory of the intermediate done history files. The user and the group + * Gets the configured permissions for the user directories and files in the * both need full permissions, this is enforced by this method. * @param conf The configuration object * @return FsPermission of the user directories diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/JobEndNotifier.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/JobEndNotifier.java index 1ce525490d14b..b2fa79808ca6d 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/JobEndNotifier.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/JobEndNotifier.java @@ -26,10 +26,10 @@ import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; +import org.apache.http.client.HttpClient; +import org.apache.http.client.config.RequestConfig; import org.apache.http.client.methods.HttpGet; -import org.apache.http.client.params.ClientPNames; -import org.apache.http.impl.client.DefaultHttpClient; -import org.apache.http.params.CoreConnectionPNames; +import org.apache.http.impl.client.HttpClientBuilder; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -67,10 +67,13 @@ private static JobEndStatusInfo createNotification(JobConf conf, private static int httpNotification(String uri, int timeout) throws IOException, URISyntaxException { - DefaultHttpClient client = new DefaultHttpClient(); - client.getParams() - .setIntParameter(CoreConnectionPNames.SO_TIMEOUT, timeout) - .setLongParameter(ClientPNames.CONN_MANAGER_TIMEOUT, (long) timeout); + HttpClientBuilder httpClientBuilder = HttpClientBuilder.create(); + httpClientBuilder.setDefaultRequestConfig( + RequestConfig.custom() + .setConnectionRequestTimeout(timeout) + .setSocketTimeout(timeout) + .build()); + HttpClient client = httpClientBuilder.build(); HttpGet httpGet = new HttpGet(new URI(uri)); httpGet.setHeader("Accept", "*/*"); return client.execute(httpGet).getStatusLine().getStatusCode(); diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/JobResourceUploader.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/JobResourceUploader.java index e16a0d0bb0edc..6a3cb80a5ed98 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/JobResourceUploader.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/JobResourceUploader.java @@ -783,9 +783,11 @@ private URI useSharedCache(URI sourceFile, String resourceName, void copyJar(Path originalJarPath, Path submitJarFile, short replication) throws IOException { jtFs.copyFromLocalFile(originalJarPath, submitJarFile); - jtFs.setReplication(submitJarFile, replication); + // The operation of setReplication requires certain permissions + // so we need to make sure it has enough permissions jtFs.setPermission(submitJarFile, new FsPermission( JobSubmissionFiles.JOB_FILE_PERMISSION)); + jtFs.setReplication(submitJarFile, replication); } private void addLog4jToDistributedCache(Job job, Path jobSubmitDir) diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/JobSubmitter.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/JobSubmitter.java index 4c983178a7f41..232b8bf13bc97 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/JobSubmitter.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/JobSubmitter.java @@ -71,7 +71,6 @@ class JobSubmitter { protected static final Logger LOG = LoggerFactory.getLogger(JobSubmitter.class); private static final String SHUFFLE_KEYGEN_ALGORITHM = "HmacSHA1"; - private static final int SHUFFLE_KEY_LENGTH = 64; private FileSystem jtFs; private ClientProtocol submitClient; private String submitHostName; @@ -177,7 +176,9 @@ JobStatus submitJobInternal(Job job, Cluster cluster) KeyGenerator keyGen; try { keyGen = KeyGenerator.getInstance(SHUFFLE_KEYGEN_ALGORITHM); - keyGen.init(SHUFFLE_KEY_LENGTH); + int shuffleKeyLength = + conf.getInt(MRJobConfig.SHUFFLE_KEY_LENGTH, MRJobConfig.DEFAULT_SHUFFLE_KEY_LENGTH); + keyGen.init(shuffleKeyLength); } catch (NoSuchAlgorithmException e) { throw new IOException("Error generating shuffle secret key", e); } diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java index 15d57a6746b13..4523565f30031 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java @@ -1302,4 +1302,6 @@ public interface MRJobConfig { */ @Unstable String INPUT_FILE_MANDATORY_PREFIX = "mapreduce.job.input.file.must."; + String SHUFFLE_KEY_LENGTH = "mapreduce.shuffle-key-length"; + int DEFAULT_SHUFFLE_KEY_LENGTH = 64; } diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/input/CompressedSplitLineReader.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/input/CompressedSplitLineReader.java index 9d0e949a10b81..74b959cf47c3e 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/input/CompressedSplitLineReader.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/input/CompressedSplitLineReader.java @@ -127,6 +127,8 @@ public CompressedSplitLineReader(SplitCompressionInputStream in, @Override protected int fillBuffer(InputStream in, byte[] buffer, boolean inDelimiter) throws IOException { + boolean alreadyReadAfterSplit = didReadAfterSplit(); + int bytesRead = in.read(buffer); // If the split ended in the middle of a record delimiter then we need @@ -135,7 +137,9 @@ protected int fillBuffer(InputStream in, byte[] buffer, boolean inDelimiter) // However if using the default delimiter and the next character is a // linefeed then next split will treat it as a delimiter all by itself // and the additional record read should not be performed. - if (inDelimiter && bytesRead > 0) { + boolean justReadAfterSplit = !alreadyReadAfterSplit && didReadAfterSplit(); + + if (justReadAfterSplit && inDelimiter && bytesRead > 0) { if (usingCRLF) { needAdditionalRecord = (buffer[0] != '\n'); } else { @@ -152,7 +156,7 @@ public int readLine(Text str, int maxLineLength, int maxBytesToConsume) if (!finished) { // only allow at most one more record to be read after the stream // reports the split ended - if (scin.getPos() > scin.getAdjustedEnd()) { + if (didReadAfterSplit()) { finished = true; } @@ -170,4 +174,8 @@ public boolean needAdditionalRecordAfterSplit() { protected void unsetNeedAdditionalRecordAfterSplit() { needAdditionalRecord = false; } + + private boolean didReadAfterSplit() throws IOException { + return scin.getPos() > scin.getAdjustedEnd(); + } } diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/BindingPathOutputCommitter.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/BindingPathOutputCommitter.java index f12678b5562bb..adde981f24723 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/BindingPathOutputCommitter.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/BindingPathOutputCommitter.java @@ -23,6 +23,10 @@ import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.StreamCapabilities; +import org.apache.hadoop.fs.statistics.IOStatistics; +import org.apache.hadoop.fs.statistics.IOStatisticsSource; +import org.apache.hadoop.fs.statistics.IOStatisticsSupport; import org.apache.hadoop.mapreduce.JobContext; import org.apache.hadoop.mapreduce.JobStatus; import org.apache.hadoop.mapreduce.TaskAttemptContext; @@ -60,7 +64,8 @@ */ @InterfaceAudience.Public @InterfaceStability.Unstable -public class BindingPathOutputCommitter extends PathOutputCommitter { +public class BindingPathOutputCommitter extends PathOutputCommitter + implements IOStatisticsSource, StreamCapabilities { /** * The classname for use in configurations. @@ -181,4 +186,22 @@ public String toString() { public PathOutputCommitter getCommitter() { return committer; } + + /** + * Pass through if the inner committer supports StreamCapabilities. + * {@inheritDoc} + */ + @Override + public boolean hasCapability(final String capability) { + if (committer instanceof StreamCapabilities) { + return ((StreamCapabilities) committer).hasCapability(capability); + } else { + return false; + } + } + + @Override + public IOStatistics getIOStatistics() { + return IOStatisticsSupport.retrieveIOStatistics(committer); + } } diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/ManifestCommitter.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/ManifestCommitter.java index 99625e8242896..024fb3ab34eb2 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/ManifestCommitter.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/ManifestCommitter.java @@ -31,6 +31,7 @@ import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.StreamCapabilities; import org.apache.hadoop.fs.statistics.IOStatisticsSource; import org.apache.hadoop.fs.statistics.impl.IOStatisticsStore; import org.apache.hadoop.mapreduce.JobContext; @@ -55,6 +56,7 @@ import static org.apache.hadoop.fs.statistics.IOStatisticsLogging.ioStatisticsToPrettyString; import static org.apache.hadoop.fs.statistics.IOStatisticsLogging.logIOStatisticsAtDebug; +import static org.apache.hadoop.mapreduce.lib.output.committer.manifest.ManifestCommitterConstants.CAPABILITY_DYNAMIC_PARTITIONING; import static org.apache.hadoop.mapreduce.lib.output.committer.manifest.ManifestCommitterConstants.OPT_DIAGNOSTICS_MANIFEST_DIR; import static org.apache.hadoop.mapreduce.lib.output.committer.manifest.ManifestCommitterConstants.OPT_SUMMARY_REPORT_DIR; import static org.apache.hadoop.mapreduce.lib.output.committer.manifest.ManifestCommitterStatisticNames.COMMITTER_TASKS_COMPLETED_COUNT; @@ -84,7 +86,7 @@ @InterfaceAudience.Public @InterfaceStability.Stable public class ManifestCommitter extends PathOutputCommitter implements - IOStatisticsSource, StageEventCallbacks { + IOStatisticsSource, StageEventCallbacks, StreamCapabilities { public static final Logger LOG = LoggerFactory.getLogger( ManifestCommitter.class); @@ -758,4 +760,15 @@ private static Path maybeSaveSummary( public IOStatisticsStore getIOStatistics() { return iostatistics; } + + /** + * The committer is compatible with spark's dynamic partitioning + * algorithm. + * @param capability string to query the stream support for. + * @return true if the requested capability is supported. + */ + @Override + public boolean hasCapability(final String capability) { + return CAPABILITY_DYNAMIC_PARTITIONING.equals(capability); + } } diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/ManifestCommitterConstants.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/ManifestCommitterConstants.java index eb344e8a27e90..fd7b3d816c103 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/ManifestCommitterConstants.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/ManifestCommitterConstants.java @@ -234,6 +234,12 @@ public final class ManifestCommitterConstants { */ public static final String CONTEXT_ATTR_TASK_ATTEMPT_ID = "ta"; + /** + * Stream Capabilities probe for spark dynamic partitioning compatibility. + */ + public static final String CAPABILITY_DYNAMIC_PARTITIONING = + "mapreduce.job.committer.dynamic.partitioning"; + private ManifestCommitterConstants() { } diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/site/markdown/manifest_committer.md b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/site/markdown/manifest_committer.md index b446be29ddd11..12fe1f0b5f2bd 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/site/markdown/manifest_committer.md +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/site/markdown/manifest_committer.md @@ -269,6 +269,76 @@ appending data are creating and writing into new partitions. job to create unique files. This is foundational for any job to generate correct data. +# Spark Dynamic Partition overwriting + +Spark has a feature called "Dynamic Partition Overwrites", + +This can be initiated in SQL +```SQL +INSERT OVERWRITE TABLE ... +``` +Or through DataSet writes where the mode is `overwrite` and the partitioning matches +that of the existing table +```scala +sparkConf.set("spark.sql.sources.partitionOverwriteMode", "dynamic") +// followed by an overwrite of a Dataset into an existing partitioned table. +eventData2 + .write + .mode("overwrite") + .partitionBy("year", "month") + .format("parquet") + .save(existingDir) +``` + +This feature is implemented in Spark, which +1. Directs the job to write its new data to a temporary directory +1. After job commit completes, scans the output to identify the leaf directories "partitions" into which data was written. +1. Deletes the content of those directories in the destination table +1. Renames the new files into the partitions. + +This is all done in spark, which takes over the tasks of scanning +the intermediate output tree, deleting partitions and of +renaming the new files. + +This feature also adds the ability for a job to write data entirely outside +the destination table, which is done by +1. writing new files into the working directory +1. spark moving them to the final destination in job commit + + +The manifest committer is compatible with dynamic partition overwrites +on Azure and Google cloud storage as together they meet the core requirements of +the extension: +1. The working directory returned in `getWorkPath()` is in the same filesystem + as the final output. +2. `rename()` is an `O(1)` operation which is safe and fast to use when committing a job. + +None of the S3A committers support this. Condition (1) is not met by +the staging committers, while (2) is not met by S3 itself. + +To use the manifest committer with dynamic partition overwrites, the +spark version must contain +[SPARK-40034](https://issues.apache.org/jira/browse/SPARK-40034) +_PathOutputCommitters to work with dynamic partition overwrite_. + +Be aware that the rename phase of the operation will be slow +if many files are renamed -this is done sequentially. +Parallel renaming would speed this up, *but could trigger the abfs overload +problems the manifest committer is designed to both minimize the risk +of and support recovery from* + +The spark side of the commit operation will be listing/treewalking +the temporary output directory (some overhead), followed by +the file promotion, done with a classic filesystem `rename()` +call. There will be no explicit rate limiting here. + +*What does this mean?* + +It means that _dynamic partitioning should not be used on Azure Storage +for SQL queries/Spark DataSet operations where many thousands of files are created. +The fact that these will suffer from performance problems before +throttling scale issues surface, should be considered a warning. + # Job Summaries in `_SUCCESS` files The original hadoop committer creates a zero byte `_SUCCESS` file in the root of the output directory @@ -585,7 +655,7 @@ There is no need to alter these values, except when writing new implementations something which is only needed if the store provides extra integration support for the committer. -## Support for concurrent test runs. +## Support for concurrent jobs to the same directory It *may* be possible to run multiple jobs targeting the same directory tree. @@ -600,6 +670,8 @@ For this to work, a number of conditions must be met: `mapreduce.fileoutputcommitter.cleanup.skipped` to `true`. * All jobs/tasks must create files with unique filenames. * All jobs must create output with the same directory partition structure. +* The job/queries MUST NOT be using Spark Dynamic Partitioning "INSERT OVERWRITE TABLE"; data may be lost. + This holds for *all* committers, not just the manifest committer. * Remember to delete the `_temporary` directory later! This has *NOT BEEN TESTED* diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/LineRecordReaderHelper.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/LineRecordReaderHelper.java new file mode 100644 index 0000000000000..fccc01ad7e602 --- /dev/null +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/LineRecordReaderHelper.java @@ -0,0 +1,54 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package org.apache.hadoop.mapred; + +import java.io.IOException; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.io.LongWritable; +import org.apache.hadoop.io.Text; +import org.apache.hadoop.mapreduce.lib.input.BaseLineRecordReaderHelper; + +public final class LineRecordReaderHelper extends + BaseLineRecordReaderHelper { + + public LineRecordReaderHelper(Path filePath, Configuration conf) { + super(filePath, conf); + } + + @Override + public long countRecords(long start, long length) throws IOException { + try (LineRecordReader reader = newReader(start, length)) { + LongWritable key = new LongWritable(); + Text value = new Text(); + + long numRecords = 0L; + while (reader.next(key, value)) { + numRecords++; + } + return numRecords; + } + } + + private LineRecordReader newReader(long start, long length) + throws IOException { + FileSplit split = new FileSplit(getFilePath(), start, length, (String[]) null); + return new LineRecordReader(getConf(), split, getRecordDelimiterBytes()); + } +} diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestJobEndNotifier.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestJobEndNotifier.java index e3d1241cdc679..46b581d2d0e22 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestJobEndNotifier.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestJobEndNotifier.java @@ -139,7 +139,7 @@ public void testLocalJobRunnerUriSubstitution() throws InterruptedException { baseUrl + "jobend?jobid=$jobId&status=$jobStatus"); JobEndNotifier.localRunnerNotification(jobConf, jobStatus); - // No need to wait for the notification to go thru since calls are + // No need to wait for the notification to go through since calls are // synchronous // Validate params diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestLineRecordReaderBZip2.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestLineRecordReaderBZip2.java new file mode 100644 index 0000000000000..1588e629c8c1e --- /dev/null +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestLineRecordReaderBZip2.java @@ -0,0 +1,31 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package org.apache.hadoop.mapred; + +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.mapreduce.lib.input.BaseLineRecordReaderHelper; +import org.apache.hadoop.mapreduce.lib.input.BaseTestLineRecordReaderBZip2; + +public final class TestLineRecordReaderBZip2 extends + BaseTestLineRecordReaderBZip2 { + + @Override + protected BaseLineRecordReaderHelper newReader(Path file) { + return new LineRecordReaderHelper(file, getConf()); + } +} diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/lib/input/BaseLineRecordReaderHelper.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/lib/input/BaseLineRecordReaderHelper.java new file mode 100644 index 0000000000000..d1e21e6e258be --- /dev/null +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/lib/input/BaseLineRecordReaderHelper.java @@ -0,0 +1,58 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package org.apache.hadoop.mapreduce.lib.input; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.Path; + +public abstract class BaseLineRecordReaderHelper { + + private final Configuration conf; + private final Path filePath; + private final byte[] recordDelimiterBytes; + + + + public BaseLineRecordReaderHelper(Path filePath, Configuration conf) { + this.conf = conf; + this.filePath = filePath; + + conf.setInt(LineRecordReader.MAX_LINE_LENGTH, Integer.MAX_VALUE); + + String delimiter = conf.get("textinputformat.record.delimiter"); + this.recordDelimiterBytes = + null != delimiter ? delimiter.getBytes(StandardCharsets.UTF_8) : null; + } + + public abstract long countRecords(long start, long length) throws IOException; + + public Configuration getConf() { + return conf; + } + + public Path getFilePath() { + return filePath; + } + + public byte[] getRecordDelimiterBytes() { + return recordDelimiterBytes; + } +} diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/lib/input/BaseTestLineRecordReaderBZip2.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/lib/input/BaseTestLineRecordReaderBZip2.java new file mode 100644 index 0000000000000..da6bedde13149 --- /dev/null +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/lib/input/BaseTestLineRecordReaderBZip2.java @@ -0,0 +1,376 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package org.apache.hadoop.mapreduce.lib.input; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.util.Arrays; +import java.util.List; + +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.io.compress.bzip2.BZip2TextFileWriter; +import org.apache.hadoop.io.compress.bzip2.BZip2Utils; + +import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.IO_FILE_BUFFER_SIZE_KEY; +import static org.apache.hadoop.io.compress.bzip2.BZip2TextFileWriter.BLOCK_SIZE; +import static org.junit.Assert.assertEquals; + +public abstract class BaseTestLineRecordReaderBZip2 { + + // LF stands for line feed + private static final byte[] LF = new byte[] {'\n'}; + // CR stands for cartridge return + private static final byte[] CR = new byte[] {'\r'}; + private static final byte[] CR_LF = new byte[] {'\r', '\n'}; + + private Configuration conf; + private FileSystem fs; + private Path tempFile; + + public Configuration getConf() { + return conf; + } + + public FileSystem getFs() { + return fs; + } + + public Path getTempFile() { + return tempFile; + } + + @Before + public void setUp() throws Exception { + conf = new Configuration(); + + Path workDir = new Path( + System.getProperty("test.build.data", "target"), + "data/" + getClass().getSimpleName()); + + fs = workDir.getFileSystem(conf); + + Path inputDir = new Path(workDir, "input"); + tempFile = new Path(inputDir, "test.txt.bz2"); + } + + @After + public void tearDown() throws Exception { + fs.delete(tempFile, /* recursive */ false); + } + + @Test + public void firstBlockEndsWithLF() throws Exception { + try (BZip2TextFileWriter writer = new BZip2TextFileWriter(tempFile, conf)) { + writer.writeManyRecords(BLOCK_SIZE, 1000, LF); + writer.writeRecord(10, LF); + writer.writeRecord(10, LF); + writer.writeRecord(10, LF); + } + assertRecordCountsPerSplit(tempFile, new long[] {1001, 2}); + } + + @Test + public void firstBlockEndsWithLFSecondBlockStartsWithLF() throws Exception { + try (BZip2TextFileWriter writer = new BZip2TextFileWriter(tempFile, conf)) { + writer.writeManyRecords(BLOCK_SIZE, 1000, LF); + // Write 254 empty rows terminating at LF, as those records will get + // rolled into the first block record due to run-length encoding, the + // 255th LF character will trigger a run to be written to the block. We + // only need 254 LF characters since the last byte written by prior + // writeManyRecords call is already a LF. + writer.writeManyRecords(254, 254, LF); + + // This LF character should be the first byte of the second block, but + // if splitting at blocks, the first split will read this record as the + // additional record. + writer.writeRecord(1, LF); + + writer.writeRecord(10, LF); + writer.writeRecord(10, LF); + } + assertRecordCountsPerSplit(tempFile, new long[] {1255, 2}); + } + + @Test + public void firstBlockEndsWithLFSecondBlockStartsWithCR() throws Exception { + try (BZip2TextFileWriter writer = new BZip2TextFileWriter(tempFile, conf)) { + writer.writeManyRecords(BLOCK_SIZE, 1000, LF); + writer.writeRecord(1, CR); + writer.writeRecord(10, LF); + writer.writeRecord(10, LF); + } + assertRecordCountsPerSplit(tempFile, new long[] {1001, 2}); + } + + @Test + public void firstBlockEndsWithCRLF() throws Exception { + try (BZip2TextFileWriter writer = new BZip2TextFileWriter(tempFile, conf)) { + writer.writeManyRecords(BLOCK_SIZE, 1000, CR_LF); + writer.writeRecord(10, LF); + writer.writeRecord(10, LF); + writer.writeRecord(10, LF); + } + assertRecordCountsPerSplit(tempFile, new long[] {1001, 2}); + } + + @Test + public void lastRecordContentSpanAcrossBlocks() + throws Exception { + try (BZip2TextFileWriter writer = new BZip2TextFileWriter(tempFile, conf)) { + writer.writeManyRecords(BLOCK_SIZE - 50, 999, LF); + writer.writeRecord(100, LF); + writer.writeRecord(10, LF); + writer.writeRecord(10, LF); + writer.writeRecord(10, LF); + } + assertRecordCountsPerSplit(tempFile, new long[] {1000, 3}); + } + + @Test + public void lastRecordOfBlockHasItsLFInNextBlock() throws Exception { + try (BZip2TextFileWriter writer = new BZip2TextFileWriter(tempFile, conf)) { + writer.writeManyRecords(BLOCK_SIZE - 50, 999, LF); + // The LF character is the first byte of the second block + writer.writeRecord(51, LF); + writer.writeRecord(10, LF); + writer.writeRecord(10, LF); + writer.writeRecord(10, LF); + } + assertRecordCountsPerSplit(tempFile, new long[] {1000, 3}); + } + + @Test + public void lastRecordOfFirstBlockHasItsCRLFInSecondBlock() throws Exception { + try (BZip2TextFileWriter writer = new BZip2TextFileWriter(tempFile, conf)) { + writer.writeManyRecords(BLOCK_SIZE - 50, 999, LF); + // Both CR + LF characters are the first two bytes of second block + writer.writeRecord(52, CR_LF); + writer.writeRecord(10, LF); + writer.writeRecord(10, LF); + writer.writeRecord(10, LF); + } + assertRecordCountsPerSplit(tempFile, new long[] {1000, 3}); + } + + @Test + public void lastRecordOfFirstBlockHasItsCRLFPartlyInSecondBlock() + throws Exception { + try (BZip2TextFileWriter writer = new BZip2TextFileWriter(tempFile, conf)) { + writer.writeManyRecords(BLOCK_SIZE - 50, 999, LF); + // The CR character is the last byte of the first block and the LF is + // the firs byte of the second block + writer.writeRecord(51, CR_LF); + writer.writeRecord(10, LF); + writer.writeRecord(10, LF); + writer.writeRecord(10, LF); + } + assertRecordCountsPerSplit(tempFile, new long[] {1000, 3}); + } + + @Test + public void lastByteInFirstBlockIsCRFirstByteInSecondBlockIsNotLF() + throws Exception { + try (BZip2TextFileWriter writer = new BZip2TextFileWriter(tempFile, conf)) { + writer.writeManyRecords(BLOCK_SIZE, 1000, CR); + writer.writeRecord(10, LF); + writer.writeRecord(10, LF); + writer.writeRecord(10, LF); + } + assertRecordCountsPerSplit(tempFile, new long[] {1001, 2}); + } + + @Test + public void usingCRDelimiterWithSmallestBufferSize() throws Exception { + // Forces calling LineReader#fillBuffer for ever byte read + conf.set(IO_FILE_BUFFER_SIZE_KEY, "1"); + + try (BZip2TextFileWriter writer = new BZip2TextFileWriter(tempFile, conf)) { + writer.writeManyRecords(BLOCK_SIZE - 50, 999, CR); + writer.writeRecord(100, CR); + writer.writeRecord(10, CR); + writer.writeRecord(10, CR); + writer.writeRecord(10, CR); + } + assertRecordCountsPerSplit(tempFile, new long[] {1000, 3}); + } + + @Test + public void delimitedByCRSpanningThreeBlocks() throws Exception { + try (BZip2TextFileWriter writer = new BZip2TextFileWriter(tempFile, conf)) { + writer.writeRecord(3 * BLOCK_SIZE, CR); + writer.writeRecord(3 * BLOCK_SIZE, CR); + writer.writeRecord(3 * BLOCK_SIZE, CR); + } + assertRecordCountsPerSplit(tempFile, + new long[] {1, 0, 1, 0, 0, 1, 0, 0, 0}); + } + + @Test + public void customDelimiterLastThreeBytesInBlockAreDelimiter() + throws Exception { + byte[] delimiter = new byte[] {'e', 'n', 'd'}; + setDelimiter(delimiter); + + try (BZip2TextFileWriter writer = new BZip2TextFileWriter(tempFile, conf)) { + writer.writeManyRecords(BLOCK_SIZE, 1000, delimiter); + writer.writeRecord(10, delimiter); + writer.writeRecord(10, delimiter); + writer.writeRecord(10, delimiter); + } + assertRecordCountsPerSplit(tempFile, new long[] {1001, 2}); + } + + @Test + public void customDelimiterDelimiterSpansAcrossBlocks() + throws Exception { + byte[] delimiter = new byte[] {'e', 'n', 'd'}; + setDelimiter(delimiter); + + try (BZip2TextFileWriter writer = new BZip2TextFileWriter(tempFile, conf)) { + writer.writeManyRecords(BLOCK_SIZE - 50, 999, delimiter); + writer.writeRecord(52, delimiter); + writer.writeRecord(10, delimiter); + writer.writeRecord(10, delimiter); + writer.writeRecord(10, delimiter); + } + assertRecordCountsPerSplit(tempFile, new long[] {1001, 2}); + } + + @Test + public void customDelimiterLastRecordDelimiterStartsAtNextBlockStart() + throws Exception { + byte[] delimiter = new byte[] {'e', 'n', 'd'}; + setDelimiter(delimiter); + + try (BZip2TextFileWriter writer = new BZip2TextFileWriter(tempFile, conf)) { + writer.writeManyRecords(BLOCK_SIZE - 50, 999, delimiter); + writer.writeRecord(53, delimiter); + writer.writeRecord(10, delimiter); + writer.writeRecord(10, delimiter); + writer.writeRecord(10, delimiter); + } + assertRecordCountsPerSplit(tempFile, new long[] {1000, 3}); + } + + @Test + public void customDelimiterLastBlockBytesShareCommonPrefixWithDelimiter() + throws Exception { + byte[] delimiter = new byte[] {'e', 'n', 'd'}; + setDelimiter(delimiter); + + try (BZip2TextFileWriter writer = new BZip2TextFileWriter(tempFile, conf)) { + writer.writeManyRecords(BLOCK_SIZE - 4, 999, delimiter); + // The first 4 bytes, "an e", will be the last 4 bytes of the first block, + // the last byte being 'e' which matches the first character of the + // delimiter "end". The first byte of the next block also matches the + // second byte of the delimiter "n"; however the next character "c" does + // not match the last character of the delimiter. Thus an additional + // record should not be read for the split that reads the first block. + // The split that reads the second block will just discard + // "nchanting tale coming to an end". + writer.write("an enchanting tale coming to an end"); + writer.writeRecord(10, delimiter); + writer.writeRecord(10, delimiter); + writer.writeRecord(10, delimiter); + } + assertRecordCountsPerSplit(tempFile, new long[] {1000, 3}); + } + + protected abstract BaseLineRecordReaderHelper newReader(Path file); + + private void assertRecordCountsPerSplit( + Path path, long[] countsIfSplitAtBlocks) throws IOException { + RecordCountAssert countAssert = + new RecordCountAssert(path, countsIfSplitAtBlocks); + countAssert.assertSingleSplit(); + countAssert.assertSplittingAtBlocks(); + countAssert.assertSplittingJustAfterSecondBlockStarts(); + } + + private class RecordCountAssert { + + private final BaseLineRecordReaderHelper reader; + private final long numBlocks; + private final long[] countsIfSplitAtBlocks; + private final long fileSize; + private final long totalRecords; + private final List nextBlockOffsets; + + RecordCountAssert( + Path path, long[] countsIfSplitAtBlocks) throws IOException { + this.reader = newReader(path); + this.countsIfSplitAtBlocks = countsIfSplitAtBlocks; + this.fileSize = getFileSize(path); + this.totalRecords = Arrays.stream(countsIfSplitAtBlocks).sum(); + this.numBlocks = countsIfSplitAtBlocks.length; + this.nextBlockOffsets = BZip2Utils.getNextBlockMarkerOffsets(path, conf); + + assertEquals(numBlocks, nextBlockOffsets.size() + 1); + } + + private void assertSingleSplit() throws IOException { + assertEquals(totalRecords, reader.countRecords(0, fileSize)); + } + + private void assertSplittingAtBlocks() throws IOException { + for (int i = 0; i < numBlocks; i++) { + long start = i == 0 ? 0 : nextBlockOffsets.get(i - 1); + long end = i == numBlocks - 1 ? fileSize : nextBlockOffsets.get(i); + long length = end - start; + + String message = "At i=" + i; + long expectedCount = countsIfSplitAtBlocks[i]; + assertEquals( + message, expectedCount, reader.countRecords(start, length)); + } + } + + private void assertSplittingJustAfterSecondBlockStarts() + throws IOException { + if (numBlocks <= 1) { + return; + } + long recordsInFirstTwoBlocks = + countsIfSplitAtBlocks[0] + countsIfSplitAtBlocks[1]; + long remainingRecords = totalRecords - recordsInFirstTwoBlocks; + + long firstSplitSize = nextBlockOffsets.get(0) + 1; + assertEquals( + recordsInFirstTwoBlocks, + reader.countRecords(0, firstSplitSize)); + assertEquals( + remainingRecords, + reader.countRecords(firstSplitSize, fileSize - firstSplitSize)); + } + } + + private long getFileSize(Path path) throws IOException { + return fs.getFileStatus(path).getLen(); + } + + private void setDelimiter(byte[] delimiter) { + conf.set("textinputformat.record.delimiter", + new String(delimiter, StandardCharsets.UTF_8)); + } +} diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/lib/input/LineRecordReaderHelper.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/lib/input/LineRecordReaderHelper.java new file mode 100644 index 0000000000000..56fc7eb4e8662 --- /dev/null +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/lib/input/LineRecordReaderHelper.java @@ -0,0 +1,62 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package org.apache.hadoop.mapreduce.lib.input; + +import java.io.IOException; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.mapreduce.TaskAttemptContext; +import org.apache.hadoop.mapreduce.TaskAttemptID; +import org.apache.hadoop.mapreduce.task.TaskAttemptContextImpl; + +public final class LineRecordReaderHelper extends + BaseLineRecordReaderHelper { + + public LineRecordReaderHelper(Path filePath, Configuration conf) { + super(filePath, conf); + } + + @Override + public long countRecords(long start, long length) throws IOException { + try (LineRecordReader reader = newReader(start, length)) { + long numRecords = 0L; + while (reader.nextKeyValue()) { + numRecords++; + } + return numRecords; + } + } + + private LineRecordReader newReader(long start, long length) + throws IOException { + FileSplit split = new FileSplit(getFilePath(), start, length, null); + + TaskAttemptContext context = + new TaskAttemptContextImpl(getConf(), new TaskAttemptID()); + + LineRecordReader reader = new LineRecordReader(getRecordDelimiterBytes()); + try { + reader.initialize(split, context); + return reader; + } catch (Throwable e) { + reader.close(); + throw e; + } + } +} diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/lib/input/TestLineRecordReaderBZip2.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/lib/input/TestLineRecordReaderBZip2.java new file mode 100644 index 0000000000000..ec01ef332fa90 --- /dev/null +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/lib/input/TestLineRecordReaderBZip2.java @@ -0,0 +1,29 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package org.apache.hadoop.mapreduce.lib.input; + +import org.apache.hadoop.fs.Path; + +public final class TestLineRecordReaderBZip2 + extends BaseTestLineRecordReaderBZip2 { + + @Override + protected BaseLineRecordReaderHelper newReader(Path file) { + return new LineRecordReaderHelper(file, getConf()); + } +} diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/TestManifestCommitProtocol.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/TestManifestCommitProtocol.java index 2212fabe54acd..3037bf33ad62f 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/TestManifestCommitProtocol.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/TestManifestCommitProtocol.java @@ -61,6 +61,7 @@ import org.apache.hadoop.mapreduce.RecordWriter; import org.apache.hadoop.mapreduce.TaskAttemptContext; import org.apache.hadoop.mapreduce.TaskAttemptID; +import org.apache.hadoop.mapreduce.lib.output.BindingPathOutputCommitter; import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat; import org.apache.hadoop.mapreduce.lib.output.MapFileOutputFormat; import org.apache.hadoop.mapreduce.lib.output.PathOutputCommitterFactory; @@ -1549,6 +1550,23 @@ public void testOutputFormatIntegration() throws Throwable { ManifestCommitter committer = (ManifestCommitter) outputFormat.getOutputCommitter(tContext); + // check path capabilities directly + Assertions.assertThat(committer.hasCapability( + ManifestCommitterConstants.CAPABILITY_DYNAMIC_PARTITIONING)) + .describedAs("dynamic partitioning capability in committer %s", + committer) + .isTrue(); + // and through a binding committer -passthrough is critical + // for the spark binding. + BindingPathOutputCommitter bindingCommitter = + new BindingPathOutputCommitter(outputDir, tContext); + Assertions.assertThat(bindingCommitter.hasCapability( + ManifestCommitterConstants.CAPABILITY_DYNAMIC_PARTITIONING)) + .describedAs("dynamic partitioning capability in committer %s", + bindingCommitter) + .isTrue(); + + // setup JobData jobData = new JobData(job, jContext, tContext, committer); setupJob(jobData); diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/TestRenameStageFailure.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/TestRenameStageFailure.java index b794f5814df36..11d2beda5d015 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/TestRenameStageFailure.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/TestRenameStageFailure.java @@ -115,7 +115,7 @@ public void setup() throws Exception { * Does this test suite require rename resilience in the store/FS? * @return true if the store operations are resilient. */ - protected boolean requireRenameResilience() { + protected boolean requireRenameResilience() throws IOException { return false; } diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/pom.xml b/hadoop-mapreduce-project/hadoop-mapreduce-client/pom.xml index d6b453fb3b034..b394fe5be18ef 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/pom.xml +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/pom.xml @@ -74,10 +74,6 @@ - - javax.ws.rs - javax.ws.rs-api - org.apache.hadoop hadoop-common diff --git a/hadoop-maven-plugins/pom.xml b/hadoop-maven-plugins/pom.xml index 37daeb82cd506..522c5a9468705 100644 --- a/hadoop-maven-plugins/pom.xml +++ b/hadoop-maven-plugins/pom.xml @@ -68,56 +68,6 @@ com.fasterxml.jackson.core jackson-databind - - org.apache.maven.plugins - maven-shade-plugin - ${maven-shade-plugin.version} - provided - - - - org.apache.maven - maven-artifact - - - org.apache.maven - maven-compat - - - org.apache.maven - maven-core - - - org.apache.maven - maven-model - - - org.apache.maven - maven-plugin-api - - - org.vafer - jdependency - - - org.sonatype.sisu - sisu-inject-plexus - - - org.apache.maven.plugin-tools - maven-plugin-annotations - - - org.sonatype.aether - aether-api - - - org.sonatype.aether - aether-util - - - org.apache.hadoop.thirdparty hadoop-shaded-guava diff --git a/hadoop-maven-plugins/src/main/java/org/apache/hadoop/maven/plugin/protoc/ProtocRunner.java b/hadoop-maven-plugins/src/main/java/org/apache/hadoop/maven/plugin/protoc/ProtocRunner.java index e83a8cd957fd1..c84bb1b38cb9f 100644 --- a/hadoop-maven-plugins/src/main/java/org/apache/hadoop/maven/plugin/protoc/ProtocRunner.java +++ b/hadoop-maven-plugins/src/main/java/org/apache/hadoop/maven/plugin/protoc/ProtocRunner.java @@ -30,6 +30,7 @@ import java.io.FileNotFoundException; import java.io.FileOutputStream; import java.io.IOException; +import java.nio.file.Files; import java.util.Arrays; import java.util.ArrayList; import java.util.HashMap; @@ -172,6 +173,7 @@ private long computeChecksum(File file) throws IOException { public void writeChecksums() throws IOException { ObjectMapper mapper = new ObjectMapper(); + Files.createDirectories(checksumFile.getParentFile().toPath()); try (BufferedOutputStream out = new BufferedOutputStream( new FileOutputStream(checksumFile))) { mapper.writeValue(out, computedChecksums); diff --git a/hadoop-maven-plugins/src/main/java/org/apache/hadoop/maven/plugin/shade/resource/ServicesResourceTransformer.java b/hadoop-maven-plugins/src/main/java/org/apache/hadoop/maven/plugin/shade/resource/ServicesResourceTransformer.java deleted file mode 100644 index 2d5de7abf1750..0000000000000 --- a/hadoop-maven-plugins/src/main/java/org/apache/hadoop/maven/plugin/shade/resource/ServicesResourceTransformer.java +++ /dev/null @@ -1,166 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - * - */ - -package org.apache.hadoop.maven.plugin.shade.resource; - -import java.io.BufferedReader; -import org.apache.maven.plugins.shade.relocation.Relocator; -import org.apache.maven.plugins.shade.resource.ResourceTransformer; -import org.codehaus.plexus.util.IOUtil; - -import java.io.ByteArrayInputStream; -import java.io.ByteArrayOutputStream; -import java.io.IOException; -import java.io.InputStream; -import java.io.InputStreamReader; -import java.io.OutputStreamWriter; -import java.io.PrintWriter; -import java.nio.charset.StandardCharsets; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.jar.JarEntry; -import java.util.jar.JarOutputStream; - -/** - * Resources transformer that appends entries in META-INF/services resources - * into a single resource. For example, if there are several - * META-INF/services/org.apache.maven.project.ProjectBuilder resources spread - * across many JARs the individual entries will all be concatenated into a - * single META-INF/services/org.apache.maven.project.ProjectBuilder resource - * packaged into the resultant JAR produced by the shading process. - * - * From following sources, only needed until MSHADE-182 gets released - * * https://s.apache.org/vwjl (source in maven-shade-plugin repo) - * * https://issues.apache.org/jira/secure/attachment/12718938/MSHADE-182.patch - * - * Has been reformatted according to Hadoop checkstyle rules and modified - * to meet Hadoop's threshold for Findbugs problems. - */ -public class ServicesResourceTransformer - implements ResourceTransformer { - - private static final String SERVICES_PATH = "META-INF/services"; - - private Map serviceEntries = new HashMap<>(); - - private List relocators; - - public boolean canTransformResource(String resource) { - if (resource.startsWith(SERVICES_PATH)) { - return true; - } - - return false; - } - - public void processResource(String resource, InputStream is, - List relocatorz) throws IOException { - ServiceStream out = serviceEntries.get(resource); - if (out == null) { - out = new ServiceStream(); - serviceEntries.put(resource, out); - } - - out.append(is); - is.close(); - - if (this.relocators == null) { - this.relocators = relocatorz; - } - } - - public boolean hasTransformedResource() { - return serviceEntries.size() > 0; - } - - public void modifyOutputStream(JarOutputStream jos) - throws IOException { - for (Map.Entry entry : serviceEntries.entrySet()) { - String key = entry.getKey(); - ServiceStream data = entry.getValue(); - - if (relocators != null) { - key = key.substring(SERVICES_PATH.length() + 1); - for (Relocator relocator : relocators) { - if (relocator.canRelocateClass(key)) { - key = relocator.relocateClass(key); - break; - } - } - - key = SERVICES_PATH + '/' + key; - } - - jos.putNextEntry(new JarEntry(key)); - - //read the content of service file for candidate classes for relocation - //presume everything is UTF8, because Findbugs barfs on default - //charset and this seems no worse a choice ¯\_(ツ)_/¯ - PrintWriter writer = new PrintWriter(new OutputStreamWriter(jos, - StandardCharsets.UTF_8)); - InputStreamReader streamReader = - new InputStreamReader(data.toInputStream(), StandardCharsets.UTF_8); - BufferedReader reader = new BufferedReader(streamReader); - String className; - - while ((className = reader.readLine()) != null) { - - if (relocators != null) { - for (Relocator relocator : relocators) { - //if the class can be relocated then relocate it - if (relocator.canRelocateClass(className)) { - className = relocator.applyToSourceContent(className); - break; - } - } - } - - writer.println(className); - writer.flush(); - } - - reader.close(); - data.reset(); - } - } - - static class ServiceStream extends ByteArrayOutputStream { - - public ServiceStream() { - super(1024); - } - - public void append(InputStream is) - throws IOException { - if (count > 0 && buf[count - 1] != '\n' && buf[count - 1] != '\r') { - write('\n'); - } - - IOUtil.copy(is, this); - } - - public InputStream toInputStream() { - return new ByteArrayInputStream(buf, 0, count); - } - - } - -} diff --git a/hadoop-project-dist/pom.xml b/hadoop-project-dist/pom.xml index 9af619aba3b2c..53e2de63c4a7d 100644 --- a/hadoop-project-dist/pom.xml +++ b/hadoop-project-dist/pom.xml @@ -134,7 +134,7 @@ false - 3.3.3 + 3.3.4 -unstable diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml index 72504c1825e90..771f019680283 100644 --- a/hadoop-project/pom.xml +++ b/hadoop-project/pom.xml @@ -37,7 +37,7 @@ true true - 9.4.44.v20210927 + 9.4.48.v20220622 _ _ @@ -69,11 +69,8 @@ 1.19.4 - 2.13.2 - 2.13.2.2 - - - 2.1.1 + 2.12.7 + 2.12.7 4.5.13 @@ -121,15 +118,15 @@ 3.2.2 1.21 1.0 - 2.8.0 + 2.11.0 3.12.0 1.1.3 1.1 - 3.1.1 - 3.6 + 3.6.1 + 3.8.0 1.4 - 1.0.1 + 2.0.2 1.0-alpha-1 3.3.1 4.0.3 @@ -144,7 +141,7 @@ 2.9.0 3.2.4 3.10.6.Final - 4.1.68.Final + 4.1.77.Final 1.1.8.2 1.7.1 @@ -187,7 +184,7 @@ 1.3.1 1.0-beta-1 900 - 1.12.132 + 1.12.262 2.3.4 1.11.2 2.1 @@ -200,7 +197,7 @@ ${hadoop.version} 1.5.4 - 1.26 + 1.31 1.7.1 2.2.4 4.13.2 @@ -801,11 +798,6 @@ jsr311-api 1.1.1 - - javax.ws.rs - javax.ws.rs-api - ${javax.ws.rs-api.version} - org.eclipse.jetty jetty-server @@ -991,6 +983,66 @@ ${netty4.version} + + io.netty + netty-codec-socks + ${netty4.version} + + + + io.netty + netty-handler-proxy + ${netty4.version} + + + + io.netty + netty-resolver + ${netty4.version} + + + + io.netty + netty-handler + ${netty4.version} + + + + io.netty + netty-buffer + ${netty4.version} + + + + io.netty + netty-transport + ${netty4.version} + + + + io.netty + netty-common + ${netty4.version} + + + + io.netty + netty-transport-native-unix-common + ${netty4.version} + + + + io.netty + netty-transport-native-epoll + ${netty4.version} + + + + io.netty + netty-codec + ${netty4.version} + + commons-io commons-io @@ -1730,6 +1782,12 @@ org.apache.kerby kerb-simplekdc ${kerby.version} + + + org.jboss.xnio + xnio-api + + org.apache.kerby diff --git a/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSUtils.java b/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSUtils.java index 3b2f5d5f26e51..1ef7fe72ae246 100644 --- a/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSUtils.java +++ b/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSUtils.java @@ -23,6 +23,7 @@ import java.net.URI; import com.aliyun.oss.common.auth.CredentialsProvider; +import org.apache.hadoop.fs.Path; import org.apache.hadoop.util.Preconditions; import org.apache.commons.lang3.StringUtils; import org.apache.hadoop.conf.Configuration; @@ -39,7 +40,7 @@ final public class AliyunOSSUtils { private static final Logger LOG = LoggerFactory.getLogger(AliyunOSSUtils.class); - private static LocalDirAllocator directoryAllocator; + private static volatile LocalDirAllocator directoryAllocator; private AliyunOSSUtils() { } @@ -171,21 +172,33 @@ public static boolean objectRepresentsDirectory(final String name, /** * Demand create the directory allocator, then create a temporary file. - * @param path prefix for the temporary file - * @param size the size of the file that is going to be written - * @param conf the Configuration object - * @return a unique temporary file - * @throws IOException IO problems + * This does not mark the file for deletion when a process exits. + * {@link LocalDirAllocator#createTmpFileForWrite( + * String, long, Configuration)}. + * @param pathStr prefix for the temporary file + * @param size the size of the file that is going to be written + * @param conf the Configuration object + * @return a unique temporary file + * @throws IOException IO problems */ - public static File createTmpFileForWrite(String path, long size, + public static File createTmpFileForWrite(String pathStr, long size, Configuration conf) throws IOException { if (conf.get(BUFFER_DIR_KEY) == null) { conf.set(BUFFER_DIR_KEY, conf.get("hadoop.tmp.dir") + "/oss"); } if (directoryAllocator == null) { - directoryAllocator = new LocalDirAllocator(BUFFER_DIR_KEY); + synchronized (AliyunOSSUtils.class) { + if (directoryAllocator == null) { + directoryAllocator = new LocalDirAllocator(BUFFER_DIR_KEY); + } + } } - return directoryAllocator.createTmpFileForWrite(path, size, conf); + Path path = directoryAllocator.getLocalPathForWrite(pathStr, + size, conf); + File dir = new File(path.getParent().toUri().getPath()); + String prefix = path.getName(); + // create a temp file on this directory + return File.createTempFile(prefix, null, dir); } /** diff --git a/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/TestAliyunOSSBlockOutputStream.java b/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/TestAliyunOSSBlockOutputStream.java index f6e0b7731dbeb..69aa0a5a7956b 100644 --- a/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/TestAliyunOSSBlockOutputStream.java +++ b/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/TestAliyunOSSBlockOutputStream.java @@ -23,19 +23,27 @@ import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.contract.ContractTestUtils; +import org.apache.hadoop.test.GenericTestUtils; import org.junit.After; import org.junit.Before; import org.junit.Rule; import org.junit.Test; import org.junit.rules.Timeout; +import java.io.File; import java.io.IOException; +import java.lang.reflect.Field; +import java.util.ArrayList; +import java.util.LinkedHashSet; import static org.apache.hadoop.fs.aliyun.oss.Constants.BUFFER_DIR_KEY; import static org.apache.hadoop.fs.aliyun.oss.Constants.MULTIPART_UPLOAD_PART_SIZE_DEFAULT; import static org.apache.hadoop.fs.aliyun.oss.Constants.MULTIPART_UPLOAD_PART_SIZE_KEY; import static org.apache.hadoop.fs.contract.ContractTestUtils.IO_CHUNK_BUFFER_SIZE; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotEquals; +import static org.junit.Assert.assertTrue; /** * Tests regular and multi-part upload functionality for @@ -201,4 +209,44 @@ private void bufferDirShouldEmpty() throws IOException { // Temporary file should be deleted assertEquals(0, files.length); } + + @Test + public void testDirectoryAllocator() throws Throwable { + Configuration conf = fs.getConf(); + File tmp = AliyunOSSUtils.createTmpFileForWrite("out-", 1024, conf); + assertTrue("not found: " + tmp, tmp.exists()); + tmp.delete(); + + // tmp should not in DeleteOnExitHook + try { + Class c = Class.forName("java.io.DeleteOnExitHook"); + Field field = c.getDeclaredField("files"); + field.setAccessible(true); + String name = field.getName(); + LinkedHashSet files = (LinkedHashSet)field.get(name); + assertTrue("in DeleteOnExitHook", files.isEmpty()); + assertFalse("in DeleteOnExitHook", + (new ArrayList<>(files)).contains(tmp.getPath())); + } catch (Exception e) { + e.printStackTrace(); + } + } + + @Test + public void testDirectoryAllocatorRR() throws Throwable { + File dir1 = GenericTestUtils.getRandomizedTestDir(); + File dir2 = GenericTestUtils.getRandomizedTestDir(); + dir1.mkdirs(); + dir2.mkdirs(); + + Configuration conf = new Configuration(); + conf.set(BUFFER_DIR_KEY, dir1 + ", " + dir2); + fs = AliyunOSSTestUtils.createTestFileSystem(conf); + File tmp1 = AliyunOSSUtils.createTmpFileForWrite("out-", 1024, conf); + tmp1.delete(); + File tmp2 = AliyunOSSUtils.createTmpFileForWrite("out-", 1024, conf); + tmp2.delete(); + assertNotEquals("round robin not working", + tmp1.getParent(), tmp2.getParent()); + } } diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/AWSCredentialProviderList.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/AWSCredentialProviderList.java index 4dbfc933f7213..f4d0a8d091249 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/AWSCredentialProviderList.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/AWSCredentialProviderList.java @@ -61,7 +61,7 @@ */ @InterfaceAudience.Private @InterfaceStability.Evolving -public class AWSCredentialProviderList implements AWSCredentialsProvider, +public final class AWSCredentialProviderList implements AWSCredentialsProvider, AutoCloseable { private static final Logger LOG = LoggerFactory.getLogger( diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/AnonymousAWSCredentialsProvider.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/AnonymousAWSCredentialsProvider.java index 0cb9fd4e57118..564c03bf731d7 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/AnonymousAWSCredentialsProvider.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/AnonymousAWSCredentialsProvider.java @@ -34,9 +34,13 @@ * Please note that users may reference this class name from configuration * property fs.s3a.aws.credentials.provider. Therefore, changing the class name * would be a backward-incompatible change. + * + * @deprecated This class will be replaced by one that implements AWS SDK V2's AwsCredentialProvider + * as part of upgrading S3A to SDK V2. See HADOOP-18073. */ @InterfaceAudience.Private @InterfaceStability.Stable +@Deprecated public class AnonymousAWSCredentialsProvider implements AWSCredentialsProvider { public static final String NAME diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java index 764a6adaca27d..c6d21f2e554ac 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java @@ -140,6 +140,7 @@ private Constants() { public static final String ASSUMED_ROLE_POLICY = "fs.s3a.assumed.role.policy"; + @SuppressWarnings("deprecation") public static final String ASSUMED_ROLE_CREDENTIALS_DEFAULT = SimpleAWSCredentialsProvider.NAME; @@ -732,6 +733,7 @@ private Constants() { @InterfaceAudience.Private @InterfaceStability.Unstable + @SuppressWarnings("deprecation") public static final Class DEFAULT_S3_CLIENT_FACTORY_IMPL = DefaultS3ClientFactory.class; @@ -1203,4 +1205,34 @@ private Constants() { * Default maximum read size in bytes during vectored reads : {@value}. */ public static final int DEFAULT_AWS_S3_VECTOR_READS_MAX_MERGED_READ_SIZE = 1253376; //1M + + /** + * Prefix of auth classes in AWS SDK V1. + */ + public static final String AWS_AUTH_CLASS_PREFIX = "com.amazonaws.auth"; + + /** + * Controls whether the prefetching input stream is enabled. + */ + public static final String PREFETCH_ENABLED_KEY = "fs.s3a.prefetch.enabled"; + + /** + * Default option as to whether the prefetching input stream is enabled. + */ + public static final boolean PREFETCH_ENABLED_DEFAULT = false; + + // If the default values are used, each file opened for reading will consume + // 64 MB of heap space (8 blocks x 8 MB each). + + /** + * The size of a single prefetched block in number of bytes. + */ + public static final String PREFETCH_BLOCK_SIZE_KEY = "fs.s3a.prefetch.block.size"; + public static final int PREFETCH_BLOCK_DEFAULT_SIZE = 8 * 1024 * 1024; + + /** + * Maximum number of blocks prefetched at any given time. + */ + public static final String PREFETCH_BLOCK_COUNT_KEY = "fs.s3a.prefetch.block.count"; + public static final int PREFETCH_BLOCK_DEFAULT_COUNT = 8; } diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/DefaultS3ClientFactory.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/DefaultS3ClientFactory.java index c374ef7397c97..f724f86e4afcd 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/DefaultS3ClientFactory.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/DefaultS3ClientFactory.java @@ -71,6 +71,7 @@ */ @InterfaceAudience.Private @InterfaceStability.Unstable +@SuppressWarnings("deprecation") public class DefaultS3ClientFactory extends Configured implements S3ClientFactory { diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Listing.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Listing.java index a1dd4d8df0247..6c39cc4b64240 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Listing.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Listing.java @@ -31,7 +31,9 @@ import org.apache.hadoop.fs.s3a.impl.ListingOperationCallbacks; import org.apache.hadoop.fs.s3a.impl.StoreContext; import org.apache.hadoop.fs.statistics.IOStatistics; +import org.apache.hadoop.fs.statistics.IOStatisticsAggregator; import org.apache.hadoop.fs.statistics.IOStatisticsSource; +import org.apache.hadoop.fs.statistics.IOStatisticsContext; import org.apache.hadoop.fs.statistics.impl.IOStatisticsStore; import org.apache.hadoop.fs.store.audit.AuditSpan; import org.apache.hadoop.util.functional.RemoteIterators; @@ -333,7 +335,7 @@ interface FileStatusAcceptor { * Thread safety: None. */ class FileStatusListingIterator - implements RemoteIterator, IOStatisticsSource { + implements RemoteIterator, IOStatisticsSource, Closeable { /** Source of objects. */ private final ObjectListingIterator source; @@ -403,6 +405,14 @@ public S3AFileStatus next() throws IOException { return status; } + /** + * Close, if called, will update + * the thread statistics context with the value. + */ + @Override + public void close() { + source.close(); + } /** * Try to retrieve another batch. * Note that for the initial batch, @@ -545,6 +555,11 @@ class ObjectListingIterator implements RemoteIterator, private final AuditSpan span; + /** + * Context statistics aggregator. + */ + private final IOStatisticsAggregator aggregator; + /** The most recent listing results. */ private S3ListResult objects; @@ -601,6 +616,8 @@ class ObjectListingIterator implements RemoteIterator, this.span = span; this.s3ListResultFuture = listingOperationCallbacks .listObjectsAsync(request, iostats, span); + this.aggregator = IOStatisticsContext.getCurrentIOStatisticsContext() + .getAggregator(); } /** @@ -693,11 +710,12 @@ public int getListingCount() { } /** - * Close, if actually called, will close the span - * this listing was created with. + * Close, if called, will update + * the thread statistics context with the value. */ @Override public void close() { + aggregator.aggregate(getIOStatistics()); } } diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ABlockOutputStream.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ABlockOutputStream.java index 8b1865c77c9eb..19943ff2f70da 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ABlockOutputStream.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ABlockOutputStream.java @@ -41,6 +41,7 @@ import com.amazonaws.services.s3.model.UploadPartRequest; import org.apache.hadoop.fs.s3a.impl.PutObjectOptions; +import org.apache.hadoop.fs.statistics.IOStatisticsAggregator; import org.apache.hadoop.util.Preconditions; import org.apache.hadoop.thirdparty.com.google.common.util.concurrent.Futures; import org.apache.hadoop.thirdparty.com.google.common.util.concurrent.ListenableFuture; @@ -165,6 +166,9 @@ class S3ABlockOutputStream extends OutputStream implements /** is client side encryption enabled? */ private final boolean isCSEEnabled; + /** Thread level IOStatistics Aggregator. */ + private final IOStatisticsAggregator threadIOStatisticsAggregator; + /** * An S3A output stream which uploads partitions in a separate pool of * threads; different {@link S3ADataBlocks.BlockFactory} @@ -201,6 +205,7 @@ class S3ABlockOutputStream extends OutputStream implements initMultipartUpload(); } this.isCSEEnabled = builder.isCSEEnabled; + this.threadIOStatisticsAggregator = builder.ioStatisticsAggregator; } /** @@ -454,11 +459,23 @@ public void close() throws IOException { */ private synchronized void cleanupOnClose() { cleanupWithLogger(LOG, getActiveBlock(), blockFactory); + mergeThreadIOStatistics(statistics.getIOStatistics()); LOG.debug("Statistics: {}", statistics); cleanupWithLogger(LOG, statistics); clearActiveBlock(); } + /** + * Merging the current thread's IOStatistics with the current IOStatistics + * context. + * + * @param streamStatistics Stream statistics to be merged into thread + * statistics aggregator. + */ + private void mergeThreadIOStatistics(IOStatistics streamStatistics) { + getThreadIOStatistics().aggregate(streamStatistics); + } + /** * Best effort abort of the multipart upload; sets * the field to null afterwards. @@ -662,6 +679,10 @@ public boolean hasCapability(String capability) { case StreamCapabilities.ABORTABLE_STREAM: return true; + // IOStatistics context support for thread-level IOStatistics. + case StreamCapabilities.IOSTATISTICS_CONTEXT: + return true; + default: return false; } @@ -701,6 +722,14 @@ public IOStatistics getIOStatistics() { return iostatistics; } + /** + * Get the IOStatistics aggregator passed in the builder. + * @return an aggregator + */ + protected IOStatisticsAggregator getThreadIOStatistics() { + return threadIOStatisticsAggregator; + } + /** * Multiple partition upload. */ @@ -1092,6 +1121,11 @@ public static final class BlockOutputStreamBuilder { */ private PutObjectOptions putOptions; + /** + * thread-level IOStatistics Aggregator. + */ + private IOStatisticsAggregator ioStatisticsAggregator; + private BlockOutputStreamBuilder() { } @@ -1108,6 +1142,7 @@ public void validate() { requireNonNull(putOptions, "null putOptions"); Preconditions.checkArgument(blockSize >= Constants.MULTIPART_MIN_SIZE, "Block size is too small: %s", blockSize); + requireNonNull(ioStatisticsAggregator, "null ioStatisticsAggregator"); } /** @@ -1229,5 +1264,17 @@ public BlockOutputStreamBuilder withPutOptions( putOptions = value; return this; } + + /** + * Set builder value. + * + * @param value new value + * @return the builder + */ + public BlockOutputStreamBuilder withIOStatisticsAggregator( + final IOStatisticsAggregator value) { + ioStatisticsAggregator = value; + return this; + } } } diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java index 40671e0d334cc..c187026450c90 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java @@ -32,12 +32,14 @@ import java.util.Collections; import java.util.Date; import java.util.EnumSet; +import java.util.Iterator; import java.util.List; import java.util.Locale; import java.util.Map; import java.util.Optional; import java.util.Set; import java.util.Objects; +import java.util.TreeSet; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ExecutorService; import java.util.concurrent.LinkedBlockingQueue; @@ -52,6 +54,8 @@ import com.amazonaws.services.s3.AmazonS3; import com.amazonaws.services.s3.Headers; import com.amazonaws.services.s3.model.CannedAccessControlList; +import com.amazonaws.services.s3.model.CompleteMultipartUploadRequest; +import com.amazonaws.services.s3.model.CompleteMultipartUploadResult; import com.amazonaws.services.s3.model.CopyObjectRequest; import com.amazonaws.services.s3.model.DeleteObjectsRequest; import com.amazonaws.services.s3.model.DeleteObjectsResult; @@ -68,6 +72,8 @@ import com.amazonaws.services.s3.model.PutObjectRequest; import com.amazonaws.services.s3.model.PutObjectResult; import com.amazonaws.services.s3.model.S3Object; +import com.amazonaws.services.s3.model.SelectObjectContentRequest; +import com.amazonaws.services.s3.model.SelectObjectContentResult; import com.amazonaws.services.s3.model.StorageClass; import com.amazonaws.services.s3.model.UploadPartRequest; import com.amazonaws.services.s3.model.UploadPartResult; @@ -79,6 +85,7 @@ import com.amazonaws.services.s3.transfer.model.UploadResult; import com.amazonaws.event.ProgressListener; +import org.apache.hadoop.fs.impl.prefetch.ExecutorServiceFuturePool; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -124,6 +131,8 @@ import org.apache.hadoop.fs.s3a.impl.StatusProbeEnum; import org.apache.hadoop.fs.s3a.impl.StoreContext; import org.apache.hadoop.fs.s3a.impl.StoreContextBuilder; +import org.apache.hadoop.fs.s3a.impl.V2Migration; +import org.apache.hadoop.fs.s3a.prefetch.S3APrefetchingInputStream; import org.apache.hadoop.fs.s3a.tools.MarkerToolOperations; import org.apache.hadoop.fs.s3a.tools.MarkerToolOperationsImpl; import org.apache.hadoop.fs.statistics.DurationTracker; @@ -131,6 +140,7 @@ import org.apache.hadoop.fs.statistics.IOStatistics; import org.apache.hadoop.fs.statistics.IOStatisticsLogging; import org.apache.hadoop.fs.statistics.IOStatisticsSource; +import org.apache.hadoop.fs.statistics.IOStatisticsContext; import org.apache.hadoop.fs.statistics.impl.IOStatisticsStore; import org.apache.hadoop.fs.store.audit.AuditEntryPoint; import org.apache.hadoop.fs.store.audit.ActiveThreadSpanSource; @@ -288,6 +298,19 @@ public class S3AFileSystem extends FileSystem implements StreamCapabilities, private TransferManager transfers; private ExecutorService boundedThreadPool; private ThreadPoolExecutor unboundedThreadPool; + + // S3 reads are prefetched asynchronously using this future pool. + private ExecutorServiceFuturePool futurePool; + + // If true, the prefetching input stream is used for reads. + private boolean prefetchEnabled; + + // Size in bytes of a single prefetch block. + private int prefetchBlockSize; + + // Size of prefetch queue (in number of blocks). + private int prefetchBlockCount; + private int executorCapacity; private long multiPartThreshold; public static final Logger LOG = LoggerFactory.getLogger(S3AFileSystem.class); @@ -385,6 +408,12 @@ public class S3AFileSystem extends FileSystem implements StreamCapabilities, */ private ArnResource accessPoint; + /** + * A cache of files that should be deleted when the FileSystem is closed + * or the JVM is exited. + */ + private final Set deleteOnExit = new TreeSet<>(); + /** Add any deprecated keys. */ @SuppressWarnings("deprecation") private static void addDeprecatedKeys() { @@ -488,6 +517,17 @@ public void initialize(URI name, Configuration originalConf) longBytesOption(conf, FS_S3A_BLOCK_SIZE, DEFAULT_BLOCKSIZE, 1); enableMultiObjectsDelete = conf.getBoolean(ENABLE_MULTI_DELETE, true); + this.prefetchEnabled = conf.getBoolean(PREFETCH_ENABLED_KEY, PREFETCH_ENABLED_DEFAULT); + long prefetchBlockSizeLong = + longBytesOption(conf, PREFETCH_BLOCK_SIZE_KEY, PREFETCH_BLOCK_DEFAULT_SIZE, + PREFETCH_BLOCK_DEFAULT_SIZE); + if (prefetchBlockSizeLong > (long) Integer.MAX_VALUE) { + throw new IOException("S3A prefatch block size exceeds int limit"); + } + this.prefetchBlockSize = (int) prefetchBlockSizeLong; + this.prefetchBlockCount = + intOption(conf, PREFETCH_BLOCK_COUNT_KEY, PREFETCH_BLOCK_DEFAULT_COUNT, 1); + initThreadPools(conf); int listVersion = conf.getInt(LIST_VERSION, DEFAULT_LIST_VERSION); @@ -593,11 +633,17 @@ public void initialize(URI name, Configuration originalConf) // amazon client exception: stop all services then throw the translation cleanupWithLogger(LOG, span); stopAllServices(); + if (this.futurePool != null) { + this.futurePool = null; + } throw translateException("initializing ", new Path(name), e); } catch (IOException | RuntimeException e) { // other exceptions: stop the services. cleanupWithLogger(LOG, span); stopAllServices(); + if (this.futurePool != null) { + this.futurePool = null; + } throw e; } } @@ -718,9 +764,13 @@ private void initThreadPools(Configuration conf) { MAX_TOTAL_TASKS, DEFAULT_MAX_TOTAL_TASKS, 1); long keepAliveTime = longOption(conf, KEEPALIVE_TIME, DEFAULT_KEEPALIVE_TIME, 0); + int numPrefetchThreads = this.prefetchEnabled ? this.prefetchBlockCount : 0; + + int activeTasksForBoundedThreadPool = maxThreads; + int waitingTasksForBoundedThreadPool = maxThreads + totalTasks + numPrefetchThreads; boundedThreadPool = BlockingThreadPoolExecutorService.newInstance( - maxThreads, - maxThreads + totalTasks, + activeTasksForBoundedThreadPool, + waitingTasksForBoundedThreadPool, keepAliveTime, TimeUnit.SECONDS, name + "-bounded"); unboundedThreadPool = new ThreadPoolExecutor( @@ -732,6 +782,16 @@ private void initThreadPools(Configuration conf) { unboundedThreadPool.allowCoreThreadTimeOut(true); executorCapacity = intOption(conf, EXECUTOR_CAPACITY, DEFAULT_EXECUTOR_CAPACITY, 1); + if (prefetchEnabled) { + final S3AInputStreamStatistics s3AInputStreamStatistics = + statisticsContext.newInputStreamStatistics(); + futurePool = new ExecutorServiceFuturePool( + new SemaphoredDelegatingExecutor( + boundedThreadPool, + activeTasksForBoundedThreadPool + waitingTasksForBoundedThreadPool, + true, + s3AInputStreamStatistics)); + } } /** @@ -836,6 +896,7 @@ public Listing getListing() { * @param dtEnabled are delegation tokens enabled? * @throws IOException failure. */ + @SuppressWarnings("deprecation") private void bindAWSClient(URI name, boolean dtEnabled) throws IOException { Configuration conf = getConf(); credentials = null; @@ -848,6 +909,7 @@ private void bindAWSClient(URI name, boolean dtEnabled) throws IOException { // with it if so. LOG.debug("Using delegation tokens"); + V2Migration.v1DelegationTokenCredentialProvidersUsed(); S3ADelegationTokens tokens = new S3ADelegationTokens(); this.delegationTokens = Optional.of(tokens); tokens.bindToFileSystem(getCanonicalUri(), @@ -888,6 +950,7 @@ private void bindAWSClient(URI name, boolean dtEnabled) throws IOException { S3ClientFactory.S3ClientCreationParameters parameters = null; parameters = new S3ClientFactory.S3ClientCreationParameters() .withCredentialSet(credentials) + .withPathUri(name) .withEndpoint(endpoint) .withMetrics(statisticsContext.newStatisticsFromAwsSdk()) .withPathStyleAccess(conf.getBoolean(PATH_STYLE_ACCESS, false)) @@ -995,13 +1058,17 @@ protected RequestFactory createRequestFactory() { String storageClassConf = getConf() .getTrimmed(STORAGE_CLASS, "") .toUpperCase(Locale.US); - StorageClass storageClass; - try { - storageClass = StorageClass.fromValue(storageClassConf); - } catch (IllegalArgumentException e) { - LOG.warn("Unknown storage class property {}: {}; falling back to default storage class", - STORAGE_CLASS, storageClassConf); - storageClass = null; + StorageClass storageClass = null; + if (!storageClassConf.isEmpty()) { + try { + storageClass = StorageClass.fromValue(storageClassConf); + } catch (IllegalArgumentException e) { + LOG.warn("Unknown storage class property {}: {}; falling back to default storage class", + STORAGE_CLASS, storageClassConf); + } + } else { + LOG.debug("Unset storage class property {}; falling back to default storage class", + STORAGE_CLASS); } return RequestFactoryImpl.builder() @@ -1170,7 +1237,7 @@ public int getDefaultPort() { * This is for internal use within the S3A code itself. * @return AmazonS3Client */ - AmazonS3 getAmazonS3Client() { + private AmazonS3 getAmazonS3Client() { return s3; } @@ -1184,6 +1251,7 @@ AmazonS3 getAmazonS3Client() { @VisibleForTesting public AmazonS3 getAmazonS3ClientForTesting(String reason) { LOG.warn("Access to S3A client requested, reason {}", reason); + V2Migration.v1S3ClientRequested(); return s3; } @@ -1484,13 +1552,23 @@ private FSDataInputStream executeOpen( auditSpan); fileInformation.applyOptions(readContext); LOG.debug("Opening '{}'", readContext); - return new FSDataInputStream( - new S3AInputStream( - readContext.build(), - createObjectAttributes(path, fileStatus), - createInputStreamCallbacks(auditSpan), + + if (this.prefetchEnabled) { + return new FSDataInputStream( + new S3APrefetchingInputStream( + readContext.build(), + createObjectAttributes(path, fileStatus), + createInputStreamCallbacks(auditSpan), + inputStreamStats)); + } else { + return new FSDataInputStream( + new S3AInputStream( + readContext.build(), + createObjectAttributes(path, fileStatus), + createInputStreamCallbacks(auditSpan), inputStreamStats, unboundedThreadPool)); + } } /** @@ -1550,14 +1628,36 @@ public CompletableFuture submit(final CallableRaisingIOE operation) { CompletableFuture result = new CompletableFuture<>(); unboundedThreadPool.submit(() -> LambdaUtils.eval(result, () -> { + LOG.debug("Starting submitted operation in {}", auditSpan.getSpanId()); try (AuditSpan span = auditSpan.activate()) { return operation.apply(); + } finally { + LOG.debug("Completed submitted operation in {}", auditSpan.getSpanId()); } })); return result; } } + /** + * Callbacks for WriteOperationHelper. + */ + private final class WriteOperationHelperCallbacksImpl + implements WriteOperationHelper.WriteOperationHelperCallbacks { + + @Override + public SelectObjectContentResult selectObjectContent(SelectObjectContentRequest request) { + return s3.selectObjectContent(request); + } + + @Override + public CompleteMultipartUploadResult completeMultipartUpload( + CompleteMultipartUploadRequest request) { + return s3.completeMultipartUpload(request); + } + } + + /** * Create the read context for reading from the referenced file, * using FS state as well as the status. @@ -1575,7 +1675,11 @@ protected S3AReadOpContext createReadContext( statistics, statisticsContext, fileStatus, - vectoredIOContext) + vectoredIOContext, + IOStatisticsContext.getCurrentIOStatisticsContext().getAggregator(), + futurePool, + prefetchBlockSize, + prefetchBlockCount) .withAuditSpan(auditSpan); openFileHelper.applyDefaultOptions(roc); return roc.build(); @@ -1742,7 +1846,9 @@ private FSDataOutputStream innerCreateFile( DOWNGRADE_SYNCABLE_EXCEPTIONS, DOWNGRADE_SYNCABLE_EXCEPTIONS_DEFAULT)) .withCSEEnabled(isCSEEnabled) - .withPutOptions(putOptions); + .withPutOptions(putOptions) + .withIOStatisticsAggregator( + IOStatisticsContext.getCurrentIOStatisticsContext().getAggregator()); return new FSDataOutputStream( new S3ABlockOutputStream(builder), null); @@ -1776,7 +1882,8 @@ public WriteOperationHelper createWriteOperationHelper(AuditSpan auditSpan) { getConf(), statisticsContext, getAuditSpanSource(), - auditSpan); + auditSpan, + new WriteOperationHelperCallbacksImpl()); } /** @@ -2257,6 +2364,7 @@ public int getMaxKeys() { @Retries.RetryTranslated @InterfaceStability.Evolving public ObjectMetadata getObjectMetadata(Path path) throws IOException { + V2Migration.v1GetObjectMetadataCalled(); return trackDurationAndSpan(INVOCATION_GET_FILE_STATUS, path, () -> getObjectMetadata(makeQualified(path), null, invoker, "getObjectMetadata")); @@ -3059,6 +3167,24 @@ public void removeKeys( @AuditEntryPoint public boolean delete(Path f, boolean recursive) throws IOException { checkNotClosed(); + return deleteWithoutCloseCheck(f, recursive); + } + + /** + * Same as delete(), except that it does not check if fs is closed. + * + * @param f the path to delete. + * @param recursive if path is a directory and set to + * true, the directory is deleted else throws an exception. In + * case of a file the recursive can be set to either true or false. + * @return true if the path existed and then was deleted; false if there + * was no path in the first place, or the corner cases of root path deletion + * have surfaced. + * @throws IOException due to inability to delete a directory or file. + */ + + @VisibleForTesting + protected boolean deleteWithoutCloseCheck(Path f, boolean recursive) throws IOException { final Path path = qualify(f); // span covers delete, getFileStatus, fake directory operations. try (AuditSpan span = createSpan(INVOCATION_DELETE.getSymbol(), @@ -3671,12 +3797,12 @@ public void copyFromLocalFile(boolean delSrc, boolean overwrite, Path src, } protected CopyFromLocalOperation.CopyFromLocalOperationCallbacks - createCopyFromLocalCallbacks() throws IOException { + createCopyFromLocalCallbacks() throws IOException { LocalFileSystem local = getLocal(getConf()); return new CopyFromLocalCallbacksImpl(local); } - protected class CopyFromLocalCallbacksImpl implements + protected final class CopyFromLocalCallbacksImpl implements CopyFromLocalOperation.CopyFromLocalOperationCallbacks { private final LocalFileSystem local; @@ -3800,6 +3926,61 @@ UploadResult waitForUploadCompletion(String key, UploadInfo uploadInfo) } } + /** + * This override bypasses checking for existence. + * + * @param f the path to delete; this may be unqualified. + * @return true, always. * @param f the path to delete. + * @return true if deleteOnExit is successful, otherwise false. + * @throws IOException IO failure + */ + @Override + public boolean deleteOnExit(Path f) throws IOException { + Path qualifedPath = makeQualified(f); + synchronized (deleteOnExit) { + deleteOnExit.add(qualifedPath); + } + return true; + } + + /** + * Cancel the scheduled deletion of the path when the FileSystem is closed. + * @param f the path to cancel deletion + * @return true if the path was found in the delete-on-exit list. + */ + @Override + public boolean cancelDeleteOnExit(Path f) { + Path qualifedPath = makeQualified(f); + synchronized (deleteOnExit) { + return deleteOnExit.remove(qualifedPath); + } + } + + /** + * Delete all paths that were marked as delete-on-exit. This recursively + * deletes all files and directories in the specified paths. It does not + * check if file exists and filesystem is closed. + * + * The time to process this operation is {@code O(paths)}, with the actual + * time dependent on the time for existence and deletion operations to + * complete, successfully or not. + */ + @Override + protected void processDeleteOnExit() { + synchronized (deleteOnExit) { + for (Iterator iter = deleteOnExit.iterator(); iter.hasNext();) { + Path path = iter.next(); + try { + deleteWithoutCloseCheck(path, true); + } catch (IOException e) { + LOG.info("Ignoring failure to deleteOnExit for path {}", path); + LOG.debug("The exception for deleteOnExit is {}", e); + } + iter.remove(); + } + } + } + /** * Close the filesystem. This shuts down all transfers. * @throws IOException IO problem diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AInputStream.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AInputStream.java index 3069f17289119..b6ac8669a6734 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AInputStream.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AInputStream.java @@ -37,6 +37,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.apache.hadoop.util.Preconditions; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.classification.VisibleForTesting; @@ -50,12 +51,14 @@ import org.apache.hadoop.fs.impl.CombinedFileRange; import org.apache.hadoop.fs.VectoredReadUtils; import org.apache.hadoop.fs.s3a.impl.ChangeTracker; +import org.apache.hadoop.fs.s3a.impl.InternalConstants; +import org.apache.hadoop.fs.s3a.impl.SDKStreamDrainer; import org.apache.hadoop.fs.s3a.statistics.S3AInputStreamStatistics; import org.apache.hadoop.fs.statistics.DurationTracker; import org.apache.hadoop.fs.statistics.IOStatistics; +import org.apache.hadoop.fs.statistics.IOStatisticsAggregator; import org.apache.hadoop.fs.statistics.IOStatisticsSource; import org.apache.hadoop.io.IOUtils; -import org.apache.hadoop.util.Preconditions; import org.apache.hadoop.util.functional.CallableRaisingIOE; import static java.util.Objects.requireNonNull; @@ -64,7 +67,6 @@ import static org.apache.hadoop.fs.VectoredReadUtils.mergeSortedRanges; import static org.apache.hadoop.fs.VectoredReadUtils.validateNonOverlappingAndReturnSortedRanges; import static org.apache.hadoop.fs.s3a.Invoker.onceTrackingDuration; -import static org.apache.hadoop.fs.statistics.impl.IOStatisticsBinding.invokeTrackingDuration; import static org.apache.hadoop.util.StringUtils.toLowerCase; import static org.apache.hadoop.util.functional.FutureIO.awaitFuture; @@ -96,10 +98,6 @@ public class S3AInputStream extends FSInputStream implements CanSetReadahead, public static final String OPERATION_OPEN = "open"; public static final String OPERATION_REOPEN = "re-open"; - /** - * size of a buffer to create when draining the stream. - */ - private static final int DRAIN_BUFFER_SIZE = 16384; /** * This is the maximum temporary buffer size we use while * populating the data in direct byte buffers during a vectored IO @@ -187,6 +185,9 @@ public class S3AInputStream extends FSInputStream implements CanSetReadahead, */ private long asyncDrainThreshold; + /** Aggregator used to aggregate per thread IOStatistics. */ + private final IOStatisticsAggregator threadIOStatistics; + /** * Create the stream. * This does not attempt to open it; that is only done on the first @@ -225,6 +226,7 @@ public S3AInputStream(S3AReadOpContext ctx, this.asyncDrainThreshold = ctx.getAsyncDrainThreshold(); this.unboundedThreadPool = unboundedThreadPool; this.vectoredIOContext = context.getVectoredIOContext(); + this.threadIOStatistics = requireNonNull(ctx.getIOStatisticsAggregator()); } /** @@ -237,6 +239,15 @@ private void setInputPolicy(S3AInputPolicy inputPolicy) { streamStatistics.inputPolicySet(inputPolicy.ordinal()); } + /** + * Get the current input policy. + * @return input policy. + */ + @VisibleForTesting + public S3AInputPolicy getInputPolicy() { + return inputPolicy; + } + /** * Opens up the stream at specified target position and for given length. * @@ -599,8 +610,7 @@ public synchronized void close() throws IOException { try { stopVectoredIOOperations.set(true); // close or abort the stream; blocking - awaitFuture(closeStream("close() operation", false, true)); - LOG.debug("Statistics of stream {}\n{}", key, streamStatistics); + closeStream("close() operation", false, true); // end the client+audit span. client.close(); // this is actually a no-op @@ -608,10 +618,23 @@ public synchronized void close() throws IOException { } finally { // merge the statistics back into the FS statistics. streamStatistics.close(); + // Collect ThreadLevel IOStats + mergeThreadIOStatistics(streamStatistics.getIOStatistics()); } } } + /** + * Merging the current thread's IOStatistics with the current IOStatistics + * context. + * + * @param streamIOStats Stream statistics to be merged into thread + * statistics aggregator. + */ + private void mergeThreadIOStatistics(IOStatistics streamIOStats) { + threadIOStatistics.aggregate(streamIOStats); + } + /** * Close a stream: decide whether to abort or close, based on * the length of the stream and the current position. @@ -647,18 +670,25 @@ private CompletableFuture closeStream( forceAbort ? "abort" : "soft"); boolean shouldAbort = forceAbort || remaining > readahead; CompletableFuture operation; + SDKStreamDrainer drainer = new SDKStreamDrainer( + uri, + object, + wrappedStream, + shouldAbort, + (int) remaining, + streamStatistics, + reason); if (blocking || shouldAbort || remaining <= asyncDrainThreshold) { - // don't bother with async io. - operation = CompletableFuture.completedFuture( - drain(shouldAbort, reason, remaining, object, wrappedStream)); + // don't bother with async IO if the caller plans to wait for + // the result, there's an abort (which is fast), or + // there is not much data to read. + operation = CompletableFuture.completedFuture(drainer.apply()); } else { LOG.debug("initiating asynchronous drain of {} bytes", remaining); - // schedule an async drain/abort with references to the fields so they - // can be reused - operation = client.submit( - () -> drain(false, reason, remaining, object, wrappedStream)); + // schedule an async drain/abort + operation = client.submit(drainer); } // either the stream is closed in the blocking call or the async call is @@ -668,117 +698,6 @@ private CompletableFuture closeStream( return operation; } - /** - * drain the stream. This method is intended to be - * used directly or asynchronously, and measures the - * duration of the operation in the stream statistics. - * @param shouldAbort force an abort; used if explicitly requested. - * @param reason reason for stream being closed; used in messages - * @param remaining remaining bytes - * @param requestObject http request object; needed to avoid GC issues. - * @param inner stream to close. - * @return was the stream aborted? - */ - private boolean drain( - final boolean shouldAbort, - final String reason, - final long remaining, - final S3Object requestObject, - final S3ObjectInputStream inner) { - - try { - return invokeTrackingDuration( - streamStatistics.initiateInnerStreamClose(shouldAbort), - () -> drainOrAbortHttpStream( - shouldAbort, - reason, - remaining, - requestObject, - inner)); - } catch (IOException e) { - // this is only here because invokeTrackingDuration() has it in its - // signature - return shouldAbort; - } - } - - /** - * Drain or abort the inner stream. - * Exceptions are swallowed. - * If a close() is attempted and fails, the operation escalates to - * an abort. - * - * This does not set the {@link #closed} flag. - * - * A reference to the stream is passed in so that the instance - * {@link #wrappedStream} field can be reused as soon as this - * method is submitted; - * @param shouldAbort force an abort; used if explicitly requested. - * @param reason reason for stream being closed; used in messages - * @param remaining remaining bytes - * @param requestObject http request object; needed to avoid GC issues. - * @param inner stream to close. - * @return was the stream aborted? - */ - private boolean drainOrAbortHttpStream( - boolean shouldAbort, - final String reason, - final long remaining, - final S3Object requestObject, - final S3ObjectInputStream inner) { - // force a use of the request object so IDEs don't warn of - // lack of use. - requireNonNull(requestObject); - - if (!shouldAbort) { - try { - // clean close. This will read to the end of the stream, - // so, while cleaner, can be pathological on a multi-GB object - - // explicitly drain the stream - long drained = 0; - byte[] buffer = new byte[DRAIN_BUFFER_SIZE]; - while (true) { - final int count = inner.read(buffer); - if (count < 0) { - // no more data is left - break; - } - drained += count; - } - LOG.debug("Drained stream of {} bytes", drained); - - // now close it - inner.close(); - // this MUST come after the close, so that if the IO operations fail - // and an abort is triggered, the initial attempt's statistics - // aren't collected. - streamStatistics.streamClose(false, drained); - } catch (Exception e) { - // exception escalates to an abort - LOG.debug("When closing {} stream for {}, will abort the stream", - uri, reason, e); - shouldAbort = true; - } - } - if (shouldAbort) { - // Abort, rather than just close, the underlying stream. Otherwise, the - // remaining object payload is read from S3 while closing the stream. - LOG.debug("Aborting stream {}", uri); - try { - inner.abort(); - } catch (Exception e) { - LOG.warn("When aborting {} stream after failing to close it for {}", - uri, reason, e); - } - streamStatistics.streamClose(true, remaining); - } - LOG.debug("Stream {} {}: {}; remaining={}", - uri, (shouldAbort ? "aborted" : "closed"), reason, - remaining); - return shouldAbort; - } - /** * Forcibly reset the stream, by aborting the connection. The next * {@code read()} operation will trigger the opening of a new HTTPS @@ -946,7 +865,6 @@ public int maxReadSizeForVectorReads() { @Override public void readVectored(List ranges, IntFunction allocate) throws IOException { - LOG.debug("Starting vectored read on path {} for ranges {} ", pathStr, ranges); checkNotClosed(); if (stopVectoredIOOperations.getAndSet(false)) { @@ -961,6 +879,7 @@ public void readVectored(List ranges, if (isOrderedDisjoint(sortedRanges, 1, minSeekForVectorReads())) { LOG.debug("Not merging the ranges as they are disjoint"); + streamStatistics.readVectoredOperationStarted(sortedRanges.size(), sortedRanges.size()); for (FileRange range: sortedRanges) { ByteBuffer buffer = allocate.apply(range.getLength()); unboundedThreadPool.submit(() -> readSingleRange(range, buffer)); @@ -970,6 +889,7 @@ public void readVectored(List ranges, List combinedFileRanges = mergeSortedRanges(sortedRanges, 1, minSeekForVectorReads(), maxReadSizeForVectorReads()); + streamStatistics.readVectoredOperationStarted(sortedRanges.size(), combinedFileRanges.size()); LOG.debug("Number of original ranges size {} , Number of combined ranges {} ", ranges.size(), combinedFileRanges.size()); for (CombinedFileRange combinedFileRange: combinedFileRanges) { @@ -1062,8 +982,8 @@ private void drainUnnecessaryData(S3ObjectInputStream objectContent, long drainQ int drainBytes = 0; int readCount; while (drainBytes < drainQuantity) { - if (drainBytes + DRAIN_BUFFER_SIZE <= drainQuantity) { - byte[] drainBuffer = new byte[DRAIN_BUFFER_SIZE]; + if (drainBytes + InternalConstants.DRAIN_BUFFER_SIZE <= drainQuantity) { + byte[] drainBuffer = new byte[InternalConstants.DRAIN_BUFFER_SIZE]; readCount = objectContent.read(drainBuffer); } else { byte[] drainBuffer = new byte[(int) (drainQuantity - drainBytes)]; @@ -1071,6 +991,7 @@ private void drainUnnecessaryData(S3ObjectInputStream objectContent, long drainQ } drainBytes += readCount; } + streamStatistics.readVectoredBytesDiscarded(drainBytes); LOG.debug("{} bytes drained from stream ", drainBytes); } @@ -1084,10 +1005,10 @@ private void drainUnnecessaryData(S3ObjectInputStream objectContent, long drainQ private void validateRangeRequest(FileRange range) throws EOFException { VectoredReadUtils.validateRangeRequest(range); if(range.getOffset() + range.getLength() > contentLength) { - LOG.warn("Requested range [{}, {}) is beyond EOF for path {}", + final String errMsg = String.format("Requested range [%d, %d) is beyond EOF for path %s", range.getOffset(), range.getLength(), pathStr); - throw new EOFException("Requested range [" + range.getOffset() +", " - + range.getLength() + ") is beyond EOF for path " + pathStr); + LOG.warn(errMsg); + throw new EOFException(errMsg); } } @@ -1133,26 +1054,22 @@ private void readSingleRange(FileRange range, ByteBuffer buffer) { private void populateBuffer(int length, ByteBuffer buffer, S3ObjectInputStream objectContent) throws IOException { + if (buffer.isDirect()) { - int readBytes = 0; - int offset = 0; - byte[] tmp = new byte[TMP_BUFFER_MAX_SIZE]; - while (readBytes < length) { - checkIfVectoredIOStopped(); - int currentLength = readBytes + TMP_BUFFER_MAX_SIZE < length ? - TMP_BUFFER_MAX_SIZE - : length - readBytes; - readByteArray(objectContent, tmp, 0, currentLength); - buffer.put(tmp, 0, currentLength); - offset = offset + currentLength; - readBytes = readBytes + currentLength; - } + VectoredReadUtils.readInDirectBuffer(length, buffer, + (position, tmp, offset, currentLength) -> { + readByteArray(objectContent, tmp, offset, currentLength); + return null; + }); buffer.flip(); } else { readByteArray(objectContent, buffer.array(), 0, length); } + // update io stats. + incrementBytesRead(length); } + /** * Read data into destination buffer from s3 object content. * @param objectContent result from S3. @@ -1324,6 +1241,11 @@ public synchronized void unbuffer() { closeStream("unbuffer()", false, false); } finally { streamStatistics.unbuffered(); + if (inputPolicy.isAdaptive()) { + S3AInputPolicy policy = S3AInputPolicy.Random; + LOG.debug("Switching to seek policy {} after unbuffer() invoked", policy); + setInputPolicy(policy); + } } } @@ -1331,6 +1253,7 @@ public synchronized void unbuffer() { public boolean hasCapability(String capability) { switch (toLowerCase(capability)) { case StreamCapabilities.IOSTATISTICS: + case StreamCapabilities.IOSTATISTICS_CONTEXT: case StreamCapabilities.READAHEAD: case StreamCapabilities.UNBUFFER: case StreamCapabilities.VECTOREDIO: diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AInstrumentation.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AInstrumentation.java index 67734b7502976..46568ec2a8d3d 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AInstrumentation.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AInstrumentation.java @@ -803,6 +803,10 @@ private final class InputStreamStatistics private final AtomicLong readOperations; private final AtomicLong readFullyOperations; private final AtomicLong seekOperations; + private final AtomicLong readVectoredOperations; + private final AtomicLong bytesDiscardedInVectoredIO; + private final AtomicLong readVectoredIncomingRanges; + private final AtomicLong readVectoredCombinedRanges; /** Bytes read by the application and any when draining streams . */ private final AtomicLong totalBytesRead; @@ -836,12 +840,24 @@ private InputStreamStatistics( StreamStatisticNames.STREAM_READ_SEEK_BYTES_SKIPPED, StreamStatisticNames.STREAM_READ_TOTAL_BYTES, StreamStatisticNames.STREAM_READ_UNBUFFERED, + StreamStatisticNames.STREAM_READ_VECTORED_COMBINED_RANGES, + StreamStatisticNames.STREAM_READ_VECTORED_INCOMING_RANGES, + StreamStatisticNames.STREAM_READ_VECTORED_OPERATIONS, + StreamStatisticNames.STREAM_READ_VECTORED_READ_BYTES_DISCARDED, StreamStatisticNames.STREAM_READ_VERSION_MISMATCHES) - .withGauges(STREAM_READ_GAUGE_INPUT_POLICY) + .withGauges(STREAM_READ_GAUGE_INPUT_POLICY, + STREAM_READ_BLOCKS_IN_FILE_CACHE.getSymbol(), + STREAM_READ_ACTIVE_PREFETCH_OPERATIONS.getSymbol(), + STREAM_READ_ACTIVE_MEMORY_IN_USE.getSymbol() + ) .withDurationTracking(ACTION_HTTP_GET_REQUEST, + ACTION_EXECUTOR_ACQUIRED, StoreStatisticNames.ACTION_FILE_OPENED, StreamStatisticNames.STREAM_READ_REMOTE_STREAM_ABORTED, - StreamStatisticNames.STREAM_READ_REMOTE_STREAM_DRAINED) + StreamStatisticNames.STREAM_READ_REMOTE_STREAM_DRAINED, + StreamStatisticNames.STREAM_READ_PREFETCH_OPERATIONS, + StreamStatisticNames.STREAM_READ_REMOTE_BLOCK_READ, + StreamStatisticNames.STREAM_READ_BLOCK_ACQUIRE_AND_READ) .build(); setIOStatistics(st); aborted = st.getCounterReference( @@ -872,6 +888,14 @@ private InputStreamStatistics( StreamStatisticNames.STREAM_READ_OPERATIONS_INCOMPLETE); readOperations = st.getCounterReference( StreamStatisticNames.STREAM_READ_OPERATIONS); + readVectoredOperations = st.getCounterReference( + StreamStatisticNames.STREAM_READ_VECTORED_OPERATIONS); + bytesDiscardedInVectoredIO = st.getCounterReference( + StreamStatisticNames.STREAM_READ_VECTORED_READ_BYTES_DISCARDED); + readVectoredIncomingRanges = st.getCounterReference( + StreamStatisticNames.STREAM_READ_VECTORED_INCOMING_RANGES); + readVectoredCombinedRanges = st.getCounterReference( + StreamStatisticNames.STREAM_READ_VECTORED_COMBINED_RANGES); readFullyOperations = st.getCounterReference( StreamStatisticNames.STREAM_READ_FULLY_OPERATIONS); seekOperations = st.getCounterReference( @@ -902,6 +926,18 @@ private long increment(String name, long value) { return incCounter(name, value); } + /** + * Increment the Statistic gauge and the local IOStatistics + * equivalent. + * @param statistic statistic + * @param v value. + * @return local IOStatistic value + */ + private long incAllGauges(Statistic statistic, long v) { + incrementGauge(statistic, v); + return incGauge(statistic.getSymbol(), v); + } + /** * {@inheritDoc}. * Increments the number of seek operations, @@ -1017,6 +1053,25 @@ public void readOperationCompleted(int requested, int actual) { } } + @Override + public void readVectoredOperationStarted(int numIncomingRanges, + int numCombinedRanges) { + readVectoredIncomingRanges.addAndGet(numIncomingRanges); + readVectoredCombinedRanges.addAndGet(numCombinedRanges); + readVectoredOperations.incrementAndGet(); + } + + @Override + public void readVectoredBytesDiscarded(int discarded) { + bytesDiscardedInVectoredIO.addAndGet(discarded); + } + + @Override + public void executorAcquired(Duration timeInQueue) { + // update the duration fields in the IOStatistics. + localIOStatistics().addTimedOperation(ACTION_EXECUTOR_ACQUIRED, timeInQueue); + } + /** * {@code close()} merges the stream statistics into the filesystem's * instrumentation instance. @@ -1281,6 +1336,37 @@ public DurationTracker initiateInnerStreamClose(final boolean abort) { ? StreamStatisticNames.STREAM_READ_REMOTE_STREAM_ABORTED : StreamStatisticNames.STREAM_READ_REMOTE_STREAM_DRAINED); } + + @Override + public DurationTracker prefetchOperationStarted() { + incAllGauges(STREAM_READ_ACTIVE_PREFETCH_OPERATIONS, 1); + return trackDuration(StreamStatisticNames.STREAM_READ_PREFETCH_OPERATIONS); + } + + @Override + public void blockAddedToFileCache() { + incAllGauges(STREAM_READ_BLOCKS_IN_FILE_CACHE, 1); + } + + @Override + public void blockRemovedFromFileCache() { + incAllGauges(STREAM_READ_BLOCKS_IN_FILE_CACHE, -1); + } + + @Override + public void prefetchOperationCompleted() { + incAllGauges(STREAM_READ_ACTIVE_PREFETCH_OPERATIONS, -1); + } + + @Override + public void memoryAllocated(int size) { + incAllGauges(STREAM_READ_ACTIVE_MEMORY_IN_USE, size); + } + + @Override + public void memoryFreed(int size) { + incAllGauges(STREAM_READ_ACTIVE_MEMORY_IN_USE, -size); + } } /** diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AReadOpContext.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AReadOpContext.java index 803b7757d252b..b19a70dec915e 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AReadOpContext.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AReadOpContext.java @@ -21,8 +21,10 @@ import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.impl.prefetch.ExecutorServiceFuturePool; import org.apache.hadoop.fs.s3a.impl.ChangeDetectionPolicy; import org.apache.hadoop.fs.s3a.statistics.S3AStatisticsContext; +import org.apache.hadoop.fs.statistics.IOStatisticsAggregator; import org.apache.hadoop.fs.store.audit.AuditSpan; import javax.annotation.Nullable; @@ -70,6 +72,18 @@ public class S3AReadOpContext extends S3AOpContext { */ private final VectoredIOContext vectoredIOContext; + /** Thread-level IOStatistics aggregator. **/ + private final IOStatisticsAggregator ioStatisticsAggregator; + + // S3 reads are prefetched asynchronously using this future pool. + private ExecutorServiceFuturePool futurePool; + + // Size in bytes of a single prefetch block. + private final int prefetchBlockSize; + + // Size of prefetch queue (in number of blocks). + private final int prefetchBlockCount; + /** * Instantiate. * @param path path of read @@ -78,6 +92,10 @@ public class S3AReadOpContext extends S3AOpContext { * @param instrumentation statistics context * @param dstFileStatus target file status * @param vectoredIOContext context for vectored read operation. + * @param ioStatisticsAggregator IOStatistics aggregator for each thread. + * @param futurePool the ExecutorServiceFuturePool instance used by async prefetches. + * @param prefetchBlockSize the size (in number of bytes) of each prefetched block. + * @param prefetchBlockCount maximum number of prefetched blocks. */ public S3AReadOpContext( final Path path, @@ -85,11 +103,24 @@ public S3AReadOpContext( @Nullable FileSystem.Statistics stats, S3AStatisticsContext instrumentation, FileStatus dstFileStatus, - VectoredIOContext vectoredIOContext) { + VectoredIOContext vectoredIOContext, + IOStatisticsAggregator ioStatisticsAggregator, + ExecutorServiceFuturePool futurePool, + int prefetchBlockSize, + int prefetchBlockCount) { + super(invoker, stats, instrumentation, dstFileStatus); this.path = requireNonNull(path); this.vectoredIOContext = requireNonNull(vectoredIOContext, "vectoredIOContext"); + this.ioStatisticsAggregator = ioStatisticsAggregator; + this.futurePool = futurePool; + Preconditions.checkArgument( + prefetchBlockSize > 0, "invalid prefetchBlockSize %d", prefetchBlockSize); + this.prefetchBlockSize = prefetchBlockSize; + Preconditions.checkArgument( + prefetchBlockCount > 0, "invalid prefetchBlockCount %d", prefetchBlockCount); + this.prefetchBlockCount = prefetchBlockCount; } /** @@ -105,6 +136,7 @@ public S3AReadOpContext build() { "invalid readahead %d", readahead); Preconditions.checkArgument(asyncDrainThreshold >= 0, "invalid drainThreshold %d", asyncDrainThreshold); + requireNonNull(ioStatisticsAggregator, "ioStatisticsAggregator"); return this; } @@ -215,6 +247,42 @@ public VectoredIOContext getVectoredIOContext() { return vectoredIOContext; } + /** + * Return the IOStatistics aggregator. + * + * @return instance of IOStatisticsAggregator. + */ + public IOStatisticsAggregator getIOStatisticsAggregator() { + return ioStatisticsAggregator; + } + + /** + * Gets the {@code ExecutorServiceFuturePool} used for asynchronous prefetches. + * + * @return the {@code ExecutorServiceFuturePool} used for asynchronous prefetches. + */ + public ExecutorServiceFuturePool getFuturePool() { + return this.futurePool; + } + + /** + * Gets the size in bytes of a single prefetch block. + * + * @return the size in bytes of a single prefetch block. + */ + public int getPrefetchBlockSize() { + return this.prefetchBlockSize; + } + + /** + * Gets the size of prefetch queue (in number of blocks). + * + * @return the size of prefetch queue (in number of blocks). + */ + public int getPrefetchBlockCount() { + return this.prefetchBlockCount; + } + @Override public String toString() { final StringBuilder sb = new StringBuilder( diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AUtils.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AUtils.java index e7e741d42c521..7cc7d635c51f0 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AUtils.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AUtils.java @@ -46,6 +46,7 @@ import org.apache.hadoop.fs.s3a.auth.delegation.EncryptionSecrets; import org.apache.hadoop.fs.s3a.auth.IAMInstanceCredentialsProvider; import org.apache.hadoop.fs.s3a.impl.NetworkBinding; +import org.apache.hadoop.fs.s3a.impl.V2Migration; import org.apache.hadoop.fs.s3native.S3xLoginHelper; import org.apache.hadoop.net.ConnectTimeoutException; import org.apache.hadoop.security.ProviderUtils; @@ -378,6 +379,7 @@ private static InterruptedIOException translateInterruptedException( } else { String name = innerCause.getClass().getName(); if (name.endsWith(".ConnectTimeoutException") + || name.endsWith(".ConnectionPoolTimeoutException") || name.endsWith("$ConnectTimeoutException")) { // TCP connection http timeout from the shaded or unshaded filenames // com.amazonaws.thirdparty.apache.http.conn.ConnectTimeoutException @@ -551,6 +553,7 @@ public static long dateToLong(final Date date) { /** * The standard AWS provider list for AWS connections. */ + @SuppressWarnings("deprecation") public static final List> STANDARD_AWS_PROVIDERS = Collections.unmodifiableList( Arrays.asList( @@ -637,6 +640,10 @@ public static AWSCredentialProviderList buildAWSProviderList( AWSCredentialProviderList providers = new AWSCredentialProviderList(); for (Class aClass : awsClasses) { + if (aClass.getName().contains(AWS_AUTH_CLASS_PREFIX)) { + V2Migration.v1ProviderReferenced(aClass.getName()); + } + if (forbidden.contains(aClass)) { throw new IOException(E_FORBIDDEN_AWS_PROVIDER + " in option " + key + ": " + aClass); diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ClientFactory.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ClientFactory.java index 34674c788901f..9010f34dc259c 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ClientFactory.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ClientFactory.java @@ -44,9 +44,12 @@ * implementing only the deprecated method will work. * See https://github.com/apache/hbase-filesystem * + * @deprecated This interface will be replaced by one which uses the AWS SDK V2 S3 client as part of + * upgrading S3A to SDK V2. See HADOOP-18073. */ @InterfaceAudience.LimitedPrivate("HBoss") @InterfaceStability.Evolving +@Deprecated public interface S3ClientFactory { /** @@ -115,6 +118,12 @@ final class S3ClientCreationParameters { */ private String userAgentSuffix = ""; + /** + * S3A path. + * added in HADOOP-18330 + */ + private URI pathUri; + /** * List of request handlers to include in the chain * of request execution in the SDK. @@ -264,5 +273,26 @@ public S3ClientCreationParameters withHeader( public Map getHeaders() { return headers; } + + /** + * Get the full s3 path. + * added in HADOOP-18330 + * @return path URI + */ + public URI getPathUri() { + return pathUri; + } + + /** + * Set full s3a path. + * added in HADOOP-18330 + * @param value new value + * @return the builder + */ + public S3ClientCreationParameters withPathUri( + final URI value) { + pathUri = value; + return this; + } } } diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/SharedInstanceCredentialProvider.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/SharedInstanceCredentialProvider.java index 5eba675cb82c2..6579a2bc3e7d2 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/SharedInstanceCredentialProvider.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/SharedInstanceCredentialProvider.java @@ -39,6 +39,6 @@ */ @InterfaceAudience.Public @InterfaceStability.Evolving -public final class SharedInstanceCredentialProvider extends - IAMInstanceCredentialsProvider { +@SuppressWarnings("deprecation") +public final class SharedInstanceCredentialProvider extends IAMInstanceCredentialsProvider { } diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/SimpleAWSCredentialsProvider.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/SimpleAWSCredentialsProvider.java index a8e08e158cac4..50a2dd5fb3fc2 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/SimpleAWSCredentialsProvider.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/SimpleAWSCredentialsProvider.java @@ -41,9 +41,13 @@ * Please note that users may reference this class name from configuration * property fs.s3a.aws.credentials.provider. Therefore, changing the class name * would be a backward-incompatible change. + * + * @deprecated This class will be replaced by one that implements AWS SDK V2's AwsCredentialProvider + * as part of upgrading S3A to SDK V2. See HADOOP-18073. */ @InterfaceAudience.Public @InterfaceStability.Stable +@Deprecated public class SimpleAWSCredentialsProvider implements AWSCredentialsProvider { public static final String NAME diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Statistic.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Statistic.java index dfe9fdf2d8d37..651769ff283bd 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Statistic.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Statistic.java @@ -308,6 +308,23 @@ public enum Statistic { StreamStatisticNames.STREAM_READ_OPERATIONS, "Count of read() operations in an input stream", TYPE_COUNTER), + STREAM_READ_VECTORED_OPERATIONS( + StreamStatisticNames.STREAM_READ_VECTORED_OPERATIONS, + "Count of readVectored() operations in an input stream.", + TYPE_COUNTER), + STREAM_READ_VECTORED_READ_BYTES_DISCARDED( + StreamStatisticNames.STREAM_READ_VECTORED_READ_BYTES_DISCARDED, + "Count of bytes discarded during readVectored() operation." + + " in an input stream", + TYPE_COUNTER), + STREAM_READ_VECTORED_INCOMING_RANGES( + StreamStatisticNames.STREAM_READ_VECTORED_INCOMING_RANGES, + "Count of incoming file ranges during readVectored() operation.", + TYPE_COUNTER), + STREAM_READ_VECTORED_COMBINED_RANGES( + StreamStatisticNames.STREAM_READ_VECTORED_COMBINED_RANGES, + "Count of combined file ranges during readVectored() operation.", + TYPE_COUNTER), STREAM_READ_REMOTE_STREAM_ABORTED( StreamStatisticNames.STREAM_READ_REMOTE_STREAM_ABORTED, "Duration of aborting a remote stream during stream IO", @@ -361,6 +378,22 @@ public enum Statistic { StreamStatisticNames.STREAM_READ_TOTAL_BYTES, "Total count of bytes read from an input stream", TYPE_COUNTER), + STREAM_READ_UNBUFFERED( + StreamStatisticNames.STREAM_READ_UNBUFFERED, + "Total count of input stream unbuffering operations", + TYPE_COUNTER), + STREAM_READ_BLOCKS_IN_FILE_CACHE( + StreamStatisticNames.STREAM_READ_BLOCKS_IN_FILE_CACHE, + "Gauge of blocks in disk cache", + TYPE_GAUGE), + STREAM_READ_ACTIVE_PREFETCH_OPERATIONS( + StreamStatisticNames.STREAM_READ_ACTIVE_PREFETCH_OPERATIONS, + "Gauge of active prefetches", + TYPE_GAUGE), + STREAM_READ_ACTIVE_MEMORY_IN_USE( + StreamStatisticNames.STREAM_READ_ACTIVE_MEMORY_IN_USE, + "Gauge of active memory in use", + TYPE_GAUGE), /* Stream Write statistics */ diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/TemporaryAWSCredentialsProvider.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/TemporaryAWSCredentialsProvider.java index 9c71ab458624e..db3d0bb13297c 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/TemporaryAWSCredentialsProvider.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/TemporaryAWSCredentialsProvider.java @@ -43,11 +43,14 @@ * * This credential provider must not fail in creation because that will * break a chain of credential providers. + * + * @deprecated This class will be replaced by one that implements AWS SDK V2's AwsCredentialProvider + * as part of upgrading S3A to SDK V2. See HADOOP-18073. */ @InterfaceAudience.Public @InterfaceStability.Stable -public class TemporaryAWSCredentialsProvider extends - AbstractSessionCredentialsProvider { +@Deprecated +public class TemporaryAWSCredentialsProvider extends AbstractSessionCredentialsProvider { public static final String NAME = "org.apache.hadoop.fs.s3a.TemporaryAWSCredentialsProvider"; diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/WriteOperationHelper.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/WriteOperationHelper.java index ce50f0b85ed73..14ffeed4a55bb 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/WriteOperationHelper.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/WriteOperationHelper.java @@ -144,6 +144,11 @@ public class WriteOperationHelper implements WriteOperations { */ private final RequestFactory requestFactory; + /** + * WriteOperationHelper callbacks. + */ + private final WriteOperationHelperCallbacks writeOperationHelperCallbacks; + /** * Constructor. * @param owner owner FS creating the helper @@ -151,13 +156,14 @@ public class WriteOperationHelper implements WriteOperations { * @param statisticsContext statistics context * @param auditSpanSource source of spans * @param auditSpan span to activate - * + * @param writeOperationHelperCallbacks callbacks used by writeOperationHelper */ protected WriteOperationHelper(S3AFileSystem owner, Configuration conf, S3AStatisticsContext statisticsContext, final AuditSpanSource auditSpanSource, - final AuditSpan auditSpan) { + final AuditSpan auditSpan, + final WriteOperationHelperCallbacks writeOperationHelperCallbacks) { this.owner = owner; this.invoker = new Invoker(new S3ARetryPolicy(conf), this::operationRetried); @@ -168,6 +174,7 @@ protected WriteOperationHelper(S3AFileSystem owner, this.auditSpanSource = auditSpanSource; this.auditSpan = checkNotNull(auditSpan); this.requestFactory = owner.getRequestFactory(); + this.writeOperationHelperCallbacks = writeOperationHelperCallbacks; } /** @@ -359,8 +366,7 @@ private CompleteMultipartUploadResult finalizeMultipartUpload( final CompleteMultipartUploadRequest request = getRequestFactory().newCompleteMultipartUploadRequest( destKey, uploadId, partETags); - return owner.getAmazonS3Client().completeMultipartUpload( - request); + return writeOperationHelperCallbacks.completeMultipartUpload(request); }); owner.finishedWrite(destKey, length, uploadResult.getETag(), uploadResult.getVersionId(), @@ -716,7 +722,7 @@ public SelectObjectContentResult select( try (DurationInfo ignored = new DurationInfo(LOG, "S3 Select operation")) { try { - return owner.getAmazonS3Client().selectObjectContent(request); + return writeOperationHelperCallbacks.selectObjectContent(request); } catch (AmazonS3Exception e) { LOG.error("Failure of S3 Select request against {}", source); @@ -758,4 +764,25 @@ public RequestFactory getRequestFactory() { return requestFactory; } + /*** + * Callbacks for writeOperationHelper. + */ + public interface WriteOperationHelperCallbacks { + + /** + * Initiates a select request. + * @param request selectObjectContent request + * @return selectObjectContentResult + */ + SelectObjectContentResult selectObjectContent(SelectObjectContentRequest request); + + /** + * Initiates a complete multi-part upload request. + * @param request Complete multi-part upload request + * @return completeMultipartUploadResult + */ + CompleteMultipartUploadResult completeMultipartUpload(CompleteMultipartUploadRequest request); + + } + } diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/AbstractAWSCredentialProvider.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/AbstractAWSCredentialProvider.java index 1f714b0555285..1815285738b0e 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/AbstractAWSCredentialProvider.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/AbstractAWSCredentialProvider.java @@ -28,7 +28,11 @@ /** * Base class for AWS credential providers which * take a URI and config in their constructor. + * + * @deprecated This class will be replaced by one that implements AWS SDK V2's AwsCredentialProvider + * as part of upgrading S3A to SDK V2. See HADOOP-18073. */ +@Deprecated public abstract class AbstractAWSCredentialProvider implements AWSCredentialsProvider { diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/AbstractSessionCredentialsProvider.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/AbstractSessionCredentialsProvider.java index 30939767e4379..c316b91116fd5 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/AbstractSessionCredentialsProvider.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/AbstractSessionCredentialsProvider.java @@ -35,8 +35,12 @@ /** * Base class for session credential support. + * + * @deprecated This class will be replaced by one that implements AWS SDK V2's AwsCredentialProvider + * as part of upgrading S3A to SDK V2. See HADOOP-18073. */ @InterfaceAudience.Private +@Deprecated public abstract class AbstractSessionCredentialsProvider extends AbstractAWSCredentialProvider { diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/AssumedRoleCredentialProvider.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/AssumedRoleCredentialProvider.java index 104e135ed79df..1e2ac16075aeb 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/AssumedRoleCredentialProvider.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/AssumedRoleCredentialProvider.java @@ -60,9 +60,13 @@ * unless overridden, creating a session name from the current user. * * Classname is used in configuration files; do not move. + * + * @deprecated This class will be replaced by one that implements AWS SDK V2's AwsCredentialProvider + * as part of upgrading S3A to SDK V2. See HADOOP-18073. */ @InterfaceAudience.Public @InterfaceStability.Evolving +@Deprecated public class AssumedRoleCredentialProvider implements AWSCredentialsProvider, Closeable { diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/IAMInstanceCredentialsProvider.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/IAMInstanceCredentialsProvider.java index 1bb30ed5c0dc9..ca9c518d30048 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/IAMInstanceCredentialsProvider.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/IAMInstanceCredentialsProvider.java @@ -40,9 +40,13 @@ * as a non-recoverable failure. *

    * It is implicitly public; marked evolving as we can change its semantics. + * + * @deprecated This class will be replaced by one that implements AWS SDK V2's AwsCredentialProvider + * as part of upgrading S3A to SDK V2. See HADOOP-18073. */ @InterfaceAudience.Public @InterfaceStability.Evolving +@Deprecated public class IAMInstanceCredentialsProvider implements AWSCredentialsProvider, Closeable { diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/MarshalledCredentialBinding.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/MarshalledCredentialBinding.java index f9e840e776a54..29e815560a8a9 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/MarshalledCredentialBinding.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/MarshalledCredentialBinding.java @@ -207,9 +207,11 @@ public static MarshalledCredentials requestSessionCredentials( stsEndpoint.isEmpty() ? null : stsEndpoint, stsRegion) .build(); - return fromSTSCredentials( - STSClientFactory.createClientConnection(tokenService, invoker) - .requestSessionCredentials(duration, TimeUnit.SECONDS)); + try (STSClientFactory.STSClient stsClient = STSClientFactory.createClientConnection( + tokenService, invoker)) { + return fromSTSCredentials(stsClient.requestSessionCredentials(duration, + TimeUnit.SECONDS)); + } } catch (SdkClientException e) { if (stsRegion.isEmpty()) { LOG.error("Region must be provided when requesting session credentials.", diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/MarshalledCredentialProvider.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/MarshalledCredentialProvider.java index 1d4956d1043ee..8bd04744cd8c0 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/MarshalledCredentialProvider.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/MarshalledCredentialProvider.java @@ -40,6 +40,7 @@ */ @InterfaceAudience.Private @InterfaceStability.Unstable +@SuppressWarnings("deprecation") public class MarshalledCredentialProvider extends AbstractSessionCredentialsProvider { diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/STSClientFactory.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/STSClientFactory.java index 8c74afcc08042..82d4fa588164d 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/STSClientFactory.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/STSClientFactory.java @@ -149,12 +149,10 @@ public static AWSSecurityTokenServiceClientBuilder builder( * @param tokenService STS instance * @param invoker invoker to use * @return an STS client bonded to that interface. - * @throws IOException on any failure */ public static STSClient createClientConnection( final AWSSecurityTokenService tokenService, - final Invoker invoker) - throws IOException { + final Invoker invoker) { return new STSClient(tokenService, invoker); } @@ -175,12 +173,9 @@ private STSClient(final AWSSecurityTokenService tokenService, @Override public void close() throws IOException { - try { - tokenService.shutdown(); - } catch (UnsupportedOperationException ignored) { - // ignore this, as it is what the STS client currently - // does. - } + // Since we are not using AbstractAWSSecurityTokenService, we + // don't need to worry about catching UnsupportedOperationException. + tokenService.shutdown(); } /** diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/SignerManager.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/SignerManager.java index cda769a789c30..e162428787cc4 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/SignerManager.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/SignerManager.java @@ -29,6 +29,7 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.s3a.auth.delegation.DelegationTokenProvider; +import org.apache.hadoop.fs.s3a.impl.V2Migration; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.util.ReflectionUtils; @@ -70,6 +71,8 @@ public void initCustomSigners() { return; } + V2Migration.v1CustomSignerUsed(); + for (String customSigner : customSigners) { String[] parts = customSigner.split(":"); if (!(parts.length == 1 || parts.length == 2 || parts.length == 3)) { diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/AbstractS3ACommitter.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/AbstractS3ACommitter.java index 78b687cc6f19c..d6044edde29dd 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/AbstractS3ACommitter.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/AbstractS3ACommitter.java @@ -40,6 +40,7 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.RemoteIterator; import org.apache.hadoop.fs.audit.AuditConstants; +import org.apache.hadoop.fs.statistics.IOStatisticsContext; import org.apache.hadoop.fs.store.audit.AuditSpan; import org.apache.hadoop.fs.store.audit.AuditSpanSource; import org.apache.hadoop.fs.s3a.S3AFileSystem; @@ -467,8 +468,7 @@ public void recoverTask(TaskAttemptContext taskContext) throws IOException { * * While the classic committers create a 0-byte file, the S3A committers * PUT up a the contents of a {@link SuccessData} file. - * - * @param context job context + * @param commitContext commit context * @param pending the pending commits * * @return the success data, even if the marker wasn't created @@ -476,7 +476,7 @@ public void recoverTask(TaskAttemptContext taskContext) throws IOException { * @throws IOException IO failure */ protected SuccessData maybeCreateSuccessMarkerFromCommits( - JobContext context, + final CommitContext commitContext, ActiveCommit pending) throws IOException { List filenames = new ArrayList<>(pending.size()); // The list of committed objects in pending is size limited in @@ -488,7 +488,13 @@ protected SuccessData maybeCreateSuccessMarkerFromCommits( // and the current statistics snapshot.aggregate(getIOStatistics()); - return maybeCreateSuccessMarker(context, filenames, snapshot); + // and include the context statistics if enabled + if (commitContext.isCollectIOStatistics()) { + snapshot.aggregate(commitContext.getIOStatisticsContext() + .getIOStatistics()); + } + + return maybeCreateSuccessMarker(commitContext.getJobContext(), filenames, snapshot); } /** @@ -729,6 +735,7 @@ private void loadAndCommit( final FileStatus status) throws IOException { final Path path = status.getPath(); + commitContext.switchToIOStatisticsContext(); try (DurationInfo ignored = new DurationInfo(LOG, "Loading and committing files in pendingset %s", path)) { @@ -775,6 +782,7 @@ private void loadAndRevert( final FileStatus status) throws IOException { final Path path = status.getPath(); + commitContext.switchToIOStatisticsContext(); try (DurationInfo ignored = new DurationInfo(LOG, false, "Committing %s", path)) { PendingSet pendingSet = PersistentCommitData.load( @@ -806,6 +814,7 @@ private void loadAndAbort( final boolean deleteRemoteFiles) throws IOException { final Path path = status.getPath(); + commitContext.switchToIOStatisticsContext(); try (DurationInfo ignored = new DurationInfo(LOG, false, "Aborting %s", path)) { PendingSet pendingSet = PersistentCommitData.load( @@ -832,6 +841,8 @@ private void loadAndAbort( /** * Start the final job commit/abort commit operations. + * If configured to collect statistics, + * The IO StatisticsContext is reset. * @param context job context * @return a commit context through which the operations can be invoked. * @throws IOException failure. @@ -840,14 +851,22 @@ protected CommitContext initiateJobOperation( final JobContext context) throws IOException { - return getCommitOperations().createCommitContext( + IOStatisticsContext ioStatisticsContext = + IOStatisticsContext.getCurrentIOStatisticsContext(); + CommitContext commitContext = getCommitOperations().createCommitContext( context, getOutputPath(), - getJobCommitThreadCount(context)); + getJobCommitThreadCount(context), + ioStatisticsContext); + commitContext.maybeResetIOStatisticsContext(); + return commitContext; } + /** * Start a ask commit/abort commit operations. * This may have a different thread count. + * If configured to collect statistics, + * The IO StatisticsContext is reset. * @param context job or task context * @return a commit context through which the operations can be invoked. * @throws IOException failure. @@ -856,10 +875,13 @@ protected CommitContext initiateTaskOperation( final JobContext context) throws IOException { - return getCommitOperations().createCommitContext( + CommitContext commitContext = getCommitOperations().createCommitContext( context, getOutputPath(), - getTaskCommitThreadCount(context)); + getTaskCommitThreadCount(context), + IOStatisticsContext.getCurrentIOStatisticsContext()); + commitContext.maybeResetIOStatisticsContext(); + return commitContext; } /** @@ -1014,7 +1036,7 @@ public void commitJob(JobContext context) throws IOException { stage = "completed"; jobCompleted(true); stage = "marker"; - successData = maybeCreateSuccessMarkerFromCommits(context, pending); + successData = maybeCreateSuccessMarkerFromCommits(commitContext, pending); stage = "cleanup"; cleanup(commitContext, false); } catch (IOException e) { diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/CommitConstants.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/CommitConstants.java index b85ce276504ba..6e2a5d8c9fbce 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/CommitConstants.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/CommitConstants.java @@ -354,4 +354,24 @@ private CommitConstants() { public static final String OPT_SUMMARY_REPORT_DIR = OPT_PREFIX + "summary.report.directory"; + /** + * Experimental feature to collect thread level IO statistics. + * When set the committers will reset the statistics in + * task setup and propagate to the job committer. + * The job comitter will include those and its own statistics. + * Do not use if the execution engine is collecting statistics, + * as the multiple reset() operations will result in incomplete + * statistics. + * Value: {@value}. + */ + public static final String S3A_COMMITTER_EXPERIMENTAL_COLLECT_IOSTATISTICS = + OPT_PREFIX + "experimental.collect.iostatistics"; + + /** + * Default value for {@link #S3A_COMMITTER_EXPERIMENTAL_COLLECT_IOSTATISTICS}. + * Value: {@value}. + */ + public static final boolean S3A_COMMITTER_EXPERIMENTAL_COLLECT_IOSTATISTICS_DEFAULT = + false; + } diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/impl/AuditContextUpdater.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/impl/AuditContextUpdater.java index 20024ba601d45..17f63e6dff7f0 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/impl/AuditContextUpdater.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/impl/AuditContextUpdater.java @@ -22,6 +22,7 @@ import org.apache.hadoop.fs.audit.CommonAuditContext; import org.apache.hadoop.fs.s3a.commit.CommitConstants; import org.apache.hadoop.mapreduce.JobContext; +import org.apache.hadoop.mapreduce.JobID; import org.apache.hadoop.mapreduce.TaskAttemptContext; import org.apache.hadoop.mapreduce.TaskAttemptID; @@ -49,12 +50,17 @@ public final class AuditContextUpdater { * @param jobContext job/task context. */ public AuditContextUpdater(final JobContext jobContext) { - this.jobId = jobContext.getJobID().toString(); + JobID contextJobID = jobContext.getJobID(); + this.jobId = contextJobID != null + ? contextJobID.toString() + : null; if (jobContext instanceof TaskAttemptContext) { // it's a task, extract info for auditing final TaskAttemptID tid = ((TaskAttemptContext) jobContext).getTaskAttemptID(); - this.taskAttemptId = tid.toString(); + this.taskAttemptId = tid != null + ? tid.toString() + : null; } else { this.taskAttemptId = null; } @@ -70,7 +76,11 @@ public AuditContextUpdater(String jobId) { */ public void updateCurrentAuditContext() { final CommonAuditContext auditCtx = currentAuditContext(); - auditCtx.put(AuditConstants.PARAM_JOB_ID, jobId); + if (jobId != null) { + auditCtx.put(AuditConstants.PARAM_JOB_ID, jobId); + } else { + currentAuditContext().remove(AuditConstants.PARAM_JOB_ID); + } if (taskAttemptId != null) { auditCtx.put(AuditConstants.PARAM_TASK_ATTEMPT_ID, taskAttemptId); } else { diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/impl/CommitContext.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/impl/CommitContext.java index 8bff165f2e8f5..c93d2d8f73913 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/impl/CommitContext.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/impl/CommitContext.java @@ -21,6 +21,7 @@ import java.io.Closeable; import java.io.FileNotFoundException; import java.io.IOException; +import java.util.Objects; import java.util.concurrent.ExecutorService; import java.util.concurrent.Future; import java.util.concurrent.LinkedBlockingQueue; @@ -36,8 +37,10 @@ import org.apache.hadoop.fs.s3a.commit.InternalCommitterConstants; import org.apache.hadoop.fs.s3a.commit.files.PendingSet; import org.apache.hadoop.fs.s3a.commit.files.SinglePendingCommit; +import org.apache.hadoop.fs.statistics.IOStatisticsContext; import org.apache.hadoop.io.IOUtils; import org.apache.hadoop.mapreduce.JobContext; +import org.apache.hadoop.mapreduce.JobID; import org.apache.hadoop.thirdparty.com.google.common.util.concurrent.ThreadFactoryBuilder; import org.apache.hadoop.util.JsonSerialization; import org.apache.hadoop.util.Preconditions; @@ -47,6 +50,8 @@ import static org.apache.hadoop.fs.s3a.Constants.THREAD_POOL_SHUTDOWN_DELAY_SECONDS; import static org.apache.hadoop.fs.s3a.commit.AbstractS3ACommitter.THREAD_PREFIX; +import static org.apache.hadoop.fs.s3a.commit.CommitConstants.S3A_COMMITTER_EXPERIMENTAL_COLLECT_IOSTATISTICS; +import static org.apache.hadoop.fs.s3a.commit.CommitConstants.S3A_COMMITTER_EXPERIMENTAL_COLLECT_IOSTATISTICS_DEFAULT; import static org.apache.hadoop.fs.s3a.commit.InternalCommitterConstants.THREAD_KEEP_ALIVE_TIME; /** @@ -123,24 +128,48 @@ public final class CommitContext implements Closeable { */ private final int committerThreads; + /** + * Should IOStatistics be collected by the committer? + */ + private final boolean collectIOStatistics; + + /** + * IOStatisticsContext to switch to in all threads + * taking part in the commit operation. + * This ensures that the IOStatistics collected in the + * worker threads will be aggregated into the total statistics + * of the thread calling the committer commit/abort methods. + */ + private final IOStatisticsContext ioStatisticsContext; + /** * Create. * @param commitOperations commit callbacks * @param jobContext job context * @param committerThreads number of commit threads + * @param ioStatisticsContext IOStatistics context of current thread */ public CommitContext( final CommitOperations commitOperations, final JobContext jobContext, - final int committerThreads) { + final int committerThreads, + final IOStatisticsContext ioStatisticsContext) { this.commitOperations = commitOperations; this.jobContext = jobContext; this.conf = jobContext.getConfiguration(); - this.jobId = jobContext.getJobID().toString(); + JobID contextJobID = jobContext.getJobID(); + // either the job ID or make one up as it will be + // used for the filename of any reports. + this.jobId = contextJobID != null + ? contextJobID.toString() + : ("job-without-id-at-" + System.currentTimeMillis()); + this.collectIOStatistics = conf.getBoolean( + S3A_COMMITTER_EXPERIMENTAL_COLLECT_IOSTATISTICS, + S3A_COMMITTER_EXPERIMENTAL_COLLECT_IOSTATISTICS_DEFAULT); + this.ioStatisticsContext = Objects.requireNonNull(ioStatisticsContext); this.auditContextUpdater = new AuditContextUpdater(jobContext); this.auditContextUpdater.updateCurrentAuditContext(); this.committerThreads = committerThreads; - buildSubmitters(); } @@ -152,15 +181,19 @@ public CommitContext( * @param conf job conf * @param jobId ID * @param committerThreads number of commit threads + * @param ioStatisticsContext IOStatistics context of current thread */ public CommitContext(final CommitOperations commitOperations, final Configuration conf, final String jobId, - final int committerThreads) { + final int committerThreads, + final IOStatisticsContext ioStatisticsContext) { this.commitOperations = commitOperations; this.jobContext = null; this.conf = conf; this.jobId = jobId; + this.collectIOStatistics = false; + this.ioStatisticsContext = Objects.requireNonNull(ioStatisticsContext); this.auditContextUpdater = new AuditContextUpdater(jobId); this.auditContextUpdater.updateCurrentAuditContext(); this.committerThreads = committerThreads; @@ -358,6 +391,44 @@ public String getJobId() { return jobId; } + /** + * Collecting thread level IO statistics? + * @return true if thread level IO stats should be collected. + */ + public boolean isCollectIOStatistics() { + return collectIOStatistics; + } + + /** + * IOStatistics context of the created thread. + * @return the IOStatistics. + */ + public IOStatisticsContext getIOStatisticsContext() { + return ioStatisticsContext; + } + + /** + * Switch to the context IOStatistics context, + * if needed. + */ + public void switchToIOStatisticsContext() { + IOStatisticsContext.setThreadIOStatisticsContext(ioStatisticsContext); + } + + /** + * Reset the IOStatistics context if statistics are being + * collected. + * Logs at info. + */ + public void maybeResetIOStatisticsContext() { + if (collectIOStatistics) { + + LOG.info("Resetting IO statistics context {}", + ioStatisticsContext.getID()); + ioStatisticsContext.reset(); + } + } + /** * Submitter for a given thread pool. */ diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/impl/CommitOperations.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/impl/CommitOperations.java index 0772e143f69d6..eb23f299a02e3 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/impl/CommitOperations.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/impl/CommitOperations.java @@ -62,6 +62,7 @@ import org.apache.hadoop.fs.statistics.DurationTracker; import org.apache.hadoop.fs.statistics.IOStatistics; import org.apache.hadoop.fs.statistics.IOStatisticsSource; +import org.apache.hadoop.fs.statistics.IOStatisticsContext; import org.apache.hadoop.mapreduce.JobContext; import org.apache.hadoop.util.DurationInfo; import org.apache.hadoop.util.Preconditions; @@ -639,15 +640,18 @@ public void jobCompleted(boolean success) { * @param context job context * @param path path for all work. * @param committerThreads thread pool size + * @param ioStatisticsContext IOStatistics context of current thread * @return the commit context to pass in. * @throws IOException failure. */ public CommitContext createCommitContext( JobContext context, Path path, - int committerThreads) throws IOException { + int committerThreads, + IOStatisticsContext ioStatisticsContext) throws IOException { return new CommitContext(this, context, - committerThreads); + committerThreads, + ioStatisticsContext); } /** @@ -668,7 +672,8 @@ public CommitContext createCommitContextForTesting( return new CommitContext(this, getStoreContext().getConfiguration(), id, - committerThreads); + committerThreads, + IOStatisticsContext.getCurrentIOStatisticsContext()); } /** diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/magic/MagicS3GuardCommitter.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/magic/MagicS3GuardCommitter.java index 007e9b3709623..9ded64eedc056 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/magic/MagicS3GuardCommitter.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/magic/MagicS3GuardCommitter.java @@ -219,6 +219,13 @@ private PendingSet innerCommitTask( } pendingSet.putExtraData(TASK_ATTEMPT_ID, taskId); pendingSet.setJobId(jobId); + // add in the IOStatistics of all the file loading + if (commitContext.isCollectIOStatistics()) { + pendingSet.getIOStatistics() + .aggregate( + commitContext.getIOStatisticsContext().getIOStatistics()); + } + Path jobAttemptPath = getJobAttemptPath(context); TaskAttemptID taskAttemptID = context.getTaskAttemptID(); Path taskOutcomePath = new Path(jobAttemptPath, diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/staging/StagingCommitter.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/staging/StagingCommitter.java index 36eae012dea7d..d764055c45bcd 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/staging/StagingCommitter.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/staging/StagingCommitter.java @@ -696,6 +696,13 @@ protected int commitTaskInternal(final TaskAttemptContext context, pendingCommits.add(commit); } + // maybe add in the IOStatistics the thread + if (commitContext.isCollectIOStatistics()) { + pendingCommits.getIOStatistics().aggregate( + commitContext.getIOStatisticsContext() + .getIOStatistics()); + } + // save the data // overwrite any existing file, so whichever task attempt // committed last wins. diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/InternalConstants.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/InternalConstants.java index 6e4946dfb53ac..2c34e7b9b6ec2 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/InternalConstants.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/InternalConstants.java @@ -51,6 +51,11 @@ public final class InternalConstants { */ public static final boolean DELETE_CONSIDERED_IDEMPOTENT = true; + /** + * size of a buffer to create when draining the stream. + */ + public static final int DRAIN_BUFFER_SIZE = 16384; + private InternalConstants() { } @@ -97,6 +102,7 @@ private InternalConstants() { static { Set keys = Stream.of( + Constants.ASYNC_DRAIN_THRESHOLD, Constants.INPUT_FADVISE, Constants.READAHEAD_RANGE) .collect(Collectors.toSet()); @@ -114,6 +120,12 @@ private InternalConstants() { public static final String THROTTLE_LOG_NAME = "org.apache.hadoop.fs.s3a.throttled"; + /** + * Name of the log for events related to the SDK V2 upgrade. + */ + public static final String SDK_V2_UPGRADE_LOG_NAME = + "org.apache.hadoop.fs.s3a.SDKV2Upgrade"; + /** Directory marker attribute: see HADOOP-16613. Value: {@value}. */ public static final String X_DIRECTORY = "application/x-directory"; diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/RequestFactoryImpl.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/RequestFactoryImpl.java index 1e6629f9c7343..ce11df0383929 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/RequestFactoryImpl.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/RequestFactoryImpl.java @@ -408,6 +408,9 @@ public PutObjectRequest newPutObjectRequest(String key, inputStream, metadata); setOptionalPutRequestParameters(putObjectRequest); putObjectRequest.setCannedAcl(cannedACL); + if (storageClass != null) { + putObjectRequest.setStorageClass(storageClass); + } return prepareRequest(putObjectRequest); } @@ -416,19 +419,22 @@ public PutObjectRequest newDirectoryMarkerRequest(String directory) { String key = directory.endsWith("/") ? directory : (directory + "/"); - // an input stream which is laways empty - final InputStream im = new InputStream() { + // an input stream which is always empty + final InputStream inputStream = new InputStream() { @Override public int read() throws IOException { return -1; } }; // preparation happens in here - final ObjectMetadata md = createObjectMetadata(0L, true); - md.setContentType(HeaderProcessing.CONTENT_TYPE_X_DIRECTORY); - PutObjectRequest putObjectRequest = - newPutObjectRequest(key, md, null, im); - return putObjectRequest; + final ObjectMetadata metadata = createObjectMetadata(0L, true); + metadata.setContentType(HeaderProcessing.CONTENT_TYPE_X_DIRECTORY); + + PutObjectRequest putObjectRequest = new PutObjectRequest(getBucket(), key, + inputStream, metadata); + setOptionalPutRequestParameters(putObjectRequest); + putObjectRequest.setCannedAcl(cannedACL); + return prepareRequest(putObjectRequest); } @Override diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/SDKStreamDrainer.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/SDKStreamDrainer.java new file mode 100644 index 0000000000000..b566f9ad42765 --- /dev/null +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/SDKStreamDrainer.java @@ -0,0 +1,325 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.s3a.impl; + +import java.io.Closeable; +import java.util.concurrent.atomic.AtomicBoolean; + +import javax.annotation.Nullable; + +import com.amazonaws.internal.SdkFilterInputStream; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import org.apache.hadoop.classification.VisibleForTesting; +import org.apache.hadoop.fs.s3a.statistics.S3AInputStreamStatistics; +import org.apache.hadoop.util.functional.CallableRaisingIOE; + +import static java.util.Objects.requireNonNull; +import static org.apache.hadoop.fs.s3a.impl.InternalConstants.DRAIN_BUFFER_SIZE; +import static org.apache.hadoop.fs.statistics.impl.IOStatisticsBinding.invokeTrackingDuration; +import static org.apache.hadoop.io.IOUtils.cleanupWithLogger; + +/** + * Drains/aborts s3 or other AWS SDK streams. + * It is callable so can be passed directly to a submitter + * for async invocation. + * A request object may be passed in; it will be implicitly + * cached until this object is GCd. + * This is because in some versions of the AWS SDK, the S3Object + * has a finalize() method which releases the http connection, + * even when the stream is still open. + * See HADOOP-17338 for details. + */ +public class SDKStreamDrainer implements CallableRaisingIOE { + + private static final Logger LOG = LoggerFactory.getLogger( + SDKStreamDrainer.class); + + /** + * URI for log messages. + */ + private final String uri; + + /** + * Request object; usually S3Object + * Never used, but needed to keep the http connection + * open long enough for draining to take place. + */ + @Nullable + private final Closeable requestObject; + + /** + * Stream from the {@link #requestObject} for draining and closing. + */ + private final SdkFilterInputStream sdkStream; + + /** + * Should the request be aborted? + */ + private final boolean shouldAbort; + + /** + * How many bytes remaining? + * This is decremented as the stream is + * drained; + * If the stream finished before the expected + * remaining value was read, this will show how many + * bytes were still expected. + */ + private int remaining; + + /** + * Statistics to update with the duration. + */ + private final S3AInputStreamStatistics streamStatistics; + + /** + * Reason? for log messages. + */ + private final String reason; + + /** + * Has the operation executed yet? + */ + private final AtomicBoolean executed = new AtomicBoolean(false); + + /** + * Any exception caught during execution. + */ + private Exception thrown; + + /** + * Was the stream aborted? + */ + private boolean aborted; + + /** + * how many bytes were drained? + */ + private int drained = 0; + + /** + * Prepare to drain the stream. + * @param uri URI for messages + * @param requestObject http request object; needed to avoid GC issues. + * @param sdkStream stream to close. + * @param shouldAbort force an abort; used if explicitly requested. + * @param streamStatistics stats to update + * @param reason reason for stream being closed; used in messages + * @param remaining remaining bytes + */ + public SDKStreamDrainer(final String uri, + @Nullable final Closeable requestObject, + final SdkFilterInputStream sdkStream, + final boolean shouldAbort, + final int remaining, + final S3AInputStreamStatistics streamStatistics, + final String reason) { + this.uri = uri; + this.requestObject = requestObject; + this.sdkStream = requireNonNull(sdkStream); + this.shouldAbort = shouldAbort; + this.remaining = remaining; + this.streamStatistics = requireNonNull(streamStatistics); + this.reason = reason; + } + + /** + * drain the stream. This method is intended to be + * used directly or asynchronously, and measures the + * duration of the operation in the stream statistics. + * @return was the stream aborted? + */ + @Override + public Boolean apply() { + try { + Boolean outcome = invokeTrackingDuration( + streamStatistics.initiateInnerStreamClose(shouldAbort), + this::drainOrAbortHttpStream); + aborted = outcome; + return outcome; + } catch (Exception e) { + thrown = e; + return aborted; + } + } + + /** + * Apply, raising any exception. + * For testing. + * @return the outcome. + * @throws Exception anything raised. + */ + @VisibleForTesting + boolean applyRaisingException() throws Exception { + Boolean outcome = apply(); + if (thrown != null) { + throw thrown; + } + return outcome; + } + + /** + * Drain or abort the inner stream. + * Exceptions are saved then swallowed. + * If a close() is attempted and fails, the operation escalates to + * an abort. + * @return true if the stream was aborted. + */ + private boolean drainOrAbortHttpStream() { + if (executed.getAndSet(true)) { + throw new IllegalStateException( + "duplicate invocation of drain operation"); + } + boolean executeAbort = shouldAbort; + LOG.debug("drain or abort reason {} remaining={} abort={}", + reason, remaining, executeAbort); + + if (!executeAbort) { + try { + // clean close. This will read to the end of the stream, + // so, while cleaner, can be pathological on a multi-GB object + + if (remaining > 0) { + // explicitly drain the stream + LOG.debug("draining {} bytes", remaining); + drained = 0; + int size = Math.min(remaining, DRAIN_BUFFER_SIZE); + byte[] buffer = new byte[size]; + // read the data; bail out early if + // the connection breaks. + // this may be a bit overaggressive on buffer underflow. + while (remaining > 0) { + final int count = sdkStream.read(buffer); + LOG.debug("read {} bytes", count); + if (count <= 0) { + // no more data is left + break; + } + drained += count; + remaining -= count; + } + LOG.debug("Drained stream of {} bytes", drained); + } + + if (remaining != 0) { + // fewer bytes than expected came back; not treating as a + // reason to escalate to an abort(). + // just log. + LOG.debug("drained fewer bytes than expected; {} remaining", + remaining); + } + + // now close it. + // if there is still data in the stream, the SDK + // will warn and escalate to an abort itself. + LOG.debug("Closing stream"); + sdkStream.close(); + + cleanupWithLogger(LOG, requestObject); + // this MUST come after the close, so that if the IO operations fail + // and an abort is triggered, the initial attempt's statistics + // aren't collected. + streamStatistics.streamClose(false, drained); + return false; + } catch (Exception e) { + // exception escalates to an abort + LOG.debug("When closing {} stream for {}, will abort the stream", + uri, reason, e); + thrown = e; + } + } + // Abort, rather than just close, the underlying stream. Otherwise, the + // remaining object payload is read from S3 while closing the stream. + LOG.debug("Aborting stream {}", uri); + try { + sdkStream.abort(); + } catch (Exception e) { + LOG.warn("When aborting {} stream after failing to close it for {}", + uri, reason, e); + thrown = e; + } finally { + cleanupWithLogger(LOG, requestObject); + } + + streamStatistics.streamClose(true, remaining); + LOG.debug("Stream {} aborted: {}; remaining={}", + uri, reason, remaining); + return true; + } + + public String getUri() { + return uri; + } + + public Object getRequestObject() { + return requestObject; + } + + public SdkFilterInputStream getSdkStream() { + return sdkStream; + } + + public boolean shouldAbort() { + return shouldAbort; + } + + public int getRemaining() { + return remaining; + } + + public S3AInputStreamStatistics getStreamStatistics() { + return streamStatistics; + } + + public String getReason() { + return reason; + } + + public boolean executed() { + return executed.get(); + } + + public Exception getThrown() { + return thrown; + } + + public int getDrained() { + return drained; + } + + public boolean aborted() { + return aborted; + } + + @Override + public String toString() { + return "SDKStreamDrainer{" + + "uri='" + uri + '\'' + + ", reason='" + reason + '\'' + + ", shouldAbort=" + shouldAbort + + ", remaining=" + remaining + + ", executed=" + executed.get() + + ", aborted=" + aborted + + ", inner=" + sdkStream + + ", thrown=" + thrown + + '}'; + } +} diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/V2Migration.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/V2Migration.java new file mode 100644 index 0000000000000..3aa8ad270eedd --- /dev/null +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/V2Migration.java @@ -0,0 +1,98 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.s3a.impl; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import org.apache.hadoop.fs.store.LogExactlyOnce; + +import static org.apache.hadoop.fs.s3a.impl.InternalConstants.SDK_V2_UPGRADE_LOG_NAME; + +/** + * This class provides utility methods required for migrating S3A to AWS Java SDK V2. + * For more information on the upgrade, see HADOOP-18073. + */ +public final class V2Migration { + + private V2Migration() { } + + public static final Logger SDK_V2_UPGRADE_LOG = LoggerFactory.getLogger(SDK_V2_UPGRADE_LOG_NAME); + + private static final LogExactlyOnce WARN_ON_DELEGATION_TOKENS = + new LogExactlyOnce(SDK_V2_UPGRADE_LOG); + + private static final LogExactlyOnce WARN_ON_GET_S3_CLIENT = + new LogExactlyOnce(SDK_V2_UPGRADE_LOG); + + private static final LogExactlyOnce WARN_OF_DIRECTLY_REFERENCED_CREDENTIAL_PROVIDER = + new LogExactlyOnce(SDK_V2_UPGRADE_LOG); + + private static final LogExactlyOnce WARN_OF_CUSTOM_SIGNER = + new LogExactlyOnce(SDK_V2_UPGRADE_LOG); + + private static final LogExactlyOnce WARN_ON_GET_OBJECT_METADATA = + new LogExactlyOnce(SDK_V2_UPGRADE_LOG); + + /** + * Warns on an AWS V1 credential provider being referenced directly. + * @param name name of the credential provider + */ + public static void v1ProviderReferenced(String name) { + WARN_OF_DIRECTLY_REFERENCED_CREDENTIAL_PROVIDER.warn( + "Directly referencing AWS SDK V1 credential provider {}. AWS SDK V1 credential " + + "providers will be removed once S3A is upgraded to SDK V2", name); + } + + /** + * Warns on the v1 s3 client being requested. + */ + public static void v1S3ClientRequested() { + WARN_ON_GET_S3_CLIENT.warn( + "getAmazonS3ClientForTesting() will be removed as part of upgrading S3A to AWS SDK V2"); + } + + /** + * Warns when v1 credential providers are used with delegation tokens. + */ + public static void v1DelegationTokenCredentialProvidersUsed() { + WARN_ON_DELEGATION_TOKENS.warn( + "The credential provider interface has changed in AWS SDK V2, custom credential " + + "providers used in delegation tokens binding classes will need to be updated once " + + "S3A is upgraded to SDK V2"); + } + + /** + * Warns on use of custom signers. + */ + public static void v1CustomSignerUsed() { + WARN_OF_CUSTOM_SIGNER.warn( + "The signer interface has changed in AWS SDK V2, custom signers will need to be updated " + + "once S3A is upgraded to SDK V2"); + } + + /** + * Warns on use of getObjectMetadata. + */ + public static void v1GetObjectMetadataCalled() { + WARN_ON_GET_OBJECT_METADATA.warn("getObjectMetadata() called. This operation and it's response " + + "will be changed as part of upgrading S3A to AWS SDK V2"); + } + +} diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/prefetch/S3ABlockManager.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/prefetch/S3ABlockManager.java new file mode 100644 index 0000000000000..adbb06bf14237 --- /dev/null +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/prefetch/S3ABlockManager.java @@ -0,0 +1,78 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hadoop.fs.s3a.prefetch; + +import java.io.IOException; +import java.nio.ByteBuffer; + +import org.apache.hadoop.fs.impl.prefetch.BlockData; +import org.apache.hadoop.fs.impl.prefetch.BlockManager; +import org.apache.hadoop.fs.impl.prefetch.Validate; + +/** + * Provides read access to S3 file one block at a time. + * + * A naive implementation of a {@code BlockManager} that provides no prefetching or caching. + * Useful baseline for comparing performance difference against {@code S3ACachingBlockManager}. + */ +public class S3ABlockManager extends BlockManager { + + /** + * Reader that reads from S3 file. + */ + private final S3ARemoteObjectReader reader; + + /** + * Constructs an instance of {@code S3ABlockManager}. + * + * @param reader a reader that reads from S3 file. + * @param blockData information about each block of the S3 file. + * + * @throws IllegalArgumentException if reader is null. + * @throws IllegalArgumentException if blockData is null. + */ + public S3ABlockManager(S3ARemoteObjectReader reader, BlockData blockData) { + super(blockData); + + Validate.checkNotNull(reader, "reader"); + + this.reader = reader; + } + + /** + * Reads into the given {@code buffer} {@code size} bytes from the underlying file + * starting at {@code startOffset}. + * + * @param buffer the buffer to read data in to. + * @param startOffset the offset at which reading starts. + * @param size the number bytes to read. + * @return number of bytes read. + */ + @Override + public int read(ByteBuffer buffer, long startOffset, int size) + throws IOException { + return reader.read(buffer, startOffset, size); + } + + @Override + public void close() { + reader.close(); + } +} diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/prefetch/S3ACachingBlockManager.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/prefetch/S3ACachingBlockManager.java new file mode 100644 index 0000000000000..f82786659da05 --- /dev/null +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/prefetch/S3ACachingBlockManager.java @@ -0,0 +1,96 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hadoop.fs.s3a.prefetch; + +import java.io.IOException; +import java.nio.ByteBuffer; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import org.apache.hadoop.fs.impl.prefetch.BlockData; +import org.apache.hadoop.fs.impl.prefetch.CachingBlockManager; +import org.apache.hadoop.fs.impl.prefetch.ExecutorServiceFuturePool; +import org.apache.hadoop.fs.impl.prefetch.Validate; +import org.apache.hadoop.fs.s3a.statistics.S3AInputStreamStatistics; + +/** + * Provides access to S3 file one block at a time. + */ +public class S3ACachingBlockManager extends CachingBlockManager { + + private static final Logger LOG = LoggerFactory.getLogger( + S3ACachingBlockManager.class); + + /** + * Reader that reads from S3 file. + */ + private final S3ARemoteObjectReader reader; + + /** + * Constructs an instance of a {@code S3ACachingBlockManager}. + * + * @param futurePool asynchronous tasks are performed in this pool. + * @param reader reader that reads from S3 file. + * @param blockData information about each block of the S3 file. + * @param bufferPoolSize size of the in-memory cache in terms of number of blocks. + * @param streamStatistics statistics for this stream. + * + * @throws IllegalArgumentException if reader is null. + */ + public S3ACachingBlockManager( + ExecutorServiceFuturePool futurePool, + S3ARemoteObjectReader reader, + BlockData blockData, + int bufferPoolSize, + S3AInputStreamStatistics streamStatistics) { + super(futurePool, blockData, bufferPoolSize, streamStatistics); + + Validate.checkNotNull(reader, "reader"); + + this.reader = reader; + } + + protected S3ARemoteObjectReader getReader() { + return this.reader; + } + + /** + * Reads into the given {@code buffer} {@code size} bytes from the underlying file + * starting at {@code startOffset}. + * + * @param buffer the buffer to read data in to. + * @param startOffset the offset at which reading starts. + * @param size the number bytes to read. + * @return number of bytes read. + */ + @Override + public int read(ByteBuffer buffer, long startOffset, int size) + throws IOException { + return this.reader.read(buffer, startOffset, size); + } + + @Override + public synchronized void close() { + this.reader.close(); + + super.close(); + } +} diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/prefetch/S3ACachingInputStream.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/prefetch/S3ACachingInputStream.java new file mode 100644 index 0000000000000..0afd071246415 --- /dev/null +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/prefetch/S3ACachingInputStream.java @@ -0,0 +1,214 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hadoop.fs.s3a.prefetch; + +import java.io.IOException; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import org.apache.hadoop.fs.impl.prefetch.BlockData; +import org.apache.hadoop.fs.impl.prefetch.BlockManager; +import org.apache.hadoop.fs.impl.prefetch.BufferData; +import org.apache.hadoop.fs.impl.prefetch.ExecutorServiceFuturePool; +import org.apache.hadoop.fs.s3a.S3AInputStream; +import org.apache.hadoop.fs.s3a.S3AReadOpContext; +import org.apache.hadoop.fs.s3a.S3ObjectAttributes; +import org.apache.hadoop.fs.s3a.statistics.S3AInputStreamStatistics; + +import static org.apache.hadoop.fs.statistics.StreamStatisticNames.STREAM_READ_BLOCK_ACQUIRE_AND_READ; +import static org.apache.hadoop.fs.statistics.impl.IOStatisticsBinding.invokeTrackingDuration; + +/** + * Provides an {@code InputStream} that allows reading from an S3 file. + * Prefetched blocks are cached to local disk if a seek away from the + * current block is issued. + */ +public class S3ACachingInputStream extends S3ARemoteInputStream { + + private static final Logger LOG = LoggerFactory.getLogger( + S3ACachingInputStream.class); + + /** + * Number of blocks queued for prefching. + */ + private final int numBlocksToPrefetch; + + private final BlockManager blockManager; + + /** + * Initializes a new instance of the {@code S3ACachingInputStream} class. + * + * @param context read-specific operation context. + * @param s3Attributes attributes of the S3 object being read. + * @param client callbacks used for interacting with the underlying S3 client. + * @param streamStatistics statistics for this stream. + * + * @throws IllegalArgumentException if context is null. + * @throws IllegalArgumentException if s3Attributes is null. + * @throws IllegalArgumentException if client is null. + */ + public S3ACachingInputStream( + S3AReadOpContext context, + S3ObjectAttributes s3Attributes, + S3AInputStream.InputStreamCallbacks client, + S3AInputStreamStatistics streamStatistics) { + super(context, s3Attributes, client, streamStatistics); + + this.numBlocksToPrefetch = this.getContext().getPrefetchBlockCount(); + int bufferPoolSize = this.numBlocksToPrefetch + 1; + this.blockManager = this.createBlockManager( + this.getContext().getFuturePool(), + this.getReader(), + this.getBlockData(), + bufferPoolSize); + int fileSize = (int) s3Attributes.getLen(); + LOG.debug("Created caching input stream for {} (size = {})", this.getName(), + fileSize); + } + + /** + * Moves the current read position so that the next read will occur at {@code pos}. + * + * @param pos the next read will take place at this position. + * + * @throws IllegalArgumentException if pos is outside of the range [0, file size]. + */ + @Override + public void seek(long pos) throws IOException { + throwIfClosed(); + throwIfInvalidSeek(pos); + + // The call to setAbsolute() returns true if the target position is valid and + // within the current block. Therefore, no additional work is needed when we get back true. + if (!getFilePosition().setAbsolute(pos)) { + LOG.info("seek({})", getOffsetStr(pos)); + // We could be here in two cases: + // -- the target position is invalid: + // We ignore this case here as the next read will return an error. + // -- it is valid but outside of the current block. + if (getFilePosition().isValid()) { + // There are two cases to consider: + // -- the seek was issued after this buffer was fully read. + // In this case, it is very unlikely that this buffer will be needed again; + // therefore we release the buffer without caching. + // -- if we are jumping out of the buffer before reading it completely then + // we will likely need this buffer again (as observed empirically for Parquet) + // therefore we issue an async request to cache this buffer. + if (!getFilePosition().bufferFullyRead()) { + blockManager.requestCaching(getFilePosition().data()); + } else { + blockManager.release(getFilePosition().data()); + } + getFilePosition().invalidate(); + blockManager.cancelPrefetches(); + } + setSeekTargetPos(pos); + } + } + + @Override + public void close() throws IOException { + // Close the BlockManager first, cancelling active prefetches, + // deleting cached files and freeing memory used by buffer pool. + blockManager.close(); + super.close(); + LOG.info("closed: {}", getName()); + } + + @Override + protected boolean ensureCurrentBuffer() throws IOException { + if (isClosed()) { + return false; + } + + if (getFilePosition().isValid() && getFilePosition() + .buffer() + .hasRemaining()) { + return true; + } + + long readPos; + int prefetchCount; + + if (getFilePosition().isValid()) { + // A sequential read results in a prefetch. + readPos = getFilePosition().absolute(); + prefetchCount = numBlocksToPrefetch; + } else { + // A seek invalidates the current position. + // We prefetch only 1 block immediately after a seek operation. + readPos = getSeekTargetPos(); + prefetchCount = 1; + } + + if (!getBlockData().isValidOffset(readPos)) { + return false; + } + + if (getFilePosition().isValid()) { + if (getFilePosition().bufferFullyRead()) { + blockManager.release(getFilePosition().data()); + } else { + blockManager.requestCaching(getFilePosition().data()); + } + } + + int toBlockNumber = getBlockData().getBlockNumber(readPos); + long startOffset = getBlockData().getStartOffset(toBlockNumber); + + for (int i = 1; i <= prefetchCount; i++) { + int b = toBlockNumber + i; + if (b < getBlockData().getNumBlocks()) { + blockManager.requestPrefetch(b); + } + } + + BufferData data = invokeTrackingDuration( + getS3AStreamStatistics() + .trackDuration(STREAM_READ_BLOCK_ACQUIRE_AND_READ), + () -> blockManager.get(toBlockNumber)); + + getFilePosition().setData(data, startOffset, readPos); + return true; + } + + @Override + public String toString() { + if (isClosed()) { + return "closed"; + } + + StringBuilder sb = new StringBuilder(); + sb.append(String.format("fpos = (%s)%n", getFilePosition())); + sb.append(blockManager.toString()); + return sb.toString(); + } + + protected BlockManager createBlockManager( + ExecutorServiceFuturePool futurePool, + S3ARemoteObjectReader reader, + BlockData blockData, + int bufferPoolSize) { + return new S3ACachingBlockManager(futurePool, reader, blockData, + bufferPoolSize, + getS3AStreamStatistics()); + } +} diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/prefetch/S3AInMemoryInputStream.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/prefetch/S3AInMemoryInputStream.java new file mode 100644 index 0000000000000..322459a958912 --- /dev/null +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/prefetch/S3AInMemoryInputStream.java @@ -0,0 +1,102 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hadoop.fs.s3a.prefetch; + +import java.io.IOException; +import java.nio.ByteBuffer; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import org.apache.hadoop.fs.impl.prefetch.BufferData; +import org.apache.hadoop.fs.s3a.S3AInputStream; +import org.apache.hadoop.fs.s3a.S3AReadOpContext; +import org.apache.hadoop.fs.s3a.S3ObjectAttributes; +import org.apache.hadoop.fs.s3a.statistics.S3AInputStreamStatistics; + +/** + * Provides an {@code InputStream} that allows reading from an S3 file. + * The entire file is read into memory before reads can begin. + * + * Use of this class is recommended only for small files that can fit + * entirely in memory. + */ +public class S3AInMemoryInputStream extends S3ARemoteInputStream { + + private static final Logger LOG = LoggerFactory.getLogger( + S3AInMemoryInputStream.class); + + private ByteBuffer buffer; + + /** + * Initializes a new instance of the {@code S3AInMemoryInputStream} class. + * + * @param context read-specific operation context. + * @param s3Attributes attributes of the S3 object being read. + * @param client callbacks used for interacting with the underlying S3 client. + * @param streamStatistics statistics for this stream. + * + * @throws IllegalArgumentException if context is null. + * @throws IllegalArgumentException if s3Attributes is null. + * @throws IllegalArgumentException if client is null. + */ + public S3AInMemoryInputStream( + S3AReadOpContext context, + S3ObjectAttributes s3Attributes, + S3AInputStream.InputStreamCallbacks client, + S3AInputStreamStatistics streamStatistics) { + super(context, s3Attributes, client, streamStatistics); + int fileSize = (int) s3Attributes.getLen(); + this.buffer = ByteBuffer.allocate(fileSize); + LOG.debug("Created in-memory input stream for {} (size = {})", + getName(), fileSize); + } + + /** + * Ensures that a non-empty valid buffer is available for immediate reading. + * It returns true when at least one such buffer is available for reading. + * It returns false on reaching the end of the stream. + * + * @return true if at least one such buffer is available for reading, false otherwise. + */ + @Override + protected boolean ensureCurrentBuffer() throws IOException { + if (isClosed()) { + return false; + } + + if (getBlockData().getFileSize() == 0) { + return false; + } + + if (!getFilePosition().isValid()) { + buffer.clear(); + int numBytesRead = + getReader().read(buffer, 0, buffer.capacity()); + if (numBytesRead <= 0) { + return false; + } + BufferData data = new BufferData(0, buffer); + getFilePosition().setData(data, 0, getSeekTargetPos()); + } + + return getFilePosition().buffer().hasRemaining(); + } +} diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/prefetch/S3APrefetchingInputStream.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/prefetch/S3APrefetchingInputStream.java new file mode 100644 index 0000000000000..76ef942ed655a --- /dev/null +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/prefetch/S3APrefetchingInputStream.java @@ -0,0 +1,260 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hadoop.fs.s3a.prefetch; + +import java.io.IOException; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.fs.CanSetReadahead; +import org.apache.hadoop.fs.FSExceptionMessages; +import org.apache.hadoop.fs.FSInputStream; +import org.apache.hadoop.fs.StreamCapabilities; +import org.apache.hadoop.fs.impl.prefetch.Validate; +import org.apache.hadoop.fs.s3a.S3AInputStream; +import org.apache.hadoop.fs.s3a.S3AReadOpContext; +import org.apache.hadoop.fs.s3a.S3ObjectAttributes; +import org.apache.hadoop.fs.s3a.statistics.S3AInputStreamStatistics; +import org.apache.hadoop.fs.statistics.IOStatistics; +import org.apache.hadoop.fs.statistics.IOStatisticsSource; + +/** + * Enhanced {@code InputStream} for reading from S3. + * + * This implementation provides improved read throughput by asynchronously prefetching + * blocks of configurable size from the underlying S3 file. + */ +public class S3APrefetchingInputStream + extends FSInputStream + implements CanSetReadahead, StreamCapabilities, IOStatisticsSource { + + private static final Logger LOG = LoggerFactory.getLogger( + S3APrefetchingInputStream.class); + + /** + * Underlying input stream used for reading S3 file. + */ + private S3ARemoteInputStream inputStream; + + /** + * Initializes a new instance of the {@code S3APrefetchingInputStream} class. + * + * @param context read-specific operation context. + * @param s3Attributes attributes of the S3 object being read. + * @param client callbacks used for interacting with the underlying S3 client. + * @param streamStatistics statistics for this stream. + * + * @throws IllegalArgumentException if context is null. + * @throws IllegalArgumentException if s3Attributes is null. + * @throws IllegalArgumentException if client is null. + */ + public S3APrefetchingInputStream( + S3AReadOpContext context, + S3ObjectAttributes s3Attributes, + S3AInputStream.InputStreamCallbacks client, + S3AInputStreamStatistics streamStatistics) { + + Validate.checkNotNull(context, "context"); + Validate.checkNotNull(s3Attributes, "s3Attributes"); + Validate.checkNotNullAndNotEmpty(s3Attributes.getBucket(), + "s3Attributes.getBucket()"); + Validate.checkNotNullAndNotEmpty(s3Attributes.getKey(), + "s3Attributes.getKey()"); + Validate.checkNotNegative(s3Attributes.getLen(), "s3Attributes.getLen()"); + Validate.checkNotNull(client, "client"); + Validate.checkNotNull(streamStatistics, "streamStatistics"); + + long fileSize = s3Attributes.getLen(); + if (fileSize <= context.getPrefetchBlockSize()) { + LOG.debug("Creating in memory input stream for {}", context.getPath()); + this.inputStream = new S3AInMemoryInputStream( + context, + s3Attributes, + client, + streamStatistics); + } else { + LOG.debug("Creating in caching input stream for {}", context.getPath()); + this.inputStream = new S3ACachingInputStream( + context, + s3Attributes, + client, + streamStatistics); + } + } + + /** + * Returns the number of bytes available for reading without blocking. + * + * @return the number of bytes available for reading without blocking. + * @throws IOException if there is an IO error during this operation. + */ + @Override + public synchronized int available() throws IOException { + throwIfClosed(); + return inputStream.available(); + } + + /** + * Gets the current position. + * + * @return the current position. + * @throws IOException if there is an IO error during this operation. + */ + @Override + public synchronized long getPos() throws IOException { + return isClosed() ? 0 : inputStream.getPos(); + } + + /** + * Reads and returns one byte from this stream. + * + * @return the next byte from this stream. + * @throws IOException if there is an IO error during this operation. + */ + @Override + public synchronized int read() throws IOException { + throwIfClosed(); + return inputStream.read(); + } + + /** + * Reads up to {@code len} bytes from this stream and copies them into + * the given {@code buffer} starting at the given {@code offset}. + * Returns the number of bytes actually copied in to the given buffer. + * + * @param buffer the buffer to copy data into. + * @param offset data is copied starting at this offset. + * @param len max number of bytes to copy. + * @return the number of bytes actually copied in to the given buffer. + * @throws IOException if there is an IO error during this operation. + */ + @Override + public synchronized int read(byte[] buffer, int offset, int len) + throws IOException { + throwIfClosed(); + return inputStream.read(buffer, offset, len); + } + + /** + * Closes this stream and releases all acquired resources. + * + * @throws IOException if there is an IO error during this operation. + */ + @Override + public synchronized void close() throws IOException { + if (inputStream != null) { + inputStream.close(); + inputStream = null; + super.close(); + } + } + + /** + * Updates internal data such that the next read will take place at the given {@code pos}. + * + * @param pos new read position. + * @throws IOException if there is an IO error during this operation. + */ + @Override + public synchronized void seek(long pos) throws IOException { + throwIfClosed(); + inputStream.seek(pos); + } + + /** + * Sets the number of bytes to read ahead each time. + * + * @param readahead the number of bytes to read ahead each time.. + */ + @Override + public synchronized void setReadahead(Long readahead) { + if (!isClosed()) { + inputStream.setReadahead(readahead); + } + } + + /** + * Indicates whether the given {@code capability} is supported by this stream. + * + * @param capability the capability to check. + * @return true if the given {@code capability} is supported by this stream, false otherwise. + */ + @Override + public boolean hasCapability(String capability) { + if (!isClosed()) { + return inputStream.hasCapability(capability); + } + + return false; + } + + /** + * Access the input stream statistics. + * This is for internal testing and may be removed without warning. + * @return the statistics for this input stream + */ + @InterfaceAudience.Private + @InterfaceStability.Unstable + public S3AInputStreamStatistics getS3AStreamStatistics() { + if (isClosed()) { + return null; + } + return inputStream.getS3AStreamStatistics(); + } + + /** + * Gets the internal IO statistics. + * + * @return the internal IO statistics. + */ + @Override + public IOStatistics getIOStatistics() { + if (isClosed()) { + return null; + } + return inputStream.getIOStatistics(); + } + + protected boolean isClosed() { + return inputStream == null; + } + + protected void throwIfClosed() throws IOException { + if (isClosed()) { + throw new IOException(FSExceptionMessages.STREAM_IS_CLOSED); + } + } + + // Unsupported functions. + + @Override + public boolean seekToNewSource(long targetPos) throws IOException { + throwIfClosed(); + return false; + } + + @Override + public boolean markSupported() { + return false; + } +} diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/prefetch/S3ARemoteInputStream.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/prefetch/S3ARemoteInputStream.java new file mode 100644 index 0000000000000..0f46a8ed5e534 --- /dev/null +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/prefetch/S3ARemoteInputStream.java @@ -0,0 +1,475 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hadoop.fs.s3a.prefetch; + +import java.io.EOFException; +import java.io.IOException; +import java.io.InputStream; +import java.nio.ByteBuffer; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.fs.CanSetReadahead; +import org.apache.hadoop.fs.FSExceptionMessages; +import org.apache.hadoop.fs.StreamCapabilities; +import org.apache.hadoop.fs.impl.prefetch.BlockData; +import org.apache.hadoop.fs.impl.prefetch.FilePosition; +import org.apache.hadoop.fs.impl.prefetch.Validate; +import org.apache.hadoop.fs.s3a.S3AInputPolicy; +import org.apache.hadoop.fs.s3a.S3AInputStream; +import org.apache.hadoop.fs.s3a.S3AReadOpContext; +import org.apache.hadoop.fs.s3a.S3ObjectAttributes; +import org.apache.hadoop.fs.s3a.impl.ChangeTracker; +import org.apache.hadoop.fs.s3a.statistics.S3AInputStreamStatistics; +import org.apache.hadoop.fs.statistics.IOStatistics; +import org.apache.hadoop.fs.statistics.IOStatisticsSource; + +import static java.util.Objects.requireNonNull; + +/** + * Provides an {@link InputStream} that allows reading from an S3 file. + */ +public abstract class S3ARemoteInputStream + extends InputStream + implements CanSetReadahead, StreamCapabilities, IOStatisticsSource { + + private static final Logger LOG = LoggerFactory.getLogger( + S3ARemoteInputStream.class); + + /** + * The S3 file read by this instance. + */ + private S3ARemoteObject remoteObject; + + /** + * Reading of S3 file takes place through this reader. + */ + private S3ARemoteObjectReader reader; + + /** + * Name of this stream. Used only for logging. + */ + private final String name; + + /** + * Indicates whether the stream has been closed. + */ + private volatile boolean closed; + + /** + * Current position within the file. + */ + private FilePosition fpos; + + /** The target of the most recent seek operation. */ + private long seekTargetPos; + + /** Information about each block of the mapped S3 file. */ + private BlockData blockData; + + /** Read-specific operation context. */ + private S3AReadOpContext context; + + /** Attributes of the S3 object being read. */ + private S3ObjectAttributes s3Attributes; + + /** Callbacks used for interacting with the underlying S3 client. */ + private S3AInputStream.InputStreamCallbacks client; + + /** Used for reporting input stream access statistics. */ + private final S3AInputStreamStatistics streamStatistics; + + private S3AInputPolicy inputPolicy; + + private final ChangeTracker changeTracker; + + private final IOStatistics ioStatistics; + + /** + * Initializes a new instance of the {@code S3ARemoteInputStream} class. + * + * @param context read-specific operation context. + * @param s3Attributes attributes of the S3 object being read. + * @param client callbacks used for interacting with the underlying S3 client. + * @param streamStatistics statistics for this stream. + * + * @throws IllegalArgumentException if context is null. + * @throws IllegalArgumentException if s3Attributes is null. + * @throws IllegalArgumentException if client is null. + */ + public S3ARemoteInputStream( + S3AReadOpContext context, + S3ObjectAttributes s3Attributes, + S3AInputStream.InputStreamCallbacks client, + S3AInputStreamStatistics streamStatistics) { + + this.context = requireNonNull(context); + this.s3Attributes = requireNonNull(s3Attributes); + this.client = requireNonNull(client); + this.streamStatistics = requireNonNull(streamStatistics); + this.ioStatistics = streamStatistics.getIOStatistics(); + this.name = S3ARemoteObject.getPath(s3Attributes); + this.changeTracker = new ChangeTracker( + this.name, + context.getChangeDetectionPolicy(), + this.streamStatistics.getChangeTrackerStatistics(), + s3Attributes); + + setInputPolicy(context.getInputPolicy()); + setReadahead(context.getReadahead()); + + long fileSize = s3Attributes.getLen(); + int bufferSize = context.getPrefetchBlockSize(); + + this.blockData = new BlockData(fileSize, bufferSize); + this.fpos = new FilePosition(fileSize, bufferSize); + this.remoteObject = getS3File(); + this.reader = new S3ARemoteObjectReader(remoteObject); + + this.seekTargetPos = 0; + } + + /** + * Gets the internal IO statistics. + * + * @return the internal IO statistics. + */ + @Override + public IOStatistics getIOStatistics() { + return ioStatistics; + } + + /** + * Access the input stream statistics. + * This is for internal testing and may be removed without warning. + * @return the statistics for this input stream + */ + @InterfaceAudience.Private + @InterfaceStability.Unstable + public S3AInputStreamStatistics getS3AStreamStatistics() { + return streamStatistics; + } + + /** + * Sets the number of bytes to read ahead each time. + * + * @param readahead the number of bytes to read ahead each time.. + */ + @Override + public synchronized void setReadahead(Long readahead) { + // We support read head by prefetching therefore we ignore the supplied value. + if (readahead != null) { + Validate.checkNotNegative(readahead, "readahead"); + } + } + + /** + * Indicates whether the given {@code capability} is supported by this stream. + * + * @param capability the capability to check. + * @return true if the given {@code capability} is supported by this stream, false otherwise. + */ + @Override + public boolean hasCapability(String capability) { + return capability.equalsIgnoreCase(StreamCapabilities.IOSTATISTICS) + || capability.equalsIgnoreCase(StreamCapabilities.READAHEAD); + } + + /** + * Set/update the input policy of the stream. + * This updates the stream statistics. + * @param inputPolicy new input policy. + */ + private void setInputPolicy(S3AInputPolicy inputPolicy) { + this.inputPolicy = inputPolicy; + streamStatistics.inputPolicySet(inputPolicy.ordinal()); + } + + /** + * Returns the number of bytes that can read from this stream without blocking. + */ + @Override + public int available() throws IOException { + throwIfClosed(); + + if (!ensureCurrentBuffer()) { + return 0; + } + + return fpos.buffer().remaining(); + } + + /** + * Gets the current position. + * + * @return the current position. + * @throws IOException if there is an IO error during this operation. + */ + public long getPos() throws IOException { + throwIfClosed(); + + if (fpos.isValid()) { + return fpos.absolute(); + } else { + return seekTargetPos; + } + } + + /** + * Moves the current read position so that the next read will occur at {@code pos}. + * + * @param pos the absolute position to seek to. + * @throws IOException if there an error during this operation. + * + * @throws IllegalArgumentException if pos is outside of the range [0, file size]. + */ + public void seek(long pos) throws IOException { + throwIfClosed(); + throwIfInvalidSeek(pos); + + if (!fpos.setAbsolute(pos)) { + fpos.invalidate(); + seekTargetPos = pos; + } + } + + /** + * Ensures that a non-empty valid buffer is available for immediate reading. + * It returns true when at least one such buffer is available for reading. + * It returns false on reaching the end of the stream. + * + * @return true if at least one such buffer is available for reading, false otherwise. + * @throws IOException if there is an IO error during this operation. + */ + protected abstract boolean ensureCurrentBuffer() throws IOException; + + @Override + public int read() throws IOException { + throwIfClosed(); + + if (remoteObject.size() == 0 + || seekTargetPos >= remoteObject.size()) { + return -1; + } + + if (!ensureCurrentBuffer()) { + return -1; + } + + incrementBytesRead(1); + + return fpos.buffer().get() & 0xff; + } + + /** + * Reads bytes from this stream and copies them into + * the given {@code buffer} starting at the beginning (offset 0). + * Returns the number of bytes actually copied in to the given buffer. + * + * @param buffer the buffer to copy data into. + * @return the number of bytes actually copied in to the given buffer. + * @throws IOException if there is an IO error during this operation. + */ + @Override + public int read(byte[] buffer) throws IOException { + return read(buffer, 0, buffer.length); + } + + /** + * Reads up to {@code len} bytes from this stream and copies them into + * the given {@code buffer} starting at the given {@code offset}. + * Returns the number of bytes actually copied in to the given buffer. + * + * @param buffer the buffer to copy data into. + * @param offset data is copied starting at this offset. + * @param len max number of bytes to copy. + * @return the number of bytes actually copied in to the given buffer. + * @throws IOException if there is an IO error during this operation. + */ + @Override + public int read(byte[] buffer, int offset, int len) throws IOException { + throwIfClosed(); + + if (len == 0) { + return 0; + } + + if (remoteObject.size() == 0 + || seekTargetPos >= remoteObject.size()) { + return -1; + } + + if (!ensureCurrentBuffer()) { + return -1; + } + + int numBytesRead = 0; + int numBytesRemaining = len; + + while (numBytesRemaining > 0) { + if (!ensureCurrentBuffer()) { + break; + } + + ByteBuffer buf = fpos.buffer(); + int bytesToRead = Math.min(numBytesRemaining, buf.remaining()); + buf.get(buffer, offset, bytesToRead); + incrementBytesRead(bytesToRead); + offset += bytesToRead; + numBytesRemaining -= bytesToRead; + numBytesRead += bytesToRead; + } + + return numBytesRead; + } + + protected S3ARemoteObject getFile() { + return remoteObject; + } + + protected S3ARemoteObjectReader getReader() { + return reader; + } + + protected S3ObjectAttributes getS3ObjectAttributes() { + return s3Attributes; + } + + protected FilePosition getFilePosition() { + return fpos; + } + + protected String getName() { + return name; + } + + protected boolean isClosed() { + return closed; + } + + protected long getSeekTargetPos() { + return seekTargetPos; + } + + protected void setSeekTargetPos(long pos) { + seekTargetPos = pos; + } + + protected BlockData getBlockData() { + return blockData; + } + + protected S3AReadOpContext getContext() { + return context; + } + + private void incrementBytesRead(int bytesRead) { + if (bytesRead > 0) { + streamStatistics.bytesRead(bytesRead); + if (getContext().getStats() != null) { + getContext().getStats().incrementBytesRead(bytesRead); + } + fpos.incrementBytesRead(bytesRead); + } + } + + protected S3ARemoteObject getS3File() { + return new S3ARemoteObject( + context, + s3Attributes, + client, + streamStatistics, + changeTracker + ); + } + + protected String getOffsetStr(long offset) { + int blockNumber = -1; + + if (blockData.isValidOffset(offset)) { + blockNumber = blockData.getBlockNumber(offset); + } + + return String.format("%d:%d", blockNumber, offset); + } + + /** + * Closes this stream and releases all acquired resources. + * + * @throws IOException if there is an IO error during this operation. + */ + @Override + public void close() throws IOException { + if (closed) { + return; + } + closed = true; + + blockData = null; + reader.close(); + reader = null; + remoteObject = null; + fpos.invalidate(); + try { + client.close(); + } finally { + streamStatistics.close(); + } + client = null; + } + + @Override + public boolean markSupported() { + return false; + } + + protected void throwIfClosed() throws IOException { + if (closed) { + throw new IOException( + name + ": " + FSExceptionMessages.STREAM_IS_CLOSED); + } + } + + protected void throwIfInvalidSeek(long pos) throws EOFException { + if (pos < 0) { + throw new EOFException(FSExceptionMessages.NEGATIVE_SEEK + " " + pos); + } + } + + // Unsupported functions. + + @Override + public void mark(int readlimit) { + throw new UnsupportedOperationException("mark not supported"); + } + + @Override + public void reset() { + throw new UnsupportedOperationException("reset not supported"); + } + + @Override + public long skip(long n) { + throw new UnsupportedOperationException("skip not supported"); + } +} diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/prefetch/S3ARemoteObject.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/prefetch/S3ARemoteObject.java new file mode 100644 index 0000000000000..3ab0022bb082e --- /dev/null +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/prefetch/S3ARemoteObject.java @@ -0,0 +1,250 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hadoop.fs.s3a.prefetch; + + +import java.io.IOException; +import java.io.InputStream; +import java.util.IdentityHashMap; +import java.util.Map; + +import com.amazonaws.services.s3.model.GetObjectRequest; +import com.amazonaws.services.s3.model.S3Object; +import com.amazonaws.services.s3.model.S3ObjectInputStream; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import org.apache.hadoop.fs.impl.prefetch.Validate; +import org.apache.hadoop.fs.s3a.Invoker; +import org.apache.hadoop.fs.s3a.S3AInputStream; +import org.apache.hadoop.fs.s3a.S3AReadOpContext; +import org.apache.hadoop.fs.s3a.S3ObjectAttributes; +import org.apache.hadoop.fs.s3a.impl.ChangeTracker; +import org.apache.hadoop.fs.s3a.impl.SDKStreamDrainer; +import org.apache.hadoop.fs.s3a.statistics.S3AInputStreamStatistics; +import org.apache.hadoop.fs.statistics.DurationTracker; + +/** + * Encapsulates low level interactions with S3 object on AWS. + */ +public class S3ARemoteObject { + + private static final Logger LOG = + LoggerFactory.getLogger(S3ARemoteObject.class); + + /** + * Read-specific operation context. + */ + private final S3AReadOpContext context; + + /** + * S3 object attributes. + */ + private final S3ObjectAttributes s3Attributes; + + /** + * Callbacks used for interacting with the underlying S3 client. + */ + private final S3AInputStream.InputStreamCallbacks client; + + /** + * Used for reporting input stream access statistics. + */ + private final S3AInputStreamStatistics streamStatistics; + + /** + * Enforces change tracking related policies. + */ + private final ChangeTracker changeTracker; + + /** + * Maps a stream returned by openForRead() to the associated S3 object. + * That allows us to close the object when closing the stream. + */ + private final Map s3Objects; + + /** + * uri of the object being read. + */ + private final String uri; + + /** + * size of a buffer to create when draining the stream. + */ + private static final int DRAIN_BUFFER_SIZE = 16384; + + /** + * Initializes a new instance of the {@code S3ARemoteObject} class. + * + * @param context read-specific operation context. + * @param s3Attributes attributes of the S3 object being read. + * @param client callbacks used for interacting with the underlying S3 client. + * @param streamStatistics statistics about stream access. + * @param changeTracker helps enforce change tracking policy. + * + * @throws IllegalArgumentException if context is null. + * @throws IllegalArgumentException if s3Attributes is null. + * @throws IllegalArgumentException if client is null. + * @throws IllegalArgumentException if streamStatistics is null. + * @throws IllegalArgumentException if changeTracker is null. + */ + public S3ARemoteObject( + S3AReadOpContext context, + S3ObjectAttributes s3Attributes, + S3AInputStream.InputStreamCallbacks client, + S3AInputStreamStatistics streamStatistics, + ChangeTracker changeTracker) { + + Validate.checkNotNull(context, "context"); + Validate.checkNotNull(s3Attributes, "s3Attributes"); + Validate.checkNotNull(client, "client"); + Validate.checkNotNull(streamStatistics, "streamStatistics"); + Validate.checkNotNull(changeTracker, "changeTracker"); + + this.context = context; + this.s3Attributes = s3Attributes; + this.client = client; + this.streamStatistics = streamStatistics; + this.changeTracker = changeTracker; + this.s3Objects = new IdentityHashMap<>(); + this.uri = this.getPath(); + } + + /** + * Gets an instance of {@code Invoker} for interacting with S3 API. + * + * @return an instance of {@code Invoker} for interacting with S3 API. + */ + public Invoker getReadInvoker() { + return context.getReadInvoker(); + } + + /** + * Gets an instance of {@code S3AInputStreamStatistics} used for reporting access metrics. + * + * @return an instance of {@code S3AInputStreamStatistics} used for reporting access metrics. + */ + public S3AInputStreamStatistics getStatistics() { + return streamStatistics; + } + + /** + * Gets the path of this file. + * + * @return the path of this file. + */ + public String getPath() { + return getPath(s3Attributes); + } + + /** + * Gets the path corresponding to the given s3Attributes. + * + * @param s3Attributes attributes of an S3 object. + * @return the path corresponding to the given s3Attributes. + */ + public static String getPath(S3ObjectAttributes s3Attributes) { + return String.format("s3a://%s/%s", s3Attributes.getBucket(), + s3Attributes.getKey()); + } + + /** + * Gets the size of this file. + * Its value is cached once obtained from AWS. + * + * @return the size of this file. + */ + public long size() { + return s3Attributes.getLen(); + } + + /** + * Opens a section of the file for reading. + * + * @param offset Start offset (0 based) of the section to read. + * @param size Size of the section to read. + * @return an {@code InputStream} corresponding to the given section of this file. + * + * @throws IOException if there is an error opening this file section for reading. + * @throws IllegalArgumentException if offset is negative. + * @throws IllegalArgumentException if offset is greater than or equal to file size. + * @throws IllegalArgumentException if size is greater than the remaining bytes. + */ + public InputStream openForRead(long offset, int size) throws IOException { + Validate.checkNotNegative(offset, "offset"); + Validate.checkLessOrEqual(offset, "offset", size(), "size()"); + Validate.checkLessOrEqual(size, "size", size() - offset, "size() - offset"); + + streamStatistics.streamOpened(); + final GetObjectRequest request = + client.newGetRequest(s3Attributes.getKey()) + .withRange(offset, offset + size - 1); + changeTracker.maybeApplyConstraint(request); + + String operation = String.format( + "%s %s at %d", S3AInputStream.OPERATION_OPEN, uri, offset); + DurationTracker tracker = streamStatistics.initiateGetRequest(); + S3Object object = null; + + try { + object = Invoker.once(operation, uri, () -> client.getObject(request)); + } catch (IOException e) { + tracker.failed(); + throw e; + } finally { + tracker.close(); + } + + changeTracker.processResponse(object, operation, offset); + InputStream stream = object.getObjectContent(); + synchronized (s3Objects) { + s3Objects.put(stream, object); + } + + return stream; + } + + void close(InputStream inputStream, int numRemainingBytes) { + S3Object obj; + synchronized (s3Objects) { + obj = s3Objects.remove(inputStream); + if (obj == null) { + throw new IllegalArgumentException("inputStream not found"); + } + } + SDKStreamDrainer drainer = new SDKStreamDrainer( + uri, + obj, + (S3ObjectInputStream)inputStream, + false, + numRemainingBytes, + streamStatistics, + "close() operation"); + if (numRemainingBytes <= context.getAsyncDrainThreshold()) { + // don't bother with async io. + drainer.apply(); + } else { + LOG.debug("initiating asynchronous drain of {} bytes", numRemainingBytes); + // schedule an async drain/abort + client.submit(drainer); + } + } + +} diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/prefetch/S3ARemoteObjectReader.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/prefetch/S3ARemoteObjectReader.java new file mode 100644 index 0000000000000..89ea77d6d0ebb --- /dev/null +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/prefetch/S3ARemoteObjectReader.java @@ -0,0 +1,175 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hadoop.fs.s3a.prefetch; + +import java.io.Closeable; +import java.io.EOFException; +import java.io.IOException; +import java.io.InputStream; +import java.net.SocketTimeoutException; +import java.nio.ByteBuffer; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import org.apache.hadoop.fs.impl.prefetch.Validate; +import org.apache.hadoop.fs.s3a.Invoker; +import org.apache.hadoop.fs.s3a.statistics.S3AInputStreamStatistics; + +import static org.apache.hadoop.fs.statistics.StreamStatisticNames.STREAM_READ_REMOTE_BLOCK_READ; +import static org.apache.hadoop.fs.statistics.impl.IOStatisticsBinding.trackDurationOfOperation; + +/** + * Provides functionality to read S3 file one block at a time. + */ +public class S3ARemoteObjectReader implements Closeable { + + private static final Logger LOG = LoggerFactory.getLogger( + S3ARemoteObjectReader.class); + + /** We read from the underlying input stream in blocks of this size. */ + private static final int READ_BUFFER_SIZE = 64 * 1024; + + /** The S3 file to read. */ + private final S3ARemoteObject remoteObject; + + /** Set to true by close(). */ + private volatile boolean closed; + + private final S3AInputStreamStatistics streamStatistics; + + /** + * Constructs an instance of {@link S3ARemoteObjectReader}. + * + * @param remoteObject The S3 file to read. + * + * @throws IllegalArgumentException if remoteObject is null. + */ + public S3ARemoteObjectReader(S3ARemoteObject remoteObject) { + Validate.checkNotNull(remoteObject, "remoteObject"); + + this.remoteObject = remoteObject; + this.streamStatistics = this.remoteObject.getStatistics(); + } + + /** + * Stars reading at {@code offset} and reads upto {@code size} bytes into {@code buffer}. + * + * @param buffer the buffer into which data is returned + * @param offset the absolute offset into the underlying file where reading starts. + * @param size the number of bytes to be read. + * + * @return number of bytes actually read. + * @throws IOException if there is an error reading from the file. + * + * @throws IllegalArgumentException if buffer is null. + * @throws IllegalArgumentException if offset is outside of the range [0, file size]. + * @throws IllegalArgumentException if size is zero or negative. + */ + public int read(ByteBuffer buffer, long offset, int size) throws IOException { + Validate.checkNotNull(buffer, "buffer"); + Validate.checkWithinRange(offset, "offset", 0, this.remoteObject.size()); + Validate.checkPositiveInteger(size, "size"); + + if (this.closed) { + return -1; + } + + int reqSize = (int) Math.min(size, this.remoteObject.size() - offset); + return readOneBlockWithRetries(buffer, offset, reqSize); + } + + @Override + public void close() { + this.closed = true; + } + + private int readOneBlockWithRetries(ByteBuffer buffer, long offset, int size) + throws IOException { + + this.streamStatistics.readOperationStarted(offset, size); + Invoker invoker = this.remoteObject.getReadInvoker(); + + int invokerResponse = + invoker.retry("read", this.remoteObject.getPath(), true, + trackDurationOfOperation(streamStatistics, + STREAM_READ_REMOTE_BLOCK_READ, () -> { + try { + this.readOneBlock(buffer, offset, size); + } catch (EOFException e) { + // the base implementation swallows EOFs. + return -1; + } catch (SocketTimeoutException e) { + throw e; + } catch (IOException e) { + this.remoteObject.getStatistics().readException(); + throw e; + } + return 0; + })); + + int numBytesRead = buffer.position(); + buffer.limit(numBytesRead); + this.remoteObject.getStatistics() + .readOperationCompleted(size, numBytesRead); + + if (invokerResponse < 0) { + return invokerResponse; + } else { + return numBytesRead; + } + } + + private void readOneBlock(ByteBuffer buffer, long offset, int size) + throws IOException { + int readSize = Math.min(size, buffer.remaining()); + if (readSize == 0) { + return; + } + + InputStream inputStream = remoteObject.openForRead(offset, readSize); + int numRemainingBytes = readSize; + byte[] bytes = new byte[READ_BUFFER_SIZE]; + + int numBytesToRead; + int numBytes; + + try { + do { + numBytesToRead = Math.min(READ_BUFFER_SIZE, numRemainingBytes); + numBytes = inputStream.read(bytes, 0, numBytesToRead); + if (numBytes < 0) { + String message = String.format( + "Unexpected end of stream: buffer[%d], readSize = %d, numRemainingBytes = %d", + buffer.capacity(), readSize, numRemainingBytes); + throw new EOFException(message); + } + + if (numBytes > 0) { + buffer.put(bytes, 0, numBytes); + numRemainingBytes -= numBytes; + } + } + while (!this.closed && (numRemainingBytes > 0)); + } finally { + remoteObject.close(inputStream, numRemainingBytes); + } + } +} diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/prefetch/package-info.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/prefetch/package-info.java new file mode 100644 index 0000000000000..f953d35dcca87 --- /dev/null +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/prefetch/package-info.java @@ -0,0 +1,29 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * High performance s3 input stream which reads in + * blocks and can cache blocks in the local filesystem. + */ + +@InterfaceAudience.Private +@InterfaceStability.Unstable +package org.apache.hadoop.fs.s3a.prefetch; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; \ No newline at end of file diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/statistics/S3AInputStreamStatistics.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/statistics/S3AInputStreamStatistics.java index 539af2bde3669..69f2c279ddaa8 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/statistics/S3AInputStreamStatistics.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/statistics/S3AInputStreamStatistics.java @@ -18,6 +18,7 @@ package org.apache.hadoop.fs.s3a.statistics; +import org.apache.hadoop.fs.impl.prefetch.PrefetchingStatistics; import org.apache.hadoop.fs.statistics.DurationTracker; /** @@ -26,7 +27,7 @@ * It also contains getters for tests. */ public interface S3AInputStreamStatistics extends AutoCloseable, - S3AStatisticInterface { + S3AStatisticInterface, PrefetchingStatistics { /** * Seek backwards, incrementing the seek and backward seek counters. @@ -96,6 +97,20 @@ void streamClose(boolean abortedConnection, */ void readOperationCompleted(int requested, int actual); + /** + * A vectored read operation has started.. + * @param numIncomingRanges number of input ranges. + * @param numCombinedRanges number of combined ranges. + */ + void readVectoredOperationStarted(int numIncomingRanges, + int numCombinedRanges); + + /** + * Number of bytes discarded during vectored read. + * @param discarded discarded bytes during vectored read. + */ + void readVectoredBytesDiscarded(int discarded); + @Override void close(); diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/statistics/impl/EmptyS3AStatisticsContext.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/statistics/impl/EmptyS3AStatisticsContext.java index 5c0995e41b3dd..d10b6484175b1 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/statistics/impl/EmptyS3AStatisticsContext.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/statistics/impl/EmptyS3AStatisticsContext.java @@ -195,6 +195,17 @@ public void readOperationCompleted(final int requested, final int actual) { } + @Override + public void readVectoredOperationStarted(int numIncomingRanges, + int numCombinedRanges) { + + } + + @Override + public void readVectoredBytesDiscarded(int discarded) { + + } + @Override public void close() { @@ -210,6 +221,41 @@ public void unbuffered() { } + @Override + public DurationTracker prefetchOperationStarted() { + return stubDurationTracker(); + } + + @Override + public void prefetchOperationCompleted() { + + } + + @Override + public void blockAddedToFileCache() { + + } + + @Override + public void blockRemovedFromFileCache() { + + } + + @Override + public void executorAcquired(Duration timeInQueue) { + + } + + @Override + public void memoryAllocated(int size) { + + } + + @Override + public void memoryFreed(int size) { + + } + /** * Return an IO statistics instance. * @return an empty IO statistics instance. @@ -343,6 +389,7 @@ public DurationTracker initiateGetRequest() { public DurationTracker initiateInnerStreamClose(final boolean abort) { return stubDurationTracker(); } + } /** diff --git a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/aws_sdk_upgrade.md b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/aws_sdk_upgrade.md new file mode 100644 index 0000000000000..e649a8d76d539 --- /dev/null +++ b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/aws_sdk_upgrade.md @@ -0,0 +1,76 @@ + + +# Upgrading S3A to AWS SDK V2 + +This document explains the upcoming work for upgrading S3A to AWS SDK V2. +This work is tracked in [HADOOP-18073](https://issues.apache.org/jira/browse/HADOOP-18073). + +## Why the upgrade? + +- Moving to SDK V2 will provide performance benefits. +For example, the [transfer manager for SDKV2](https://aws.amazon.com/blogs/developer/introducing-amazon-s3-transfer-manager-in-the-aws-sdk-for-java-2-x/) +is built using java bindings of the AWS Common Runtime S3 +client (https://github.com/awslabs/aws-crt-java) (CRT). +CRT is a set of packages written in C, designed for maximising performance when interacting with AWS +services such as S3. +- New features such as [additional checksum algorithms](https://aws.amazon.com/blogs/aws/new-additional-checksum-algorithms-for-amazon-s3/) +which S3A will benefit from are not available in SDKV1. + +## What's changing? + +The [SDK V2](https://github.com/aws/aws-sdk-java-v2) for S3 is very different from +[SDK V1](https://github.com/aws/aws-sdk-java), and brings breaking changes for S3A. +A complete list of the changes can be found in the [Changelog](https://github.com/aws/aws-sdk-java-v2/blob/master/docs/LaunchChangelog.md#41-s3-changes). + +The major changes and how this affects S3A are listed below. + +### Package Change + +Package names have changed, all classes in SDK V2 are under `software.amazon.awssdk`, SDK V1 classes +were under `com.amazonaws`. + +### Credential Providers + +- Interface change: [com.amazonaws.auth.AWSCredentialsProvider](https://github.com/aws/aws-sdk-java/blob/master/aws-java-sdk-core/src/main/java/com/amazonaws/auth/AWSCredentialsProvider.java) +has been replaced by [software.amazon.awssdk.auth.credentials.AwsCredentialsProvider](https://github.com/aws/aws-sdk-java-v2/blob/master/core/auth/src/main/java/software/amazon/awssdk/auth/credentials/AwsCredentialsProvider.java). +- Credential provider class changes: the package and class names of credential providers have +changed. + +The change in interface will mean that custom credential providers will need to be updated to now +implement `AwsCredentialsProvider` instead of `AWSCredentialProvider`. + +Due to change in class names, references to SDK V1 credential providers +in `fs.s3a.aws.credentials.provider` will need to be updated to reference V2 providers. + +### Delegation Tokens + +Custom credential providers used in delegation token binding classes will also need to be updated. + +### AmazonS3 replaced by S3Client + +The s3 client is an instance of `S3Client` in V2 rather than `AmazonS3`. + +For this reason, the `S3ClientFactory` will be deprecated and replaced by one that creates a V2 +`S3Client`. + +The `getAmazonS3ClientForTesting()` method will also be updated to return the `S3Client`. + +### Signers + +Interface change: [com.amazonaws.auth.Signer](https://github.com/aws/aws-sdk-java/blob/master/aws-java-sdk-core/src/main/java/com/amazonaws/auth/Signer.java) +has been replaced by [software.amazon.awssdk.core.signer.Signer](https://github.com/aws/aws-sdk-java-v2/blob/master/core/sdk-core/src/main/java/software/amazon/awssdk/core/signer/Signer.java). + +The change in signers will mean the custom signers will need to be updated to implement the new +interface. diff --git a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md index 7c0a49f8fbeda..cd7793bfa92d3 100644 --- a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md +++ b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md @@ -47,6 +47,8 @@ full details. * [Auditing](./auditing.html). * [Auditing Architecture](./auditing_architecture.html). * [Testing](./testing.html) +* [Prefetching](./prefetching.html) +* [Upcoming upgrade to AWS Java SDK V2](./aws_sdk_upgrade.html) ## Overview @@ -234,6 +236,9 @@ needs the credentials needed to interact with buckets. The client supports multiple authentication mechanisms and can be configured as to which mechanisms to use, and their order of use. Custom implementations of `com.amazonaws.auth.AWSCredentialsProvider` may also be used. +However, with the upcoming upgrade to AWS Java SDK V2, these classes will need to be +updated to implement `software.amazon.awssdk.auth.credentials.AwsCredentialsProvider`. +For more information see [Upcoming upgrade to AWS Java SDK V2](./aws_sdk_upgrade.html). *Important*: The S3A connector no longer supports username and secrets in URLs of the form `s3a://key:secret@bucket/`. @@ -1090,6 +1095,29 @@ options are covered in [Testing](./testing.md). + + fs.s3a.prefetch.enabled + false + + Enables prefetching and caching when reading from input stream. + + + + + fs.s3a.prefetch.block.size + 8MB + + The size of a single prefetched block of data. + + + + + fs.s3a.prefetch.block.count + 8 + + Maximum number of blocks prefetched concurrently at any given time. + + ``` ## Retry and Recovery diff --git a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/prefetching.md b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/prefetching.md new file mode 100644 index 0000000000000..9b4e888d604ca --- /dev/null +++ b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/prefetching.md @@ -0,0 +1,192 @@ + + +# S3A Prefetching + +This document explains the `S3PrefetchingInputStream` and the various components it uses. + +This input stream implements prefetching and caching to improve read performance of the input +stream. +A high level overview of this feature was published in +[Pinterest Engineering's blog post titled "Improving efficiency and reducing runtime using S3 read optimization"](https://medium.com/pinterest-engineering/improving-efficiency-and-reducing-runtime-using-s3-read-optimization-b31da4b60fa0). + +With prefetching, the input stream divides the remote file into blocks of a fixed size, associates +buffers to these blocks and then reads data into these buffers asynchronously. +It also potentially caches these blocks. + +### Basic Concepts + +* **Remote File**: A binary blob of data stored on some storage device. +* **Block File**: Local file containing a block of the remote file. +* **Block**: A file is divided into a number of blocks. +The size of the first n-1 blocks is same, and the size of the last block may be same or smaller. +* **Block based reading**: The granularity of read is one block. +That is, either an entire block is read and returned or none at all. +Multiple blocks may be read in parallel. + +### Configuring the stream + +|Property |Meaning |Default | +|---|---|---| +|`fs.s3a.prefetch.enabled` |Enable the prefetch input stream |`false` | +|`fs.s3a.prefetch.block.size` |Size of a block |`8M` | +|`fs.s3a.prefetch.block.count` |Number of blocks to prefetch |`8` | + +### Key Components + +`S3PrefetchingInputStream` - When prefetching is enabled, S3AFileSystem will return an instance of +this class as the input stream. +Depending on the remote file size, it will either use +the `S3InMemoryInputStream` or the `S3CachingInputStream` as the underlying input stream. + +`S3InMemoryInputStream` - Underlying input stream used when the remote file size < configured block +size. +Will read the entire remote file into memory. + +`S3CachingInputStream` - Underlying input stream used when remote file size > configured block size. +Uses asynchronous prefetching of blocks and caching to improve performance. + +`BlockData` - Holds information about the blocks in a remote file, such as: + +* Number of blocks in the remote file +* Block size +* State of each block (initially all blocks have state *NOT_READY*). +Other states are: Queued, Ready, Cached. + +`BufferData` - Holds the buffer and additional information about it such as: + +* The block number this buffer is for +* State of the buffer (Unknown, Blank, Prefetching, Caching, Ready, Done). +Initial state of a buffer is blank. + +`CachingBlockManager` - Implements reading data into the buffer, prefetching and caching. + +`BufferPool` - Manages a fixed sized pool of buffers. +It's used by `CachingBlockManager` to acquire buffers. + +`S3File` - Implements operations to interact with S3 such as opening and closing the input stream to +the remote file in S3. + +`S3Reader` - Implements reading from the stream opened by `S3File`. +Reads from this input stream in blocks of 64KB. + +`FilePosition` - Provides functionality related to tracking the position in the file. +Also gives access to the current buffer in use. + +`SingleFilePerBlockCache` - Responsible for caching blocks to the local file system. +Each cache block is stored on the local disk as a separate block file. + +### Operation + +#### S3InMemoryInputStream + +For a remote file with size 5MB, and block size = 8MB, since file size is less than the block size, +the `S3InMemoryInputStream` will be used. + +If the caller makes the following read calls: + +``` +in.read(buffer, 0, 3MB); +in.read(buffer, 0, 2MB); +``` + +When the first read is issued, there is no buffer in use yet. +The `S3InMemoryInputStream` gets the data in this remote file by calling the `ensureCurrentBuffer()` +method, which ensures that a buffer with data is available to be read from. + +The `ensureCurrentBuffer()` then: + +* Reads data into a buffer by calling `S3Reader.read(ByteBuffer buffer, long offset, int size)`. +* `S3Reader` uses `S3File` to open an input stream to the remote file in S3 by making + a `getObject()` request with range as `(0, filesize)`. +* The `S3Reader` reads the entire remote file into the provided buffer, and once reading is complete + closes the S3 stream and frees all underlying resources. +* Now the entire remote file is in a buffer, set this data in `FilePosition` so it can be accessed + by the input stream. + +The read operation now just gets the required bytes from the buffer in `FilePosition`. + +When the second read is issued, there is already a valid buffer which can be used. +Don't do anything else, just read the required bytes from this buffer. + +#### S3CachingInputStream + +If there is a remote file with size 40MB and block size = 8MB, the `S3CachingInputStream` will be +used. + +##### Sequential Reads + +If the caller makes the following calls: + +``` +in.read(buffer, 0, 5MB) +in.read(buffer, 0, 8MB) +``` + +For the first read call, there is no valid buffer yet. +`ensureCurrentBuffer()` is called, and for the first `read()`, prefetch count is set as 1. + +The current block (block 0) is read synchronously, while the blocks to be prefetched (block 1) is +read asynchronously. + +The `CachingBlockManager` is responsible for getting buffers from the buffer pool and reading data +into them. This process of acquiring the buffer pool works as follows: + +* The buffer pool keeps a map of allocated buffers and a pool of available buffers. +The size of this pool is = prefetch block count + 1. +If the prefetch block count is 8, the buffer pool has a size of 9. +* If the pool is not yet at capacity, create a new buffer and add it to the pool. +* If it's at capacity, check if any buffers with state = done can be released. +Releasing a buffer means removing it from allocated and returning it back to the pool of available +buffers. +* If there are no buffers with state = done currently then nothing will be released, so retry the + above step at a fixed interval a few times till a buffer becomes available. +* If after multiple retries there are still no available buffers, release a buffer in the ready state. +The buffer for the block furthest from the current block is released. + +Once a buffer has been acquired by `CachingBlockManager`, if the buffer is in a *READY* state, it is +returned. +This means that data was already read into this buffer asynchronously by a prefetch. +If it's state is *BLANK* then data is read into it using +`S3Reader.read(ByteBuffer buffer, long offset, int size).` + +For the second read call, `in.read(buffer, 0, 8MB)`, since the block sizes are of 8MB and only 5MB +of block 0 has been read so far, 3MB of the required data will be read from the current block 0. +Once all data has been read from this block, `S3CachingInputStream` requests the next block ( +block 1), which will already have been prefetched and so it can just start reading from it. +Also, while reading from block 1 it will also issue prefetch requests for the next blocks. +The number of blocks to be prefetched is determined by `fs.s3a.prefetch.block.count`. + +##### Random Reads + +If the caller makes the following calls: + +``` +in.read(buffer, 0, 5MB) +in.seek(10MB) +in.read(buffer, 0, 4MB) +in.seek(2MB) +in.read(buffer, 0, 4MB) +``` + +The `CachingInputStream` also caches prefetched blocks. +This happens when a `seek()` is issued for outside the current block and the current block still has +not been fully read. + +For the above read sequence, when the `seek(10MB)` call is issued, block 0 has not been read +completely so cache it as the caller will probably want to read from it again. + +When `seek(2MB)` is called, the position is back inside block 0. +The next read can now be satisfied from the locally cached block file, which is typically orders of +magnitude faster than a network based read. \ No newline at end of file diff --git a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/testing.md b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/testing.md index 44525c8ccd943..bbe1d0f808194 100644 --- a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/testing.md +++ b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/testing.md @@ -1156,6 +1156,14 @@ We need a run through of the CLI to see if there have been changes there which cause problems, especially whether new log messages have surfaced, or whether some packaging change breaks that CLI. +It is always interesting when doing this to enable IOStatistics reporting +```xml + + fs.iostatistics.logging.level + info + +``` + From the root of the project, create a command line release `mvn package -Pdist -DskipTests -Dmaven.javadoc.skip=true -DskipShade`; 1. Change into the `hadoop-dist/target/hadoop-x.y.z-SNAPSHOT` dir. @@ -1274,6 +1282,21 @@ bin/hadoop s3guard markers -clean -verbose $BUCKET # expect success and exit code of 0 bin/hadoop s3guard markers -audit -verbose $BUCKET +# --------------------------------------------------- +# Copy to from local +# --------------------------------------------------- + +time bin/hadoop fs -copyFromLocal -t 10 share/hadoop/tools/lib/*aws*jar $BUCKET/ + +# expect the iostatistics object_list_request value to be O(directories) +bin/hadoop fs -ls -R $BUCKET/ + +# expect the iostatistics object_list_request and op_get_content_summary values to be 1 +bin/hadoop fs -du -h -s $BUCKET/ + +mkdir tmp +time bin/hadoop fs -copyToLocal -t 10 $BUCKET/\*aws\* tmp + # --------------------------------------------------- # S3 Select on Landsat # --------------------------------------------------- @@ -1283,6 +1306,17 @@ export LANDSATGZ=s3a://landsat-pds/scene_list.gz bin/hadoop s3guard select -header use -compression gzip $LANDSATGZ \ "SELECT s.entityId,s.cloudCover FROM S3OBJECT s WHERE s.cloudCover < '0.0' LIMIT 100" + +# --------------------------------------------------- +# Cloudstore +# check out and build https://github.com/steveloughran/cloudstore +# then for these tests, set CLOUDSTORE env var to point to the JAR +# --------------------------------------------------- + +bin/hadoop jar $CLOUDSTORE storediag $BUCKET + +time bin/hadoop jar $CLOUDSTORE bandwidth 64M $BUCKET/testfile + ``` ### Other tests diff --git a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/troubleshooting_s3a.md b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/troubleshooting_s3a.md index 903310a94bfe8..ef9f7f123e55c 100644 --- a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/troubleshooting_s3a.md +++ b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/troubleshooting_s3a.md @@ -21,12 +21,13 @@ Common problems working with S3 are: 1. [Classpath setup](#classpath) -1. [Authentication](#authentication) -1. [Access Denial](#access_denied) -1. [Connectivity Problems](#connectivity) -1. [File System Semantics](#semantics) -1. [Encryption](#encryption) -1. [Other Errors](#other) +2. [Authentication](#authentication) +3. [Access Denial](#access_denied) +4. [Connectivity Problems](#connectivity) +5. [File System Semantics](#semantics) +6. [Encryption](#encryption) +7. [Other Errors](#other) +8. [SDK Upgrade Warnings](#upgrade_warnings) This document also includes some [best pactises](#best) to aid troubleshooting. @@ -1989,3 +1990,51 @@ com.amazonaws.SdkClientException: Unable to execute HTTP request: When this happens, try to set `fs.s3a.connection.request.timeout` to a larger value or disable it completely by setting it to `0`. + +## SDK Upgrade Warnings + +S3A will soon be upgraded to [AWS's Java SDK V2](https://github.com/aws/aws-sdk-java-v2). +For more information on the upgrade and what's changing, see +[Upcoming upgrade to AWS Java SDK V2](./aws_sdk_upgrade.html). + +S3A logs the following warnings for things that will be changing in the upgrade. To disable these +logs, comment out `log4j.logger.org.apache.hadoop.fs.s3a.SDKV2Upgrade` in log4j.properties. + +### `Directly referencing AWS SDK V1 credential provider` + +This will be logged when an AWS credential provider is referenced directly in +`fs.s3a.aws.credentials.provider`. +For example, `com.amazonaws.auth.AWSSessionCredentialsProvider` + +To stop this warning, remove any AWS credential providers from `fs.s3a.aws.credentials.provider`. +Instead, use S3A's credential providers. + +### `getAmazonS3ClientForTesting() will be removed` + +This will be logged when `getAmazonS3ClientForTesting()` is called to get the S3 Client. With V2, +the S3 client will change from type `com.amazonaws.services.s3.AmazonS3` to +`software.amazon.awssdk.services.s3.S3Client`, and so this method will be removed. + +### +### `Custom credential providers used in delegation tokens binding classes will need to be updated` + +This will be logged when delegation tokens are used. +Delegation tokens allow the use of custom binding classes which can implement custom credential +providers. +These credential providers will currently be implementing +`com.amazonaws.auth.AWSCredentialsProvider` and will need to be updated to implement +`software.amazon.awssdk.auth.credentials.AwsCredentialsProvider`. + +### +### `The signer interface has changed in AWS SDK V2, custom signers will need to be updated` + +This will be logged when a custom signer is used. +Custom signers will currently be implementing `com.amazonaws.auth.Signer` and will need to be +updated to implement `software.amazon.awssdk.core.signer.Signer`. + +### +### `getObjectMetadata() called. This operation and it's response will be changed` + +This will be logged when `getObjectMetadata` is called. In SDK V2, this operation has changed to +`headObject()` and will return a response of the type `HeadObjectResponse`. + diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/ITestS3AContractVectoredRead.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/ITestS3AContractVectoredRead.java index 18a727dcdceed..4c357e288c84f 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/ITestS3AContractVectoredRead.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/ITestS3AContractVectoredRead.java @@ -19,28 +19,41 @@ package org.apache.hadoop.fs.contract.s3a; import java.io.EOFException; +import java.io.IOException; import java.io.InterruptedIOException; import java.util.ArrayList; import java.util.List; +import java.util.concurrent.CompletableFuture; import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FSDataInputStream; import org.apache.hadoop.fs.FileRange; +import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.contract.AbstractContractVectoredReadTest; import org.apache.hadoop.fs.contract.AbstractFSContract; import org.apache.hadoop.fs.s3a.Constants; import org.apache.hadoop.fs.s3a.S3AFileSystem; import org.apache.hadoop.fs.s3a.S3ATestUtils; +import org.apache.hadoop.fs.statistics.IOStatistics; +import org.apache.hadoop.fs.statistics.StoreStatisticNames; +import org.apache.hadoop.fs.statistics.StreamStatisticNames; import org.apache.hadoop.test.LambdaTestUtils; +import static org.apache.hadoop.fs.contract.ContractTestUtils.returnBuffersToPoolPostRead; import static org.apache.hadoop.fs.contract.ContractTestUtils.validateVectoredReadResult; +import static org.apache.hadoop.fs.statistics.IOStatisticAssertions.verifyStatisticCounterValue; +import static org.apache.hadoop.fs.statistics.IOStatisticsLogging.ioStatisticsToPrettyString; import static org.apache.hadoop.test.MoreAsserts.assertEqual; public class ITestS3AContractVectoredRead extends AbstractContractVectoredReadTest { + private static final Logger LOG = LoggerFactory.getLogger(ITestS3AContractVectoredRead.class); + public ITestS3AContractVectoredRead(String bufferType) { super(bufferType); } @@ -156,4 +169,188 @@ public void testSameRanges() throws Exception { List fileRanges = getSampleSameRanges(); verifyExceptionalVectoredRead(fs, fileRanges, UnsupportedOperationException.class); } + + /** + * As the minimum seek value is 4*1024, the first three ranges will be + * merged into and other two will remain as it is. + * */ + @Test + public void testNormalReadVsVectoredReadStatsCollection() throws Exception { + + try (S3AFileSystem fs = getTestFileSystemWithReadAheadDisabled()) { + List fileRanges = new ArrayList<>(); + fileRanges.add(FileRange.createFileRange(10 * 1024, 100)); + fileRanges.add(FileRange.createFileRange(8 * 1024, 100)); + fileRanges.add(FileRange.createFileRange(14 * 1024, 100)); + fileRanges.add(FileRange.createFileRange(2 * 1024 - 101, 100)); + fileRanges.add(FileRange.createFileRange(40 * 1024, 1024)); + + FileStatus fileStatus = fs.getFileStatus(path(VECTORED_READ_FILE_NAME)); + CompletableFuture builder = + fs.openFile(path(VECTORED_READ_FILE_NAME)) + .withFileStatus(fileStatus) + .build(); + try (FSDataInputStream in = builder.get()) { + in.readVectored(fileRanges, getAllocate()); + validateVectoredReadResult(fileRanges, DATASET); + returnBuffersToPoolPostRead(fileRanges, getPool()); + + // audit the io statistics for this stream + IOStatistics st = in.getIOStatistics(); + LOG.info("IOStats after readVectored operation {}", ioStatisticsToPrettyString(st)); + + // the vectored io operation must be tracked + verifyStatisticCounterValue(st, + StreamStatisticNames.STREAM_READ_VECTORED_OPERATIONS, + 1); + + // the vectored io operation is being called with 5 input ranges. + verifyStatisticCounterValue(st, + StreamStatisticNames.STREAM_READ_VECTORED_INCOMING_RANGES, + 5); + + // 5 input ranges got combined in 3 as some of them are close. + verifyStatisticCounterValue(st, + StreamStatisticNames.STREAM_READ_VECTORED_COMBINED_RANGES, + 3); + + // number of bytes discarded will be based on the above input ranges. + verifyStatisticCounterValue(st, + StreamStatisticNames.STREAM_READ_VECTORED_READ_BYTES_DISCARDED, + 5944); + + verifyStatisticCounterValue(st, + StoreStatisticNames.ACTION_HTTP_GET_REQUEST, + 3); + + // read bytes should match the sum of requested length for each input ranges. + verifyStatisticCounterValue(st, + StreamStatisticNames.STREAM_READ_BYTES, + 1424); + + } + + CompletableFuture builder1 = + fs.openFile(path(VECTORED_READ_FILE_NAME)) + .withFileStatus(fileStatus) + .build(); + + try (FSDataInputStream in = builder1.get()) { + for (FileRange range : fileRanges) { + byte[] temp = new byte[range.getLength()]; + in.readFully((int) range.getOffset(), temp, 0, range.getLength()); + } + + // audit the statistics for this stream + IOStatistics st = in.getIOStatistics(); + LOG.info("IOStats after read fully operation {}", ioStatisticsToPrettyString(st)); + + verifyStatisticCounterValue(st, + StreamStatisticNames.STREAM_READ_VECTORED_OPERATIONS, + 0); + + // all other counter values consistent. + verifyStatisticCounterValue(st, + StreamStatisticNames.STREAM_READ_VECTORED_READ_BYTES_DISCARDED, + 0); + verifyStatisticCounterValue(st, + StoreStatisticNames.ACTION_HTTP_GET_REQUEST, + 5); + + // read bytes should match the sum of requested length for each input ranges. + verifyStatisticCounterValue(st, + StreamStatisticNames.STREAM_READ_BYTES, + 1424); + } + // validate stats are getting merged at fs instance level. + IOStatistics fsStats = fs.getIOStatistics(); + // only 1 vectored io call is made in this fs instance. + verifyStatisticCounterValue(fsStats, + StreamStatisticNames.STREAM_READ_VECTORED_OPERATIONS, + 1); + // 8 get requests were made in this fs instance. + verifyStatisticCounterValue(fsStats, + StoreStatisticNames.ACTION_HTTP_GET_REQUEST, + 8); + + verifyStatisticCounterValue(fsStats, + StreamStatisticNames.STREAM_READ_BYTES, + 2848); + } + } + + @Test + public void testMultiVectoredReadStatsCollection() throws Exception { + try (S3AFileSystem fs = getTestFileSystemWithReadAheadDisabled()) { + List ranges1 = getConsecutiveRanges(); + List ranges2 = getConsecutiveRanges(); + FileStatus fileStatus = fs.getFileStatus(path(VECTORED_READ_FILE_NAME)); + CompletableFuture builder = + fs.openFile(path(VECTORED_READ_FILE_NAME)) + .withFileStatus(fileStatus) + .build(); + try (FSDataInputStream in = builder.get()) { + in.readVectored(ranges1, getAllocate()); + in.readVectored(ranges2, getAllocate()); + validateVectoredReadResult(ranges1, DATASET); + validateVectoredReadResult(ranges2, DATASET); + returnBuffersToPoolPostRead(ranges1, getPool()); + returnBuffersToPoolPostRead(ranges2, getPool()); + + // audit the io statistics for this stream + IOStatistics st = in.getIOStatistics(); + + // 2 vectored io calls are made above. + verifyStatisticCounterValue(st, + StreamStatisticNames.STREAM_READ_VECTORED_OPERATIONS, + 2); + + // 2 vectored io operation is being called with 2 input ranges. + verifyStatisticCounterValue(st, + StreamStatisticNames.STREAM_READ_VECTORED_INCOMING_RANGES, + 4); + + // 2 ranges are getting merged in 1 during both vectored io operation. + verifyStatisticCounterValue(st, + StreamStatisticNames.STREAM_READ_VECTORED_COMBINED_RANGES, + 2); + + // number of bytes discarded will be 0 as the ranges are consecutive. + verifyStatisticCounterValue(st, + StreamStatisticNames.STREAM_READ_VECTORED_READ_BYTES_DISCARDED, + 0); + // only 2 http get request will be made because ranges in both range list will be merged + // to 1 because they are consecutive. + verifyStatisticCounterValue(st, + StoreStatisticNames.ACTION_HTTP_GET_REQUEST, + 2); + // read bytes should match the sum of requested length for each input ranges. + verifyStatisticCounterValue(st, + StreamStatisticNames.STREAM_READ_BYTES, + 2000); + } + IOStatistics fsStats = fs.getIOStatistics(); + // 2 vectored io calls are made in this fs instance. + verifyStatisticCounterValue(fsStats, + StreamStatisticNames.STREAM_READ_VECTORED_OPERATIONS, + 2); + // 2 get requests were made in this fs instance. + verifyStatisticCounterValue(fsStats, + StoreStatisticNames.ACTION_HTTP_GET_REQUEST, + 2); + } + } + + private S3AFileSystem getTestFileSystemWithReadAheadDisabled() throws IOException { + Configuration conf = getFileSystem().getConf(); + // also resetting the min seek and max size values is important + // as this same test suite has test which overrides these params. + S3ATestUtils.removeBaseAndBucketOverrides(conf, + Constants.READAHEAD_RANGE, + Constants.AWS_S3_VECTOR_READS_MAX_MERGED_READ_SIZE, + Constants.AWS_S3_VECTOR_READS_MIN_SEEK_SIZE); + S3ATestUtils.disableFilesystemCaching(conf); + conf.setInt(Constants.READAHEAD_RANGE, 0); + return S3ATestUtils.createTestFileSystem(conf); + } } diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/AbstractS3AMockTest.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/AbstractS3AMockTest.java index a80a24881fa74..a46303f339678 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/AbstractS3AMockTest.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/AbstractS3AMockTest.java @@ -62,6 +62,7 @@ public void setup() throws Exception { s3 = fs.getAmazonS3ClientForTesting("mocking"); } + @SuppressWarnings("deprecation") public Configuration createConfiguration() { Configuration conf = new Configuration(); conf.setClass(S3_CLIENT_FACTORY_IMPL, MockS3ClientFactory.class, diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/AbstractS3ATestBase.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/AbstractS3ATestBase.java index 213f94432c4ac..e90ad8b73efae 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/AbstractS3ATestBase.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/AbstractS3ATestBase.java @@ -27,6 +27,7 @@ import org.apache.hadoop.fs.contract.s3a.S3AContract; import org.apache.hadoop.fs.s3a.tools.MarkerTool; import org.apache.hadoop.fs.statistics.IOStatisticsSnapshot; +import org.apache.hadoop.fs.statistics.IOStatisticsContext; import org.apache.hadoop.fs.store.audit.AuditSpan; import org.apache.hadoop.fs.store.audit.AuditSpanSource; import org.apache.hadoop.io.IOUtils; @@ -38,6 +39,7 @@ import java.io.FileNotFoundException; import java.io.IOException; +import java.util.concurrent.atomic.AtomicReference; import static org.apache.hadoop.fs.contract.ContractTestUtils.dataset; import static org.apache.hadoop.fs.contract.ContractTestUtils.writeDataset; @@ -66,6 +68,15 @@ public abstract class AbstractS3ATestBase extends AbstractFSContractTestBase */ private AuditSpanSource spanSource; + /** + * Atomic references to be used to re-throw an Exception or an ASE + * caught inside a lambda function. + */ + private static final AtomicReference FUTURE_EXCEPTION = + new AtomicReference<>(); + private static final AtomicReference FUTURE_ASE = + new AtomicReference<>(); + /** * Get the source. * @return span source @@ -99,6 +110,9 @@ public void setup() throws Exception { S3AFileSystem.initializeClass(); super.setup(); setSpanSource(getFileSystem()); + // Reset the current context's thread IOStatistics.` + // this ensures that the context stats will always be from the test case + IOStatisticsContext.getCurrentIOStatisticsContext().reset(); } @Override @@ -263,4 +277,53 @@ public void skipIfClientSideEncryption() { Assume.assumeTrue("Skipping test if CSE is enabled", !getFileSystem().isCSEEnabled()); } + + /** + * If an exception is caught while doing some work in a Lambda function, + * store it in an atomic reference to be thrown later on. + * @param exception Exception caught. + */ + public static void setFutureException(Exception exception) { + FUTURE_EXCEPTION.set(exception); + } + + /** + * If an Assertion is caught while doing some work in a Lambda function, + * store it in an atomic reference to be thrown later on. + * + * @param ase Assertion Error caught. + */ + public static void setFutureAse(AssertionError ase) { + FUTURE_ASE.set(ase); + } + + /** + * throw the caught exception from the atomic reference and also clear the + * atomic reference so that we don't rethrow in another test. + * + * @throws Exception the exception caught. + */ + public static void maybeReThrowFutureException() throws Exception { + if (FUTURE_EXCEPTION.get() != null) { + Exception exceptionToThrow = FUTURE_EXCEPTION.get(); + // reset the atomic ref before throwing. + setFutureAse(null); + throw exceptionToThrow; + } + } + + /** + * throw the Assertion error from the atomic reference and also clear the + * atomic reference so that we don't rethrow in another test. + * + * @throws Exception Assertion error caught. + */ + public static void maybeReThrowFutureASE() throws Exception { + if (FUTURE_ASE.get() != null) { + AssertionError aseToThrow = FUTURE_ASE.get(); + // reset the atomic ref before throwing. + setFutureAse(null); + throw aseToThrow; + } + } } diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AAWSCredentialsProvider.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AAWSCredentialsProvider.java index 30f1f71eeea88..c13c3f48b8466 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AAWSCredentialsProvider.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AAWSCredentialsProvider.java @@ -157,17 +157,21 @@ public void testBadCredentials() throws Exception { } @Test + @SuppressWarnings("deprecation") public void testAnonymousProvider() throws Exception { Configuration conf = new Configuration(); conf.set(AWS_CREDENTIALS_PROVIDER, AnonymousAWSCredentialsProvider.class.getName()); Path testFile = getCSVTestPath(conf); - FileSystem fs = FileSystem.newInstance(testFile.toUri(), conf); - assertNotNull(fs); - assertTrue(fs instanceof S3AFileSystem); - FileStatus stat = fs.getFileStatus(testFile); - assertNotNull(stat); - assertEquals(testFile, stat.getPath()); + try (FileSystem fs = FileSystem.newInstance(testFile.toUri(), conf)) { + assertNotNull("S3AFileSystem instance must not be null", fs); + assertTrue("FileSystem must be the instance of S3AFileSystem", fs instanceof S3AFileSystem); + FileStatus stat = fs.getFileStatus(testFile); + assertNotNull("FileStatus with qualified path must not be null", stat); + assertEquals( + "The qualified path returned by getFileStatus should be same as the original file", + testFile, stat.getPath()); + } } } diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3ACannedACLs.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3ACannedACLs.java index 7424b6f90b3c3..1071582cc67d2 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3ACannedACLs.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3ACannedACLs.java @@ -35,6 +35,7 @@ import org.apache.hadoop.fs.contract.ContractTestUtils; import org.apache.hadoop.fs.s3a.audit.S3AAuditConstants; import org.apache.hadoop.fs.s3a.impl.StoreContext; +import org.apache.hadoop.fs.store.audit.AuditSpan; import static org.apache.hadoop.fs.s3a.Constants.CANNED_ACL; import static org.apache.hadoop.fs.s3a.S3ATestUtils.removeBaseAndBucketOverrides; @@ -64,15 +65,17 @@ protected Configuration createConfiguration() { @Test public void testCreatedObjectsHaveACLs() throws Throwable { S3AFileSystem fs = getFileSystem(); - Path dir = methodPath(); - fs.mkdirs(dir); - assertObjectHasLoggingGrant(dir, false); - Path path = new Path(dir, "1"); - ContractTestUtils.touch(fs, path); - assertObjectHasLoggingGrant(path, true); - Path path2 = new Path(dir, "2"); - fs.rename(path, path2); - assertObjectHasLoggingGrant(path2, true); + try (AuditSpan span = span()) { + Path dir = methodPath(); + fs.mkdirs(dir); + assertObjectHasLoggingGrant(dir, false); + Path path = new Path(dir, "1"); + ContractTestUtils.touch(fs, path); + assertObjectHasLoggingGrant(path, true); + Path path2 = new Path(dir, "2"); + fs.rename(path, path2); + assertObjectHasLoggingGrant(path2, true); + } } /** diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AConfiguration.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AConfiguration.java index 61d12747c0a58..26d00bc7d359a 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AConfiguration.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AConfiguration.java @@ -414,6 +414,7 @@ public void testRequestTimeout() throws Exception { } @Test + @SuppressWarnings("deprecation") public void testCloseIdempotent() throws Throwable { conf = new Configuration(); fs = S3ATestUtils.createTestFileSystem(conf); diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3ADeleteOnExit.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3ADeleteOnExit.java new file mode 100644 index 0000000000000..31c58de629b5f --- /dev/null +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3ADeleteOnExit.java @@ -0,0 +1,105 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.fs.s3a; + +import org.junit.Test; + +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; + +import static org.apache.hadoop.fs.contract.ContractTestUtils.createFile; +import static org.apache.hadoop.fs.contract.ContractTestUtils.dataset; + +/** + * Test deleteOnExit for S3A. + * The following cases for deleteOnExit are tested: + * 1. A nonexist file, which is added to deleteOnExit set. + * 2. An existing file + * 3. A file is added to deleteOnExist set first, then created. + * 4. A directory with some files under it. + */ +public class ITestS3ADeleteOnExit extends AbstractS3ATestBase { + + private static final String PARENT_DIR_PATH_STR = "testDeleteOnExitDir"; + private static final String NON_EXIST_FILE_PATH_STR = + PARENT_DIR_PATH_STR + "/nonExistFile"; + private static final String INORDER_FILE_PATH_STR = + PARENT_DIR_PATH_STR + "/inOrderFile"; + private static final String OUT_OF_ORDER_FILE_PATH_STR = + PARENT_DIR_PATH_STR + "/outOfOrderFile"; + private static final String SUBDIR_PATH_STR = + PARENT_DIR_PATH_STR + "/subDir"; + private static final String FILE_UNDER_SUBDIR_PATH_STR = + SUBDIR_PATH_STR + "/subDirFile"; + + @Test + public void testDeleteOnExit() throws Exception { + FileSystem fs = getFileSystem(); + + // Get a new filesystem object which is same as fs. + FileSystem s3aFs = new S3AFileSystem(); + s3aFs.initialize(fs.getUri(), fs.getConf()); + Path nonExistFilePath = path(NON_EXIST_FILE_PATH_STR); + Path inOrderFilePath = path(INORDER_FILE_PATH_STR); + Path outOfOrderFilePath = path(OUT_OF_ORDER_FILE_PATH_STR); + Path subDirPath = path(SUBDIR_PATH_STR); + Path fileUnderSubDirPath = path(FILE_UNDER_SUBDIR_PATH_STR); + + // 1. set up the test directory. + Path dir = path("testDeleteOnExitDir"); + s3aFs.mkdirs(dir); + + // 2. Add a nonexisting file to DeleteOnExit set. + s3aFs.deleteOnExit(nonExistFilePath); + assertPathDoesNotExist("File " + NON_EXIST_FILE_PATH_STR + " should not exist", + nonExistFilePath); + + // 3. create a file and then add it to DeleteOnExit set. + byte[] data = dataset(16, 'a', 26); + createFile(s3aFs, inOrderFilePath, true, data); + assertPathExists("File " + INORDER_FILE_PATH_STR + " should exist", inOrderFilePath); + s3aFs.deleteOnExit(inOrderFilePath); + + // 4. add a path to DeleteOnExit set first, then create it. + s3aFs.deleteOnExit(outOfOrderFilePath); + createFile(s3aFs, outOfOrderFilePath, true, data); + assertPathExists("File " + OUT_OF_ORDER_FILE_PATH_STR + " should exist", + outOfOrderFilePath); + + // 5. create a subdirectory, a file under it, and add subdirectory DeleteOnExit set. + s3aFs.mkdirs(subDirPath); + s3aFs.deleteOnExit(subDirPath); + createFile(s3aFs, fileUnderSubDirPath, true, data); + assertPathExists("Directory " + SUBDIR_PATH_STR + " should exist", subDirPath); + assertPathExists("File " + FILE_UNDER_SUBDIR_PATH_STR + " should exist", + fileUnderSubDirPath); + + s3aFs.close(); + + // After s3aFs is closed, make sure that all files/directories in deleteOnExit + // set are deleted. + assertPathDoesNotExist("File " + NON_EXIST_FILE_PATH_STR + " should not exist", + nonExistFilePath); + assertPathDoesNotExist("File " + INORDER_FILE_PATH_STR + " should not exist", + inOrderFilePath); + assertPathDoesNotExist("File " + OUT_OF_ORDER_FILE_PATH_STR + " should not exist", + outOfOrderFilePath); + assertPathDoesNotExist("Directory " + SUBDIR_PATH_STR + " should not exist", + subDirPath); + } +} diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AEndpointRegion.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AEndpointRegion.java index 761dd558063ba..add6502d7da71 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AEndpointRegion.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AEndpointRegion.java @@ -165,6 +165,7 @@ private void createMarsNorth2Client(Configuration conf) throws Exception { * @throws URISyntaxException parse problems. * @throws IOException IO problems */ + @SuppressWarnings("deprecation") private AmazonS3 createS3Client(Configuration conf, String endpoint, String expectedRegion) @@ -176,6 +177,7 @@ private AmazonS3 createS3Client(Configuration conf, S3ClientFactory.S3ClientCreationParameters parameters = new S3ClientFactory.S3ClientCreationParameters() .withCredentialSet(new AnonymousAWSCredentialsProvider()) + .withPathUri(new URI("s3a://localhost/")) .withEndpoint(endpoint) .withMetrics(new EmptyS3AStatisticsContext() .newStatisticsFromAwsSdk()); diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AIOStatisticsContext.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AIOStatisticsContext.java new file mode 100644 index 0000000000000..70dc5ee476c47 --- /dev/null +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AIOStatisticsContext.java @@ -0,0 +1,497 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.s3a; + +import java.io.IOException; +import java.io.UncheckedIOException; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; + +import org.assertj.core.api.Assertions; +import org.junit.Test; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FSDataInputStream; +import org.apache.hadoop.fs.FSDataOutputStream; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.StreamCapabilities; +import org.apache.hadoop.fs.statistics.IOStatistics; +import org.apache.hadoop.fs.statistics.StoreStatisticNames; +import org.apache.hadoop.fs.statistics.IOStatisticsContext; +import org.apache.hadoop.fs.statistics.impl.IOStatisticsContextImpl; +import org.apache.hadoop.util.concurrent.HadoopExecutors; +import org.apache.hadoop.util.functional.CloseableTaskPoolSubmitter; +import org.apache.hadoop.util.functional.TaskPool; + +import static org.apache.hadoop.fs.contract.ContractTestUtils.assertCapabilities; +import static org.apache.hadoop.fs.contract.ContractTestUtils.dataset; +import static org.apache.hadoop.fs.contract.ContractTestUtils.writeDataset; +import static org.apache.hadoop.fs.s3a.S3ATestUtils.disablePrefetching; +import static org.apache.hadoop.fs.statistics.IOStatisticAssertions.assertThatStatisticCounter; +import static org.apache.hadoop.fs.statistics.IOStatisticAssertions.verifyStatisticCounterValue; +import static org.apache.hadoop.fs.statistics.StreamStatisticNames.STREAM_READ_BYTES; +import static org.apache.hadoop.fs.statistics.StreamStatisticNames.STREAM_WRITE_BYTES; +import static org.apache.hadoop.fs.statistics.impl.IOStatisticsContextIntegration.enableIOStatisticsContext; +import static org.apache.hadoop.fs.statistics.impl.IOStatisticsContextIntegration.getCurrentIOStatisticsContext; +import static org.apache.hadoop.fs.statistics.impl.IOStatisticsContextIntegration.getThreadSpecificIOStatisticsContext; +import static org.apache.hadoop.fs.statistics.impl.IOStatisticsContextIntegration.setThreadIOStatisticsContext; +import static org.apache.hadoop.util.functional.RemoteIterators.foreach; + +/** + * Tests to verify the Thread-level IOStatistics. + */ +public class ITestS3AIOStatisticsContext extends AbstractS3ATestBase { + + private static final int SMALL_THREADS = 2; + private static final int BYTES_BIG = 100; + private static final int BYTES_SMALL = 50; + private static final String[] IOSTATISTICS_CONTEXT_CAPABILITY = + new String[] {StreamCapabilities.IOSTATISTICS_CONTEXT}; + private ExecutorService executor; + + @Override + protected Configuration createConfiguration() { + Configuration configuration = super.createConfiguration(); + disablePrefetching(configuration); + enableIOStatisticsContext(); + return configuration; + } + + @Override + public void setup() throws Exception { + super.setup(); + executor = HadoopExecutors.newFixedThreadPool(SMALL_THREADS); + } + + @Override + public void teardown() throws Exception { + if (executor != null) { + executor.shutdown(); + } + super.teardown(); + } + + /** + * Verify that S3AInputStream aggregates per thread IOStats collection + * correctly. + */ + @Test + public void testS3AInputStreamIOStatisticsContext() + throws Exception { + S3AFileSystem fs = getFileSystem(); + Path path = path(getMethodName()); + byte[] data = dataset(256, 'a', 'z'); + byte[] readDataFirst = new byte[BYTES_BIG]; + byte[] readDataSecond = new byte[BYTES_SMALL]; + writeDataset(fs, path, data, data.length, 1024, true); + + CountDownLatch latch = new CountDownLatch(SMALL_THREADS); + + try { + + for (int i = 0; i < SMALL_THREADS; i++) { + executor.submit(() -> { + try { + // get the thread context and reset + IOStatisticsContext context = + getAndResetThreadStatisticsContext(); + try (FSDataInputStream in = fs.open(path)) { + // Assert the InputStream's stream capability to support + // IOStatisticsContext. + assertCapabilities(in, IOSTATISTICS_CONTEXT_CAPABILITY, null); + in.seek(50); + in.read(readDataFirst); + } + assertContextBytesRead(context, BYTES_BIG); + // Stream is closed for a thread. Re-open and do more operations. + try (FSDataInputStream in = fs.open(path)) { + in.seek(100); + in.read(readDataSecond); + } + assertContextBytesRead(context, BYTES_BIG + BYTES_SMALL); + + latch.countDown(); + } catch (Exception e) { + latch.countDown(); + setFutureException(e); + LOG.error("An error occurred while doing a task in the thread", e); + } catch (AssertionError ase) { + latch.countDown(); + setFutureAse(ase); + throw ase; + } + }); + } + // wait for tasks to finish. + latch.await(); + } finally { + executor.shutdown(); + } + + // Check if an Exception or ASE was caught while the test threads were running. + maybeReThrowFutureException(); + maybeReThrowFutureASE(); + + } + + /** + * get the thread context and reset. + * @return thread context + */ + private static IOStatisticsContext getAndResetThreadStatisticsContext() { + assertTrue("thread-level IOStatistics should be enabled by default", + IOStatisticsContext.enabled()); + IOStatisticsContext context = + IOStatisticsContext.getCurrentIOStatisticsContext(); + context.reset(); + return context; + } + + /** + * Verify that S3ABlockOutputStream aggregates per thread IOStats collection + * correctly. + */ + @Test + public void testS3ABlockOutputStreamIOStatisticsContext() + throws Exception { + S3AFileSystem fs = getFileSystem(); + Path path = path(getMethodName()); + byte[] writeDataFirst = new byte[BYTES_BIG]; + byte[] writeDataSecond = new byte[BYTES_SMALL]; + + final ExecutorService executorService = + HadoopExecutors.newFixedThreadPool(SMALL_THREADS); + CountDownLatch latch = new CountDownLatch(SMALL_THREADS); + + try { + for (int i = 0; i < SMALL_THREADS; i++) { + executorService.submit(() -> { + try { + // get the thread context and reset + IOStatisticsContext context = + getAndResetThreadStatisticsContext(); + try (FSDataOutputStream out = fs.create(path)) { + // Assert the OutputStream's stream capability to support + // IOStatisticsContext. + assertCapabilities(out, IOSTATISTICS_CONTEXT_CAPABILITY, null); + out.write(writeDataFirst); + } + assertContextBytesWrite(context, BYTES_BIG); + + // Stream is closed for a thread. Re-open and do more operations. + try (FSDataOutputStream out = fs.create(path)) { + out.write(writeDataSecond); + } + assertContextBytesWrite(context, BYTES_BIG + BYTES_SMALL); + latch.countDown(); + } catch (Exception e) { + latch.countDown(); + setFutureException(e); + LOG.error("An error occurred while doing a task in the thread", e); + } catch (AssertionError ase) { + latch.countDown(); + setFutureAse(ase); + throw ase; + } + }); + } + // wait for tasks to finish. + latch.await(); + } finally { + executorService.shutdown(); + } + + // Check if an Excp or ASE was caught while the test threads were running. + maybeReThrowFutureException(); + maybeReThrowFutureASE(); + } + + /** + * Verify stats collection and aggregation for constructor thread, Junit + * thread and a worker thread. + */ + @Test + public void testThreadIOStatisticsForDifferentThreads() + throws IOException, InterruptedException { + S3AFileSystem fs = getFileSystem(); + Path path = path(getMethodName()); + byte[] data = new byte[BYTES_BIG]; + long threadIdForTest = Thread.currentThread().getId(); + IOStatisticsContext context = + getAndResetThreadStatisticsContext(); + Assertions.assertThat(((IOStatisticsContextImpl)context).getThreadID()) + .describedAs("Thread ID of %s", context) + .isEqualTo(threadIdForTest); + Assertions.assertThat(((IOStatisticsContextImpl)context).getID()) + .describedAs("ID of %s", context) + .isGreaterThan(0); + + // Write in the Junit thread. + try (FSDataOutputStream out = fs.create(path)) { + out.write(data); + } + + // Read in the Junit thread. + try (FSDataInputStream in = fs.open(path)) { + in.read(data); + } + + // Worker thread work and wait for it to finish. + TestWorkerThread workerThread = new TestWorkerThread(path, null); + long workerThreadID = workerThread.getId(); + LOG.info("Worker thread ID: {} ", workerThreadID); + workerThread.start(); + workerThread.join(); + + assertThreadStatisticsForThread(threadIdForTest, BYTES_BIG); + assertThreadStatisticsForThread(workerThreadID, BYTES_SMALL); + } + + /** + * Verify stats collection and aggregation for constructor thread, Junit + * thread and a worker thread. + */ + @Test + public void testThreadSharingIOStatistics() + throws IOException, InterruptedException { + S3AFileSystem fs = getFileSystem(); + Path path = path(getMethodName()); + byte[] data = new byte[BYTES_BIG]; + long threadIdForTest = Thread.currentThread().getId(); + IOStatisticsContext context = + getAndResetThreadStatisticsContext(); + + + // Write in the Junit thread. + try (FSDataOutputStream out = fs.create(path)) { + out.write(data); + } + + // Read in the Junit thread. + try (FSDataInputStream in = fs.open(path)) { + in.read(data); + } + + // Worker thread will share the same context. + TestWorkerThread workerThread = new TestWorkerThread(path, context); + long workerThreadID = workerThread.getId(); + workerThread.start(); + workerThread.join(); + + assertThreadStatisticsForThread(threadIdForTest, BYTES_BIG + BYTES_SMALL); + + } + + /** + * Test to verify if setting the current IOStatisticsContext removes the + * current context and creates a new instance of it. + */ + @Test + public void testSettingNullIOStatisticsContext() { + IOStatisticsContext ioStatisticsContextBefore = + getCurrentIOStatisticsContext(); + // Set the current IOStatisticsContext to null, which should remove the + // context and set a new one. + setThreadIOStatisticsContext(null); + // Get the context again after setting. + IOStatisticsContext ioStatisticsContextAfter = + getCurrentIOStatisticsContext(); + //Verify the context ID after setting to null is different than the previous + // one. + Assertions.assertThat(ioStatisticsContextBefore.getID()) + .describedAs("A new IOStaticsContext should be set after setting the " + + "current to null") + .isNotEqualTo(ioStatisticsContextAfter.getID()); + } + + /** + * Assert bytes written by the statistics context. + * + * @param context statistics context. + * @param bytes expected bytes. + */ + private void assertContextBytesWrite(IOStatisticsContext context, + int bytes) { + verifyStatisticCounterValue( + context.getIOStatistics(), + STREAM_WRITE_BYTES, + bytes); + } + + /** + * Assert bytes read by the statistics context. + * + * @param context statistics context. + * @param readBytes expected bytes. + */ + private void assertContextBytesRead(IOStatisticsContext context, + int readBytes) { + verifyStatisticCounterValue( + context.getIOStatistics(), + STREAM_READ_BYTES, + readBytes); + } + + /** + * Assert fixed bytes wrote and read for a particular thread ID. + * + * @param testThreadId thread ID. + * @param expectedBytesWrittenAndRead expected bytes. + */ + private void assertThreadStatisticsForThread(long testThreadId, + int expectedBytesWrittenAndRead) { + LOG.info("Thread ID to be asserted: {}", testThreadId); + IOStatisticsContext ioStatisticsContext = + getThreadSpecificIOStatisticsContext(testThreadId); + Assertions.assertThat(ioStatisticsContext) + .describedAs("IOStatisticsContext for %d", testThreadId) + .isNotNull(); + + + IOStatistics ioStatistics = ioStatisticsContext.snapshot(); + + + assertThatStatisticCounter(ioStatistics, + STREAM_WRITE_BYTES) + .describedAs("Bytes written are not as expected for thread : %s", + testThreadId) + .isEqualTo(expectedBytesWrittenAndRead); + + assertThatStatisticCounter(ioStatistics, + STREAM_READ_BYTES) + .describedAs("Bytes read are not as expected for thread : %s", + testThreadId) + .isEqualTo(expectedBytesWrittenAndRead); + } + + @Test + public void testListingStatisticsContext() throws Throwable { + describe("verify the list operations update on close()"); + + S3AFileSystem fs = getFileSystem(); + Path path = methodPath(); + fs.mkdirs(methodPath()); + + // after all setup, get the reset context + IOStatisticsContext context = + getAndResetThreadStatisticsContext(); + IOStatistics ioStatistics = context.getIOStatistics(); + + fs.listStatus(path); + verifyStatisticCounterValue(ioStatistics, + StoreStatisticNames.OBJECT_LIST_REQUEST, + 1); + + context.reset(); + foreach(fs.listStatusIterator(path), i -> {}); + verifyStatisticCounterValue(ioStatistics, + StoreStatisticNames.OBJECT_LIST_REQUEST, + 1); + + context.reset(); + foreach(fs.listLocatedStatus(path), i -> {}); + verifyStatisticCounterValue(ioStatistics, + StoreStatisticNames.OBJECT_LIST_REQUEST, + 1); + + context.reset(); + foreach(fs.listFiles(path, true), i -> {}); + verifyStatisticCounterValue(ioStatistics, + StoreStatisticNames.OBJECT_LIST_REQUEST, + 1); + } + + @Test + public void testListingThroughTaskPool() throws Throwable { + describe("verify the list operations are updated through taskpool"); + + S3AFileSystem fs = getFileSystem(); + Path path = methodPath(); + fs.mkdirs(methodPath()); + + // after all setup, get the reset context + IOStatisticsContext context = + getAndResetThreadStatisticsContext(); + IOStatistics ioStatistics = context.getIOStatistics(); + + CloseableTaskPoolSubmitter submitter = + new CloseableTaskPoolSubmitter(executor); + TaskPool.foreach(fs.listStatusIterator(path)) + .executeWith(submitter) + .run(i -> {}); + + verifyStatisticCounterValue(ioStatistics, + StoreStatisticNames.OBJECT_LIST_REQUEST, + 1); + + } + + /** + * Simulating doing some work in a separate thread. + * If constructed with an IOStatisticsContext then + * that context is switched to before performing the IO. + */ + private class TestWorkerThread extends Thread implements Runnable { + private final Path workerThreadPath; + + private final IOStatisticsContext ioStatisticsContext; + + /** + * create. + * @param workerThreadPath thread path. + * @param ioStatisticsContext optional statistics context * + */ + TestWorkerThread( + final Path workerThreadPath, + final IOStatisticsContext ioStatisticsContext) { + this.workerThreadPath = workerThreadPath; + this.ioStatisticsContext = ioStatisticsContext; + } + + @Override + public void run() { + // Setting the worker thread's name. + Thread.currentThread().setName("worker thread"); + S3AFileSystem fs = getFileSystem(); + byte[] data = new byte[BYTES_SMALL]; + + // maybe switch context + if (ioStatisticsContext != null) { + IOStatisticsContext.setThreadIOStatisticsContext(ioStatisticsContext); + } + // Storing context in a field to not lose the reference in a GC. + IOStatisticsContext ioStatisticsContextWorkerThread = + getCurrentIOStatisticsContext(); + + // Write in the worker thread. + try (FSDataOutputStream out = fs.create(workerThreadPath)) { + out.write(data); + } catch (IOException e) { + throw new UncheckedIOException("Failure while writing", e); + } + + //Read in the worker thread. + try (FSDataInputStream in = fs.open(workerThreadPath)) { + in.read(data); + } catch (IOException e) { + throw new UncheckedIOException("Failure while reading", e); + } + } + } +} diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3APrefetchingInputStream.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3APrefetchingInputStream.java new file mode 100644 index 0000000000000..24f74b3a0212e --- /dev/null +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3APrefetchingInputStream.java @@ -0,0 +1,198 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.s3a; + +import java.net.URI; + +import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FSDataInputStream; +import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.contract.ContractTestUtils; +import org.apache.hadoop.fs.s3a.performance.AbstractS3ACostTest; +import org.apache.hadoop.fs.statistics.IOStatistics; +import org.apache.hadoop.fs.statistics.StoreStatisticNames; +import org.apache.hadoop.fs.statistics.StreamStatisticNames; + +import static org.apache.hadoop.fs.s3a.Constants.PREFETCH_BLOCK_DEFAULT_SIZE; +import static org.apache.hadoop.fs.s3a.Constants.PREFETCH_BLOCK_SIZE_KEY; +import static org.apache.hadoop.fs.s3a.Constants.PREFETCH_ENABLED_KEY; +import static org.apache.hadoop.fs.statistics.IOStatisticAssertions.assertThatStatisticMaximum; +import static org.apache.hadoop.fs.statistics.IOStatisticAssertions.verifyStatisticCounterValue; +import static org.apache.hadoop.fs.statistics.IOStatisticAssertions.verifyStatisticGaugeValue; +import static org.apache.hadoop.fs.statistics.StoreStatisticNames.SUFFIX_MAX; +import static org.apache.hadoop.io.IOUtils.cleanupWithLogger; + +/** + * Test the prefetching input stream, validates that the underlying S3ACachingInputStream and + * S3AInMemoryInputStream are working as expected. + */ +public class ITestS3APrefetchingInputStream extends AbstractS3ACostTest { + + public ITestS3APrefetchingInputStream() { + super(true); + } + + private static final Logger LOG = + LoggerFactory.getLogger(ITestS3APrefetchingInputStream.class); + + private static final int S_1K = 1024; + private static final int S_1M = S_1K * S_1K; + // Path for file which should have length > block size so S3ACachingInputStream is used + private Path largeFile; + private FileSystem largeFileFS; + private int numBlocks; + private int blockSize; + private long largeFileSize; + // Size should be < block size so S3AInMemoryInputStream is used + private static final int SMALL_FILE_SIZE = S_1K * 16; + + + @Override + public Configuration createConfiguration() { + Configuration conf = super.createConfiguration(); + S3ATestUtils.removeBaseAndBucketOverrides(conf, PREFETCH_ENABLED_KEY); + conf.setBoolean(PREFETCH_ENABLED_KEY, true); + return conf; + } + + @Override + public void teardown() throws Exception { + super.teardown(); + cleanupWithLogger(LOG, largeFileFS); + largeFileFS = null; + } + + private void openFS() throws Exception { + Configuration conf = getConfiguration(); + + largeFile = new Path(DEFAULT_CSVTEST_FILE); + blockSize = conf.getInt(PREFETCH_BLOCK_SIZE_KEY, PREFETCH_BLOCK_DEFAULT_SIZE); + largeFileFS = new S3AFileSystem(); + largeFileFS.initialize(new URI(DEFAULT_CSVTEST_FILE), getConfiguration()); + FileStatus fileStatus = largeFileFS.getFileStatus(largeFile); + largeFileSize = fileStatus.getLen(); + numBlocks = calculateNumBlocks(largeFileSize, blockSize); + } + + private static int calculateNumBlocks(long largeFileSize, int blockSize) { + if (largeFileSize == 0) { + return 0; + } else { + return ((int) (largeFileSize / blockSize)) + (largeFileSize % blockSize > 0 ? 1 : 0); + } + } + + @Test + public void testReadLargeFileFully() throws Throwable { + describe("read a large file fully, uses S3ACachingInputStream"); + IOStatistics ioStats; + openFS(); + + try (FSDataInputStream in = largeFileFS.open(largeFile)) { + ioStats = in.getIOStatistics(); + + byte[] buffer = new byte[S_1M * 10]; + long bytesRead = 0; + + while (bytesRead < largeFileSize) { + in.readFully(buffer, 0, (int) Math.min(buffer.length, largeFileSize - bytesRead)); + bytesRead += buffer.length; + // Blocks are fully read, no blocks should be cached + verifyStatisticGaugeValue(ioStats, StreamStatisticNames.STREAM_READ_BLOCKS_IN_FILE_CACHE, + 0); + } + + // Assert that first block is read synchronously, following blocks are prefetched + verifyStatisticCounterValue(ioStats, StreamStatisticNames.STREAM_READ_PREFETCH_OPERATIONS, + numBlocks - 1); + verifyStatisticCounterValue(ioStats, StoreStatisticNames.ACTION_HTTP_GET_REQUEST, numBlocks); + verifyStatisticCounterValue(ioStats, StreamStatisticNames.STREAM_READ_OPENED, numBlocks); + } + // Verify that once stream is closed, all memory is freed + verifyStatisticGaugeValue(ioStats, StreamStatisticNames.STREAM_READ_ACTIVE_MEMORY_IN_USE, 0); + assertThatStatisticMaximum(ioStats, + StoreStatisticNames.ACTION_EXECUTOR_ACQUIRED + SUFFIX_MAX).isGreaterThan(0); + } + + @Test + public void testRandomReadLargeFile() throws Throwable { + describe("random read on a large file, uses S3ACachingInputStream"); + IOStatistics ioStats; + openFS(); + + try (FSDataInputStream in = largeFileFS.open(largeFile)) { + ioStats = in.getIOStatistics(); + + byte[] buffer = new byte[blockSize]; + + // Don't read the block completely so it gets cached on seek + in.read(buffer, 0, blockSize - S_1K * 10); + in.seek(blockSize + S_1K * 10); + // Backwards seek, will use cached block + in.seek(S_1K * 5); + in.read(); + + verifyStatisticCounterValue(ioStats, StoreStatisticNames.ACTION_HTTP_GET_REQUEST, 2); + verifyStatisticCounterValue(ioStats, StreamStatisticNames.STREAM_READ_OPENED, 2); + verifyStatisticCounterValue(ioStats, StreamStatisticNames.STREAM_READ_PREFETCH_OPERATIONS, 1); + // block 0 is cached when we seek to block 1, block 1 is cached as it is being prefetched + // when we seek out of block 0, see cancelPrefetches() + verifyStatisticGaugeValue(ioStats, StreamStatisticNames.STREAM_READ_BLOCKS_IN_FILE_CACHE, 2); + } + verifyStatisticGaugeValue(ioStats, StreamStatisticNames.STREAM_READ_BLOCKS_IN_FILE_CACHE, 0); + verifyStatisticGaugeValue(ioStats, StreamStatisticNames.STREAM_READ_ACTIVE_MEMORY_IN_USE, 0); + assertThatStatisticMaximum(ioStats, + StoreStatisticNames.ACTION_EXECUTOR_ACQUIRED + SUFFIX_MAX).isGreaterThan(0); + } + + @Test + public void testRandomReadSmallFile() throws Throwable { + describe("random read on a small file, uses S3AInMemoryInputStream"); + + byte[] data = ContractTestUtils.dataset(SMALL_FILE_SIZE, 'a', 26); + Path smallFile = path("randomReadSmallFile"); + ContractTestUtils.writeDataset(getFileSystem(), smallFile, data, data.length, 16, true); + + try (FSDataInputStream in = getFileSystem().open(smallFile)) { + IOStatistics ioStats = in.getIOStatistics(); + + byte[] buffer = new byte[SMALL_FILE_SIZE]; + + in.read(buffer, 0, S_1K * 4); + in.seek(S_1K * 12); + in.read(buffer, 0, S_1K * 4); + + verifyStatisticCounterValue(ioStats, StoreStatisticNames.ACTION_HTTP_GET_REQUEST, 1); + verifyStatisticCounterValue(ioStats, StreamStatisticNames.STREAM_READ_OPENED, 1); + verifyStatisticCounterValue(ioStats, StreamStatisticNames.STREAM_READ_PREFETCH_OPERATIONS, 0); + // The buffer pool is not used + verifyStatisticGaugeValue(ioStats, StreamStatisticNames.STREAM_READ_ACTIVE_MEMORY_IN_USE, 0); + // no prefetch ops, so no action_executor_acquired + assertThatStatisticMaximum(ioStats, + StoreStatisticNames.ACTION_EXECUTOR_ACQUIRED + SUFFIX_MAX).isEqualTo(-1); + } + } + +} diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3ARequesterPays.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3ARequesterPays.java index 9a818d037e4c0..d3925d35a99d3 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3ARequesterPays.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3ARequesterPays.java @@ -31,6 +31,8 @@ import org.apache.hadoop.fs.statistics.StreamStatisticNames; import static org.apache.hadoop.fs.s3a.Constants.ALLOW_REQUESTER_PAYS; +import static org.apache.hadoop.fs.s3a.Constants.PREFETCH_ENABLED_DEFAULT; +import static org.apache.hadoop.fs.s3a.Constants.PREFETCH_ENABLED_KEY; import static org.apache.hadoop.fs.s3a.Constants.S3A_BUCKET_PROBE; import static org.apache.hadoop.test.LambdaTestUtils.intercept; @@ -78,11 +80,16 @@ public void testRequesterPaysOptionSuccess() throws Throwable { inputStream.seek(0); inputStream.readByte(); - // Verify > 1 call was made, so we're sure it is correctly configured for each request - IOStatisticAssertions - .assertThatStatisticCounter(inputStream.getIOStatistics(), - StreamStatisticNames.STREAM_READ_OPENED) - .isGreaterThan(1); + if (conf.getBoolean(PREFETCH_ENABLED_KEY, PREFETCH_ENABLED_DEFAULT)) { + // For S3APrefetchingInputStream, verify a call was made + IOStatisticAssertions.assertThatStatisticCounter(inputStream.getIOStatistics(), + StreamStatisticNames.STREAM_READ_OPENED).isEqualTo(1); + } else { + // For S3AInputStream, verify > 1 call was made, + // so we're sure it is correctly configured for each request + IOStatisticAssertions.assertThatStatisticCounter(inputStream.getIOStatistics(), + StreamStatisticNames.STREAM_READ_OPENED).isGreaterThan(1); + } // Check list calls work without error fs.listFiles(requesterPaysPath.getParent(), false); diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AStorageClass.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AStorageClass.java index e141ef5aa32f3..7c56f8d2ea050 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AStorageClass.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AStorageClass.java @@ -19,10 +19,14 @@ package org.apache.hadoop.fs.s3a; import java.nio.file.AccessDeniedException; +import java.util.Arrays; +import java.util.Collection; import java.util.Map; import org.assertj.core.api.Assertions; import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; @@ -30,6 +34,10 @@ import org.apache.hadoop.fs.contract.ContractTestUtils; import org.apache.hadoop.fs.contract.s3a.S3AContract; +import static org.apache.hadoop.fs.s3a.Constants.FAST_UPLOAD_BUFFER; +import static org.apache.hadoop.fs.s3a.Constants.FAST_UPLOAD_BUFFER_ARRAY; +import static org.apache.hadoop.fs.s3a.Constants.FAST_UPLOAD_BUFFER_DISK; +import static org.apache.hadoop.fs.s3a.Constants.FAST_UPLOAD_BYTEBUFFER; import static org.apache.hadoop.fs.s3a.Constants.STORAGE_CLASS; import static org.apache.hadoop.fs.s3a.Constants.STORAGE_CLASS_GLACIER; import static org.apache.hadoop.fs.s3a.Constants.STORAGE_CLASS_REDUCED_REDUNDANCY; @@ -43,13 +51,33 @@ /** * Tests of storage class. */ +@RunWith(Parameterized.class) public class ITestS3AStorageClass extends AbstractS3ATestBase { + /** + * HADOOP-18339. Parameterized the test for different fast upload buffer types + * to ensure the storage class configuration works with all of them. + */ + @Parameterized.Parameters(name = "fast-upload-buffer-{0}") + public static Collection params() { + return Arrays.asList(new Object[][]{ + {FAST_UPLOAD_BUFFER_DISK}, + {FAST_UPLOAD_BUFFER_ARRAY} + }); + } + + private final String fastUploadBufferType; + + public ITestS3AStorageClass(String fastUploadBufferType) { + this.fastUploadBufferType = fastUploadBufferType; + } + @Override protected Configuration createConfiguration() { Configuration conf = super.createConfiguration(); disableFilesystemCaching(conf); - removeBaseAndBucketOverrides(conf, STORAGE_CLASS); + removeBaseAndBucketOverrides(conf, STORAGE_CLASS, FAST_UPLOAD_BUFFER); + conf.set(FAST_UPLOAD_BUFFER, fastUploadBufferType); return conf; } diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3ATemporaryCredentials.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3ATemporaryCredentials.java index 112d0fcb50275..0778662542d88 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3ATemporaryCredentials.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3ATemporaryCredentials.java @@ -71,6 +71,7 @@ public class ITestS3ATemporaryCredentials extends AbstractS3ATestBase { private static final Logger LOG = LoggerFactory.getLogger(ITestS3ATemporaryCredentials.class); + @SuppressWarnings("deprecation") private static final String TEMPORARY_AWS_CREDENTIALS = TemporaryAWSCredentialsProvider.NAME; @@ -125,13 +126,14 @@ public void testSTS() throws IOException { credentials, getStsEndpoint(conf), getStsRegion(conf)); - STSClientFactory.STSClient clientConnection = - STSClientFactory.createClientConnection( - builder.build(), - new Invoker(new S3ARetryPolicy(conf), Invoker.LOG_EVENT)); - Credentials sessionCreds = clientConnection - .requestSessionCredentials(TEST_SESSION_TOKEN_DURATION_SECONDS, - TimeUnit.SECONDS); + Credentials sessionCreds; + try (STSClientFactory.STSClient clientConnection = + STSClientFactory.createClientConnection(builder.build(), + new Invoker(new S3ARetryPolicy(conf), Invoker.LOG_EVENT))) { + sessionCreds = clientConnection + .requestSessionCredentials( + TEST_SESSION_TOKEN_DURATION_SECONDS, TimeUnit.SECONDS); + } // clone configuration so changes here do not affect the base FS. Configuration conf2 = new Configuration(conf); @@ -174,6 +176,7 @@ protected String getStsRegion(final Configuration conf) { } @Test + @SuppressWarnings("deprecation") public void testTemporaryCredentialValidation() throws Throwable { Configuration conf = new Configuration(); conf.set(ACCESS_KEY, "accesskey"); @@ -357,6 +360,7 @@ public void testSessionCredentialsEndpointNoRegion() throws Throwable { * @return the caught exception. * @throws Exception any unexpected exception. */ + @SuppressWarnings("deprecation") public E expectedSessionRequestFailure( final Class clazz, final String endpoint, @@ -379,11 +383,12 @@ public E expectedSessionRequestFailure( Invoker invoker = new Invoker(new S3ARetryPolicy(conf), LOG_AT_ERROR); - STSClientFactory.STSClient stsClient - = STSClientFactory.createClientConnection(tokenService, - invoker); - - return stsClient.requestSessionCredentials(30, TimeUnit.MINUTES); + try (STSClientFactory.STSClient stsClient = + STSClientFactory.createClientConnection( + tokenService, invoker)) { + return stsClient.requestSessionCredentials( + 30, TimeUnit.MINUTES); + } }); } } @@ -413,6 +418,7 @@ public void testTemporaryCredentialValidationOnLoad() throws Throwable { return sc.toString(); }); } + @Test public void testEmptyTemporaryCredentialValidation() throws Throwable { Configuration conf = new Configuration(); diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AUnbuffer.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AUnbuffer.java index 3d7ee0882efa4..3a2d1b1b09a49 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AUnbuffer.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AUnbuffer.java @@ -20,6 +20,7 @@ import org.apache.hadoop.fs.FSDataInputStream; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.StreamCapabilities; import org.apache.hadoop.fs.contract.ContractTestUtils; import org.apache.hadoop.fs.s3a.statistics.S3AInputStreamStatistics; import org.apache.hadoop.fs.statistics.IOStatistics; @@ -33,6 +34,7 @@ import java.io.IOException; +import static org.apache.hadoop.fs.contract.ContractTestUtils.skip; import static org.apache.hadoop.fs.s3a.Statistic.STREAM_READ_BYTES; import static org.apache.hadoop.fs.s3a.Statistic.STREAM_READ_BYTES_READ_CLOSE; import static org.apache.hadoop.fs.s3a.Statistic.STREAM_READ_TOTAL_BYTES; @@ -72,6 +74,7 @@ public void testUnbuffer() throws IOException { IOStatisticsSnapshot iostats = new IOStatisticsSnapshot(); // Open file, read half the data, and then call unbuffer try (FSDataInputStream inputStream = getFileSystem().open(dest)) { + skipIfCannotUnbuffer(inputStream); assertTrue(inputStream.getWrappedStream() instanceof S3AInputStream); int bytesToRead = 8; readAndAssertBytesRead(inputStream, bytesToRead); @@ -138,6 +141,7 @@ public void testUnbufferStreamStatistics() throws IOException { Object streamStatsStr; try { inputStream = fs.open(dest); + skipIfCannotUnbuffer(inputStream); streamStatsStr = demandStringifyIOStatisticsSource(inputStream); LOG.info("initial stream statistics {}", streamStatsStr); @@ -192,6 +196,12 @@ private boolean isObjectStreamOpen(FSDataInputStream inputStream) { return ((S3AInputStream) inputStream.getWrappedStream()).isObjectStreamOpen(); } + private void skipIfCannotUnbuffer(FSDataInputStream inputStream) { + if (!inputStream.hasCapability(StreamCapabilities.UNBUFFER)) { + skip("input stream does not support unbuffer"); + } + } + /** * Read the specified number of bytes from the given * {@link FSDataInputStream} and assert that diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/MockS3AFileSystem.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/MockS3AFileSystem.java index 2eb35daab4c2b..a859cd534bb02 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/MockS3AFileSystem.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/MockS3AFileSystem.java @@ -47,6 +47,7 @@ import org.apache.hadoop.fs.s3a.impl.StubContextAccessor; import org.apache.hadoop.fs.s3a.statistics.CommitterStatistics; import org.apache.hadoop.fs.s3a.statistics.impl.EmptyS3AStatisticsContext; +import org.apache.hadoop.fs.s3a.test.MinimalWriteOperationHelperCallbacks; import org.apache.hadoop.fs.statistics.DurationTrackerFactory; import org.apache.hadoop.util.Progressable; @@ -176,7 +177,8 @@ public void initialize(URI name, Configuration originalConf) conf, new EmptyS3AStatisticsContext(), noopAuditor(conf), - AuditTestSupport.NOOP_SPAN); + AuditTestSupport.NOOP_SPAN, + new MinimalWriteOperationHelperCallbacks()); } @Override diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/MockS3ClientFactory.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/MockS3ClientFactory.java index bd121ba2728eb..3240309aef971 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/MockS3ClientFactory.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/MockS3ClientFactory.java @@ -31,6 +31,7 @@ * An {@link S3ClientFactory} that returns Mockito mocks of the {@link AmazonS3} * interface suitable for unit testing. */ +@SuppressWarnings("deprecation") public class MockS3ClientFactory implements S3ClientFactory { @Override diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/S3ATestUtils.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/S3ATestUtils.java index 48cb52c5ac29c..08ef7edf43b0a 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/S3ATestUtils.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/S3ATestUtils.java @@ -40,7 +40,9 @@ import org.apache.hadoop.fs.s3a.impl.StatusProbeEnum; import org.apache.hadoop.fs.s3a.impl.StoreContext; import org.apache.hadoop.fs.s3a.impl.StoreContextBuilder; +import org.apache.hadoop.fs.s3a.prefetch.S3APrefetchingInputStream; import org.apache.hadoop.fs.s3a.statistics.BlockOutputStreamStatistics; +import org.apache.hadoop.fs.s3a.statistics.S3AInputStreamStatistics; import org.apache.hadoop.fs.s3a.statistics.impl.EmptyS3AStatisticsContext; import org.apache.hadoop.fs.s3native.S3xLoginHelper; import org.apache.hadoop.io.DataInputBuffer; @@ -69,6 +71,7 @@ import java.io.Closeable; import java.io.File; import java.io.IOException; +import java.io.InputStream; import java.net.URI; import java.net.URISyntaxException; import java.text.DateFormat; @@ -605,6 +608,7 @@ public static void unsetHadoopCredentialProviders(final Configuration conf) { * @return a set of credentials * @throws IOException on a failure */ + @SuppressWarnings("deprecation") public static AWSCredentialsProvider buildAwsCredentialsProvider( final Configuration conf) throws IOException { @@ -1457,4 +1461,49 @@ public static void skipIfEncryptionNotSet(Configuration configuration, + " in " + secrets); } } + + /** + * Get the input stream statistics of an input stream. + * Raises an exception if the inner stream is not an S3A input stream + * or prefetching input stream + * @param in wrapper + * @return the statistics for the inner stream + */ + public static S3AInputStreamStatistics getInputStreamStatistics( + FSDataInputStream in) { + + InputStream inner = in.getWrappedStream(); + if (inner instanceof S3AInputStream) { + return ((S3AInputStream) inner).getS3AStreamStatistics(); + } else if (inner instanceof S3APrefetchingInputStream) { + return ((S3APrefetchingInputStream) inner).getS3AStreamStatistics(); + } else { + throw new AssertionError("Not an S3AInputStream or S3APrefetchingInputStream: " + inner); + } + } + + /** + * Get the inner stream of an input stream. + * Raises an exception if the inner stream is not an S3A input stream + * @param in wrapper + * @return the inner stream + * @throws AssertionError if the inner stream is of the wrong type + */ + public static S3AInputStream getS3AInputStream( + FSDataInputStream in) { + InputStream inner = in.getWrappedStream(); + if (inner instanceof S3AInputStream) { + return (S3AInputStream) inner; + } else { + throw new AssertionError("Not an S3AInputStream: " + inner); + } + } + + /** + * Disable Prefetching streams from S3AFileSystem in tests. + * @param conf Configuration to remove the prefetch property from. + */ + public static void disablePrefetching(Configuration conf) { + removeBaseAndBucketOverrides(conf, PREFETCH_ENABLED_KEY); + } } diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AAWSCredentialsProvider.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AAWSCredentialsProvider.java index f273e68371e58..6030005d10fb3 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AAWSCredentialsProvider.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AAWSCredentialsProvider.java @@ -95,6 +95,7 @@ public void testProviderFailureError() throws Exception { } @Test + @SuppressWarnings("deprecation") public void testInstantiationChain() throws Throwable { Configuration conf = new Configuration(false); conf.set(AWS_CREDENTIALS_PROVIDER, @@ -114,6 +115,7 @@ public void testInstantiationChain() throws Throwable { } @Test + @SuppressWarnings("deprecation") public void testDefaultChain() throws Exception { URI uri1 = new URI("s3a://bucket1"), uri2 = new URI("s3a://bucket2"); Configuration conf = new Configuration(false); @@ -138,6 +140,7 @@ public void testDefaultChainNoURI() throws Exception { } @Test + @SuppressWarnings("deprecation") public void testConfiguredChain() throws Exception { URI uri1 = new URI("s3a://bucket1"), uri2 = new URI("s3a://bucket2"); List> expectedClasses = @@ -156,6 +159,7 @@ public void testConfiguredChain() throws Exception { } @Test + @SuppressWarnings("deprecation") public void testConfiguredChainUsesSharedInstanceProfile() throws Exception { URI uri1 = new URI("s3a://bucket1"), uri2 = new URI("s3a://bucket2"); Configuration conf = new Configuration(false); @@ -368,6 +372,7 @@ private static void assertCredentialProviders( * @see S3ATestUtils#authenticationContains(Configuration, String). */ @Test + @SuppressWarnings("deprecation") public void testAuthenticationContainsProbes() { Configuration conf = new Configuration(false); assertFalse("found AssumedRoleCredentialProvider", diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3ABlockOutputStream.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3ABlockOutputStream.java index 08f4f8bc9df5c..ffa2c81e58adf 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3ABlockOutputStream.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3ABlockOutputStream.java @@ -24,6 +24,8 @@ import org.apache.hadoop.fs.s3a.commit.PutTracker; import org.apache.hadoop.fs.s3a.impl.PutObjectOptions; import org.apache.hadoop.fs.s3a.statistics.impl.EmptyS3AStatisticsContext; +import org.apache.hadoop.fs.s3a.test.MinimalWriteOperationHelperCallbacks; +import org.apache.hadoop.fs.statistics.IOStatisticsContext; import org.apache.hadoop.util.Progressable; import org.junit.Before; import org.junit.Test; @@ -67,7 +69,11 @@ private S3ABlockOutputStream.BlockOutputStreamBuilder mockS3ABuilder() { .withProgress(progressable) .withPutTracker(putTracker) .withWriteOperations(oHelper) - .withPutOptions(PutObjectOptions.keepingDirs()); + .withPutOptions(PutObjectOptions.keepingDirs()) + .withIOStatisticsAggregator( + IOStatisticsContext.getCurrentIOStatisticsContext() + .getAggregator()); + return builder; } @@ -97,7 +103,8 @@ public void testWriteOperationHelperPartLimits() throws Throwable { conf, new EmptyS3AStatisticsContext(), noopAuditor(conf), - AuditTestSupport.NOOP_SPAN); + AuditTestSupport.NOOP_SPAN, + new MinimalWriteOperationHelperCallbacks()); ByteArrayInputStream inputStream = new ByteArrayInputStream( "a".getBytes()); // first one works diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3ADeleteOnExit.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3ADeleteOnExit.java new file mode 100644 index 0000000000000..62a99d7209263 --- /dev/null +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3ADeleteOnExit.java @@ -0,0 +1,98 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.s3a; + +import static org.apache.hadoop.fs.s3a.Constants.FS_S3A; +import static org.junit.Assert.assertEquals; +import static org.mockito.ArgumentMatchers.argThat; +import static org.mockito.Mockito.when; + +import java.io.IOException; +import java.net.URI; +import java.util.Date; + +import com.amazonaws.services.s3.AmazonS3; +import com.amazonaws.services.s3.model.GetObjectMetadataRequest; +import com.amazonaws.services.s3.model.ObjectMetadata; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.Path; + +import org.junit.Test; +import org.mockito.ArgumentMatcher; + +/** + * deleteOnExit test for S3A. + */ +public class TestS3ADeleteOnExit extends AbstractS3AMockTest { + + static class TestS3AFileSystem extends S3AFileSystem { + private int deleteOnDnExitCount; + + public int getDeleteOnDnExitCount() { + return deleteOnDnExitCount; + } + + @Override + public boolean deleteOnExit(Path f) throws IOException { + deleteOnDnExitCount++; + return super.deleteOnExit(f); + } + + // This is specifically designed for deleteOnExit processing. + // In this specific case, deleteWithoutCloseCheck() will only be called in the path of + // processDeleteOnExit. + @Override + protected boolean deleteWithoutCloseCheck(Path f, boolean recursive) throws IOException { + boolean result = super.deleteWithoutCloseCheck(f, recursive); + deleteOnDnExitCount--; + return result; + } + } + + @Test + public void testDeleteOnExit() throws Exception { + Configuration conf = createConfiguration(); + TestS3AFileSystem testFs = new TestS3AFileSystem(); + URI uri = URI.create(FS_S3A + "://" + BUCKET); + // unset S3CSE property from config to avoid pathIOE. + conf.unset(Constants.S3_ENCRYPTION_ALGORITHM); + testFs.initialize(uri, conf); + AmazonS3 testS3 = testFs.getAmazonS3ClientForTesting("mocking"); + + Path path = new Path("/file"); + String key = path.toUri().getPath().substring(1); + ObjectMetadata meta = new ObjectMetadata(); + meta.setContentLength(1L); + meta.setLastModified(new Date(2L)); + when(testS3.getObjectMetadata(argThat(correctGetMetadataRequest(BUCKET, key)))) + .thenReturn(meta); + + testFs.deleteOnExit(path); + testFs.close(); + assertEquals(0, testFs.getDeleteOnDnExitCount()); + } + + private ArgumentMatcher correctGetMetadataRequest( + String bucket, String key) { + return request -> request != null + && request.getBucketName().equals(bucket) + && request.getKey().equals(key); + } +} diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AUnbuffer.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AUnbuffer.java index 204f1aa09394a..0e105c25c3a45 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AUnbuffer.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AUnbuffer.java @@ -34,8 +34,8 @@ import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyInt; +import static org.mockito.Mockito.atLeast; import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; @@ -74,6 +74,6 @@ public void testUnbuffer() throws IOException { stream.unbuffer(); // Verify that unbuffer closed the object stream - verify(objectStream, times(1)).close(); + verify(objectStream, atLeast(1)).close(); } } diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/ITestAssumeRole.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/ITestAssumeRole.java index 51eac7e8cc349..9fb09b4cede52 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/ITestAssumeRole.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/ITestAssumeRole.java @@ -140,6 +140,7 @@ private E expectFileSystemCreateFailure( } @Test + @SuppressWarnings("deprecation") public void testCreateCredentialProvider() throws IOException { describe("Create the credential provider"); @@ -153,6 +154,7 @@ public void testCreateCredentialProvider() throws IOException { } @Test + @SuppressWarnings("deprecation") public void testCreateCredentialProviderNoURI() throws IOException { describe("Create the credential provider"); @@ -170,6 +172,7 @@ public void testCreateCredentialProviderNoURI() throws IOException { * @return a configuration set to use to the role ARN. * @throws JsonProcessingException problems working with JSON policies. */ + @SuppressWarnings("deprecation") protected Configuration createValidRoleConf() throws JsonProcessingException { String roleARN = getAssumedRoleARN(); @@ -183,6 +186,7 @@ protected Configuration createValidRoleConf() throws JsonProcessingException { } @Test + @SuppressWarnings("deprecation") public void testAssumedInvalidRole() throws Throwable { Configuration conf = new Configuration(); conf.set(ASSUMED_ROLE_ARN, ROLE_ARN_EXAMPLE); @@ -200,6 +204,7 @@ public void testAssumeRoleFSBadARN() throws Exception { } @Test + @SuppressWarnings("deprecation") public void testAssumeRoleNoARN() throws Exception { describe("Attemnpt to create the FS with no ARN"); Configuration conf = createAssumedRoleConfig(); @@ -232,6 +237,7 @@ public void testAssumeRoleFSBadPolicy2() throws Exception { } @Test + @SuppressWarnings("deprecation") public void testAssumeRoleCannotAuthAssumedRole() throws Exception { describe("Assert that you can't use assumed roles to auth assumed roles"); @@ -245,13 +251,13 @@ public void testAssumeRoleCannotAuthAssumedRole() throws Exception { } @Test + @SuppressWarnings("deprecation") public void testAssumeRoleBadInnerAuth() throws Exception { describe("Try to authenticate with a keypair with spaces"); Configuration conf = createAssumedRoleConfig(); unsetHadoopCredentialProviders(conf); - conf.set(ASSUMED_ROLE_CREDENTIALS_PROVIDER, - SimpleAWSCredentialsProvider.NAME); + conf.set(ASSUMED_ROLE_CREDENTIALS_PROVIDER, SimpleAWSCredentialsProvider.NAME); conf.set(ACCESS_KEY, "not valid"); conf.set(SECRET_KEY, "not secret"); expectFileSystemCreateFailure(conf, @@ -261,13 +267,13 @@ public void testAssumeRoleBadInnerAuth() throws Exception { } @Test + @SuppressWarnings("deprecation") public void testAssumeRoleBadInnerAuth2() throws Exception { describe("Try to authenticate with an invalid keypair"); Configuration conf = createAssumedRoleConfig(); unsetHadoopCredentialProviders(conf); - conf.set(ASSUMED_ROLE_CREDENTIALS_PROVIDER, - SimpleAWSCredentialsProvider.NAME); + conf.set(ASSUMED_ROLE_CREDENTIALS_PROVIDER, SimpleAWSCredentialsProvider.NAME); conf.set(ACCESS_KEY, "notvalid"); conf.set(SECRET_KEY, "notsecret"); expectFileSystemCreateFailure(conf, @@ -345,6 +351,7 @@ private Configuration createAssumedRoleConfig(String roleARN) { } @Test + @SuppressWarnings("deprecation") public void testAssumeRoleUndefined() throws Throwable { describe("Verify that you cannot instantiate the" + " AssumedRoleCredentialProvider without a role ARN"); @@ -356,6 +363,7 @@ public void testAssumeRoleUndefined() throws Throwable { } @Test + @SuppressWarnings("deprecation") public void testAssumedIllegalDuration() throws Throwable { describe("Expect the constructor to fail if the session is to short"); Configuration conf = new Configuration(); @@ -529,6 +537,7 @@ public Path methodPath() throws IOException { * don't break. */ @Test + @SuppressWarnings("deprecation") public void testAssumedRoleRetryHandler() throws Throwable { try(AssumedRoleCredentialProvider provider = new AssumedRoleCredentialProvider(getFileSystem().getUri(), diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/ITestCustomSigner.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/ITestCustomSigner.java index cbba326d5ecae..a829d470e7a66 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/ITestCustomSigner.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/ITestCustomSigner.java @@ -20,7 +20,6 @@ import java.io.IOException; import java.security.PrivilegedExceptionAction; -import java.util.Arrays; import java.util.HashMap; import java.util.Map; import java.util.Objects; @@ -228,10 +227,13 @@ private String parseBucketFromHost(String host) { if (service.contains("s3-accesspoint") || service.contains("s3-outposts") || service.contains("s3-object-lambda")) { // If AccessPoint then bucketName is of format `accessPoint-accountId`; - String[] accessPointBits = hostBits[0].split("-"); - int lastElem = accessPointBits.length - 1; - String accountId = accessPointBits[lastElem]; - String accessPointName = String.join("", Arrays.copyOf(accessPointBits, lastElem)); + String[] accessPointBits = bucketName.split("-"); + String accountId = accessPointBits[accessPointBits.length - 1]; + // Extract the access point name from bucket name. eg: if bucket name is + // test-custom-signer-, get the access point name test-custom-signer by removing + // - from the bucket name. + String accessPointName = + bucketName.substring(0, bucketName.length() - (accountId.length() + 1)); Arn arn = Arn.builder() .withAccountId(accountId) .withPartition("aws") diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/RoleTestUtils.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/RoleTestUtils.java index 186887d745bfc..37c2dce4e1d72 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/RoleTestUtils.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/RoleTestUtils.java @@ -146,6 +146,7 @@ public static void assertTouchForbidden(final FileSystem fs, final Path path) * @param roleARN ARN of role * @return the new configuration */ + @SuppressWarnings("deprecation") public static Configuration newAssumedRoleConfig( final Configuration srcConf, final String roleARN) { diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/delegation/ITestSessionDelegationInFileystem.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/delegation/ITestSessionDelegationInFileystem.java index bc223bad457c1..9598ef084fa49 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/delegation/ITestSessionDelegationInFileystem.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/delegation/ITestSessionDelegationInFileystem.java @@ -254,6 +254,7 @@ public void testGetDTfromFileSystem() throws Throwable { } @Test + @SuppressWarnings("deprecation") public void testAddTokensFromFileSystem() throws Throwable { describe("verify FileSystem.addDelegationTokens() collects tokens"); S3AFileSystem fs = getFileSystem(); @@ -576,6 +577,7 @@ public void testDelegationBindingMismatch2() throws Throwable { * @return result of the HEAD * @throws Exception failure */ + @SuppressWarnings("deprecation") protected ObjectMetadata readLandsatMetadata(final S3AFileSystem delegatedFS) throws Exception { AWSCredentialProviderList testingCreds @@ -589,6 +591,7 @@ protected ObjectMetadata readLandsatMetadata(final S3AFileSystem delegatedFS) S3ClientFactory.S3ClientCreationParameters parameters = null; parameters = new S3ClientFactory.S3ClientCreationParameters() .withCredentialSet(testingCreds) + .withPathUri(new URI("s3a://localhost/")) .withEndpoint(DEFAULT_ENDPOINT) .withMetrics(new EmptyS3AStatisticsContext() .newStatisticsFromAwsSdk()) diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/delegation/ITestSessionDelegationTokens.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/delegation/ITestSessionDelegationTokens.java index 7a70ac95fc4d9..fab7ffdbb76f8 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/delegation/ITestSessionDelegationTokens.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/delegation/ITestSessionDelegationTokens.java @@ -249,7 +249,7 @@ public void testCreateWithRenewer() throws Throwable { * @return the retrieved DT. This is only for error reporting. * @throws IOException failure. */ - @SuppressWarnings("OptionalGetWithoutIsPresent") + @SuppressWarnings({"OptionalGetWithoutIsPresent", "deprecation"}) protected AbstractS3ATokenIdentifier verifyCredentialPropagation( final S3AFileSystem fs, final MarshalledCredentials session, @@ -259,8 +259,7 @@ protected AbstractS3ATokenIdentifier verifyCredentialPropagation( // clear any credential paths to ensure they don't get picked up and used // for authentication. unsetHadoopCredentialProviders(conf); - conf.set(DELEGATION_TOKEN_CREDENTIALS_PROVIDER, - TemporaryAWSCredentialsProvider.NAME); + conf.set(DELEGATION_TOKEN_CREDENTIALS_PROVIDER, TemporaryAWSCredentialsProvider.NAME); session.setSecretsInConfiguration(conf); try(S3ADelegationTokens delegationTokens2 = new S3ADelegationTokens()) { delegationTokens2.bindToFileSystem( diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/AbstractCommitITest.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/AbstractCommitITest.java index 9d9000cafb936..9987257527176 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/AbstractCommitITest.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/AbstractCommitITest.java @@ -26,6 +26,7 @@ import java.util.List; import org.assertj.core.api.Assertions; +import org.junit.AfterClass; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -38,6 +39,9 @@ import org.apache.hadoop.fs.s3a.AbstractS3ATestBase; import org.apache.hadoop.fs.s3a.S3AFileSystem; import org.apache.hadoop.fs.s3a.commit.files.SuccessData; +import org.apache.hadoop.fs.statistics.IOStatisticsLogging; +import org.apache.hadoop.fs.statistics.IOStatisticsSnapshot; +import org.apache.hadoop.fs.statistics.IOStatisticsSupport; import org.apache.hadoop.mapreduce.JobContext; import org.apache.hadoop.mapreduce.RecordWriter; import org.apache.hadoop.mapreduce.TaskAttemptContext; @@ -66,6 +70,12 @@ public abstract class AbstractCommitITest extends AbstractS3ATestBase { private static final Logger LOG = LoggerFactory.getLogger(AbstractCommitITest.class); + /** + * Job statistics accrued across all test cases. + */ + private static final IOStatisticsSnapshot JOB_STATISTICS = + IOStatisticsSupport.snapshotIOStatistics(); + /** * Helper class for commit operations and assertions. */ @@ -92,7 +102,8 @@ protected Configuration createConfiguration() { FS_S3A_COMMITTER_NAME, FS_S3A_COMMITTER_STAGING_CONFLICT_MODE, FS_S3A_COMMITTER_STAGING_UNIQUE_FILENAMES, - FAST_UPLOAD_BUFFER); + FAST_UPLOAD_BUFFER, + S3A_COMMITTER_EXPERIMENTAL_COLLECT_IOSTATISTICS); conf.setBoolean(MAGIC_COMMITTER_ENABLED, DEFAULT_MAGIC_COMMITTER_ENABLED); conf.setLong(MIN_MULTIPART_THRESHOLD, MULTIPART_MIN_SIZE); @@ -100,9 +111,15 @@ protected Configuration createConfiguration() { conf.set(FAST_UPLOAD_BUFFER, FAST_UPLOAD_BUFFER_ARRAY); // and bind the report dir conf.set(OPT_SUMMARY_REPORT_DIR, reportDir.toURI().toString()); + conf.setBoolean(S3A_COMMITTER_EXPERIMENTAL_COLLECT_IOSTATISTICS, true); return conf; } + @AfterClass + public static void printStatistics() { + LOG.info("Aggregate job statistics {}\n", + IOStatisticsLogging.ioStatisticsToPrettyString(JOB_STATISTICS)); + } /** * Get the log; can be overridden for test case log. * @return a log. @@ -397,6 +414,8 @@ public static SuccessData validateSuccessFile(final Path outputPath, /** * Load a success file; fail if the file is empty/nonexistent. + * The statistics in {@link #JOB_STATISTICS} are updated with + * the statistics from the success file * @param fs filesystem * @param outputPath directory containing the success file. * @param origin origin of the file @@ -426,6 +445,8 @@ public static SuccessData loadSuccessFile(final FileSystem fs, String body = ContractTestUtils.readUTF8(fs, success, -1); LOG.info("Loading committer success file {}. Actual contents=\n{}", success, body); - return SuccessData.load(fs, success); + SuccessData successData = SuccessData.load(fs, success); + JOB_STATISTICS.aggregate(successData.getIOStatistics()); + return successData; } } diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/staging/TestStagingDirectoryOutputCommitter.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/staging/TestStagingDirectoryOutputCommitter.java index b92605ca25357..439ef9aa44fcb 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/staging/TestStagingDirectoryOutputCommitter.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/staging/TestStagingDirectoryOutputCommitter.java @@ -34,6 +34,7 @@ import org.apache.hadoop.fs.s3a.commit.InternalCommitterConstants; import org.apache.hadoop.fs.s3a.commit.impl.CommitContext; import org.apache.hadoop.fs.s3a.commit.impl.CommitOperations; +import org.apache.hadoop.fs.statistics.IOStatisticsContext; import static org.apache.hadoop.fs.s3a.commit.CommitConstants.*; import static org.apache.hadoop.fs.s3a.commit.staging.StagingTestBase.*; @@ -92,7 +93,8 @@ protected void verifyFailureConflictOutcome() throws Exception { // this is done by calling the preCommit method directly, final CommitContext commitContext = new CommitOperations(getWrapperFS()). - createCommitContext(getJob(), getOutputPath(), 0); + createCommitContext(getJob(), getOutputPath(), 0, + IOStatisticsContext.getCurrentIOStatisticsContext()); committer.preCommitJob(commitContext, AbstractS3ACommitter.ActiveCommit.empty()); reset(mockFS); diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/staging/integration/ITestDirectoryCommitProtocol.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/staging/integration/ITestDirectoryCommitProtocol.java index 36857669553f6..b19662c0117fd 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/staging/integration/ITestDirectoryCommitProtocol.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/staging/integration/ITestDirectoryCommitProtocol.java @@ -37,6 +37,7 @@ import static org.apache.hadoop.fs.s3a.commit.CommitConstants.CONFLICT_MODE_APPEND; import static org.apache.hadoop.fs.s3a.commit.CommitConstants.FS_S3A_COMMITTER_STAGING_CONFLICT_MODE; +import static org.apache.hadoop.fs.s3a.commit.CommitConstants.S3A_COMMITTER_EXPERIMENTAL_COLLECT_IOSTATISTICS; /** ITest of the low level protocol methods. */ public class ITestDirectoryCommitProtocol extends ITestStagingCommitProtocol { @@ -51,6 +52,14 @@ protected String getCommitterName() { return CommitConstants.COMMITTER_NAME_DIRECTORY; } + @Override + protected Configuration createConfiguration() { + Configuration conf = super.createConfiguration(); + // turn off stats collection to verify that it works + conf.setBoolean(S3A_COMMITTER_EXPERIMENTAL_COLLECT_IOSTATISTICS, false); + return conf; + } + @Override protected AbstractS3ACommitter createCommitter( Path outputPath, diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/impl/TestSDKStreamDrainer.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/impl/TestSDKStreamDrainer.java new file mode 100644 index 0000000000000..33a44a9ad78f7 --- /dev/null +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/impl/TestSDKStreamDrainer.java @@ -0,0 +1,343 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.s3a.impl; + +import java.io.IOException; + +import com.amazonaws.internal.SdkFilterInputStream; +import org.assertj.core.api.Assertions; +import org.junit.Test; + +import org.apache.hadoop.test.HadoopTestBase; + +import static org.apache.hadoop.fs.s3a.impl.InternalConstants.DRAIN_BUFFER_SIZE; +import static org.apache.hadoop.fs.s3a.statistics.impl.EmptyS3AStatisticsContext.EMPTY_INPUT_STREAM_STATISTICS; +import static org.apache.hadoop.test.LambdaTestUtils.intercept; + +/** + * Unit tests for stream draining. + */ +public class TestSDKStreamDrainer extends HadoopTestBase { + + public static final int BYTES = 100; + + /** + * Aborting does as asked. + */ + @Test + public void testDrainerAborted() throws Throwable { + assertAborted(drainer(BYTES, true, stream())); + } + + /** + * Create a stream of the default length. + * @return a stream. + */ + private static FakeSDKInputStream stream() { + return new FakeSDKInputStream(BYTES); + } + + /** + * a normal drain; all bytes are read. No abort. + */ + @Test + public void testDrainerDrained() throws Throwable { + assertBytesReadNotAborted( + drainer(BYTES, false, stream()), + BYTES); + } + + /** + * Empty streams are fine. + */ + @Test + public void testEmptyStream() throws Throwable { + int size = 0; + assertBytesReadNotAborted( + drainer(size, false, new FakeSDKInputStream(size)), + size); + } + + /** + * Single char read; just a safety check on the test stream more than + * the production code. + */ + @Test + public void testSingleChar() throws Throwable { + int size = 1; + assertBytesReadNotAborted( + drainer(size, false, new FakeSDKInputStream(size)), + size); + } + + /** + * a read spanning multiple buffers. + */ + @Test + public void testMultipleBuffers() throws Throwable { + int size = DRAIN_BUFFER_SIZE + 1; + assertBytesReadNotAborted( + drainer(size, false, new FakeSDKInputStream(size)), + size); + } + + /** + * Read of exactly one buffer. + */ + @Test + public void testExactlyOneBuffer() throws Throwable { + int size = DRAIN_BUFFER_SIZE; + assertBytesReadNotAborted( + drainer(size, false, new FakeSDKInputStream(size)), + size); + } + + /** + * Less data than expected came back. not escalated. + */ + @Test + public void testStreamUnderflow() throws Throwable { + int size = 50; + assertBytesReadNotAborted( + drainer(BYTES, false, new FakeSDKInputStream(size)), + size); + } + + /** + * Test a drain where a read triggers an IOE; this must escalate + * to an abort. + */ + @Test + public void testReadFailure() throws Throwable { + int threshold = 50; + SDKStreamDrainer drainer = new SDKStreamDrainer("s3://example/", + null, + new FakeSDKInputStream(BYTES, threshold), + false, + BYTES, + EMPTY_INPUT_STREAM_STATISTICS, "test"); + intercept(IOException.class, "", () -> + drainer.applyRaisingException()); + + assertAborted(drainer); + } + + /** + * abort does not read(), so the exception will not surface. + */ + @Test + public void testReadFailureDoesNotSurfaceInAbort() throws Throwable { + int threshold = 50; + SDKStreamDrainer drainer = new SDKStreamDrainer("s3://example/", + null, + new FakeSDKInputStream(BYTES, threshold), + true, + BYTES, + EMPTY_INPUT_STREAM_STATISTICS, "test"); + drainer.applyRaisingException(); + + assertAborted(drainer); + } + + /** + * make sure the underlying stream read code works. + */ + @Test + public void testFakeStreamRead() throws Throwable { + FakeSDKInputStream stream = stream(); + int count = 0; + while (stream.read() > 0) { + count++; + } + Assertions.assertThat(count) + .describedAs("bytes read from %s", stream) + .isEqualTo(BYTES); + } + + /** + * Create a drainer and invoke it, rethrowing any exception + * which occurred during the draining. + * @param remaining bytes remaining in the stream + * @param shouldAbort should we abort? + * @param in input stream. + * @return the drainer + * @throws Throwable something went wrong + */ + private SDKStreamDrainer drainer(int remaining, + boolean shouldAbort, + FakeSDKInputStream in) throws Throwable { + SDKStreamDrainer drainer = new SDKStreamDrainer("s3://example/", + null, + in, + shouldAbort, + remaining, + EMPTY_INPUT_STREAM_STATISTICS, "test"); + drainer.applyRaisingException(); + return drainer; + } + + + /** + * The draining aborted. + * @param drainer drainer to assert on. + * @return the drainer. + */ + private SDKStreamDrainer assertAborted(SDKStreamDrainer drainer) { + Assertions.assertThat(drainer) + .matches(SDKStreamDrainer::aborted, "aborted"); + return drainer; + } + + /** + * The draining was not aborted. + * @param drainer drainer to assert on. + * @return the drainer. + */ + private SDKStreamDrainer assertNotAborted(SDKStreamDrainer drainer) { + Assertions.assertThat(drainer) + .matches(d -> !d.aborted(), "is not aborted"); + return drainer; + } + + /** + * The draining was not aborted and {@code bytes} were read. + * @param drainer drainer to assert on. + * @param bytes expected byte count + * @return the drainer. + */ + private SDKStreamDrainer assertBytesReadNotAborted(SDKStreamDrainer drainer, + int bytes) { + return assertBytesRead(assertNotAborted(drainer), bytes); + } + + /** + * Assert {@code bytes} were read. + * @param drainer drainer to assert on. + * @param bytes expected byte count + * @return the drainer. + */ + private static SDKStreamDrainer assertBytesRead(final SDKStreamDrainer drainer, + final int bytes) { + Assertions.assertThat(drainer) + .describedAs("bytes read by %s", drainer) + .extracting(SDKStreamDrainer::getDrained) + .isEqualTo(bytes); + return drainer; + } + + + /** + * Fake stream; generates data dynamically. + * Only overrides the methods used in stream draining. + */ + private static final class FakeSDKInputStream extends SdkFilterInputStream { + + private final int capacity; + + private final int readToRaiseIOE; + + private int bytesRead; + + private boolean closed; + + private boolean aborted; + + /** + * read up to the capacity; optionally trigger an IOE. + * @param capacity total capacity. + * @param readToRaiseIOE position to raise an IOE, or -1 + */ + private FakeSDKInputStream(final int capacity, final int readToRaiseIOE) { + super(null); + this.capacity = capacity; + this.readToRaiseIOE = readToRaiseIOE; + } + + /** + * read up to the capacity. + * @param capacity total capacity. + */ + private FakeSDKInputStream(final int capacity) { + this(capacity, -1); + } + + @Override + public void abort() { + aborted = true; + } + + @Override + protected boolean isAborted() { + return aborted; + } + + @Override + public int read() throws IOException { + if (bytesRead >= capacity) { + // EOF + return -1; + } + bytesRead++; + if (readToRaiseIOE > 0 && bytesRead >= readToRaiseIOE) { + throw new IOException("IOE triggered on reading byte " + bytesRead); + } + return (int) '0' + (bytesRead % 10); + } + + @Override + public int read(final byte[] bytes, final int off, final int len) + throws IOException { + int count = 0; + + try { + while (count < len) { + int r = read(); + if (r < 0) { + break; + } + bytes[off + count] = (byte) r; + count++; + } + } catch (IOException e) { + if (count == 0) { + // first byte + throw e; + } + // otherwise break loop + } + return count; + } + + @Override + public void close() throws IOException { + closed = true; + } + + @Override + public String toString() { + return "FakeSDKInputStream{" + + "capacity=" + capacity + + ", readToRaiseIOE=" + readToRaiseIOE + + ", bytesRead=" + bytesRead + + ", closed=" + closed + + ", aborted=" + aborted + + "} " + super.toString(); + } + } +} diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/performance/ITestUnbufferDraining.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/performance/ITestUnbufferDraining.java new file mode 100644 index 0000000000000..a03f181cb3893 --- /dev/null +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/performance/ITestUnbufferDraining.java @@ -0,0 +1,287 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.s3a.performance; + +import java.io.IOException; + +import org.assertj.core.api.Assertions; +import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FSDataInputStream; +import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.contract.ContractTestUtils; +import org.apache.hadoop.fs.s3a.S3AFileSystem; +import org.apache.hadoop.fs.s3a.S3AInputPolicy; +import org.apache.hadoop.fs.s3a.S3AInputStream; +import org.apache.hadoop.fs.statistics.IOStatistics; +import org.apache.hadoop.io.IOUtils; + +import static org.apache.hadoop.fs.Options.OpenFileOptions.FS_OPTION_OPENFILE_READ_POLICY; +import static org.apache.hadoop.fs.Options.OpenFileOptions.FS_OPTION_OPENFILE_READ_POLICY_WHOLE_FILE; +import static org.apache.hadoop.fs.contract.ContractTestUtils.dataset; +import static org.apache.hadoop.fs.s3a.Constants.ASYNC_DRAIN_THRESHOLD; +import static org.apache.hadoop.fs.s3a.Constants.ESTABLISH_TIMEOUT; +import static org.apache.hadoop.fs.s3a.Constants.INPUT_FADVISE; +import static org.apache.hadoop.fs.s3a.Constants.MAXIMUM_CONNECTIONS; +import static org.apache.hadoop.fs.s3a.Constants.MAX_ERROR_RETRIES; +import static org.apache.hadoop.fs.s3a.Constants.PREFETCH_ENABLED_KEY; +import static org.apache.hadoop.fs.s3a.Constants.READAHEAD_RANGE; +import static org.apache.hadoop.fs.s3a.Constants.REQUEST_TIMEOUT; +import static org.apache.hadoop.fs.s3a.Constants.RETRY_LIMIT; +import static org.apache.hadoop.fs.s3a.Constants.SOCKET_TIMEOUT; +import static org.apache.hadoop.fs.s3a.S3ATestUtils.removeBaseAndBucketOverrides; +import static org.apache.hadoop.fs.statistics.IOStatisticAssertions.verifyStatisticCounterValue; +import static org.apache.hadoop.fs.statistics.IOStatisticsSupport.retrieveIOStatistics; +import static org.apache.hadoop.fs.statistics.StreamStatisticNames.STREAM_READ_ABORTED; +import static org.apache.hadoop.fs.statistics.StreamStatisticNames.STREAM_READ_SEEK_POLICY_CHANGED; +import static org.apache.hadoop.fs.statistics.StreamStatisticNames.STREAM_READ_UNBUFFERED; + +/** + * Test stream unbuffer performance/behavior with stream draining + * and aborting. + */ +public class ITestUnbufferDraining extends AbstractS3ACostTest { + + private static final Logger LOG = + LoggerFactory.getLogger(ITestUnbufferDraining.class); + + /** + * Readahead range to use, sets drain threshold too. + */ + public static final int READAHEAD = 1000; + + /** + * How big a file to create? + */ + public static final int FILE_SIZE = 50_000; + + /** + * Number of attempts to unbuffer on each stream. + */ + public static final int ATTEMPTS = 10; + + /** + * Test FS with a tiny connection pool and + * no recovery. + */ + private FileSystem brittleFS; + + /** + * Create with markers kept, always. + */ + public ITestUnbufferDraining() { + super(false); + } + + @Override + public Configuration createConfiguration() { + Configuration conf = super.createConfiguration(); + removeBaseAndBucketOverrides(conf, + ASYNC_DRAIN_THRESHOLD, + ESTABLISH_TIMEOUT, + INPUT_FADVISE, + MAX_ERROR_RETRIES, + MAXIMUM_CONNECTIONS, + PREFETCH_ENABLED_KEY, + READAHEAD_RANGE, + REQUEST_TIMEOUT, + RETRY_LIMIT, + SOCKET_TIMEOUT); + + return conf; + } + + @Override + public void setup() throws Exception { + super.setup(); + + // now create a new FS with minimal http capacity and recovery + // a separate one is used to avoid test teardown suffering + // from the lack of http connections and short timeouts. + Configuration conf = getConfiguration(); + // kick off async drain for any data + conf.setInt(ASYNC_DRAIN_THRESHOLD, 1); + conf.setInt(MAXIMUM_CONNECTIONS, 2); + conf.setInt(MAX_ERROR_RETRIES, 1); + conf.setInt(ESTABLISH_TIMEOUT, 1000); + conf.setInt(READAHEAD_RANGE, READAHEAD); + conf.setInt(RETRY_LIMIT, 1); + + brittleFS = FileSystem.newInstance(getFileSystem().getUri(), conf); + } + + @Override + public void teardown() throws Exception { + super.teardown(); + FileSystem bfs = getBrittleFS(); + FILESYSTEM_IOSTATS.aggregate(retrieveIOStatistics(bfs)); + IOUtils.cleanupWithLogger(LOG, bfs); + } + + public FileSystem getBrittleFS() { + return brittleFS; + } + + /** + * Test stream close performance/behavior with stream draining + * and unbuffer. + */ + @Test + public void testUnbufferDraining() throws Throwable { + + describe("unbuffer draining"); + FileStatus st = createTestFile(); + + IOStatistics brittleStats = retrieveIOStatistics(getBrittleFS()); + long originalUnbuffered = lookupCounter(brittleStats, + STREAM_READ_UNBUFFERED); + + int offset = FILE_SIZE - READAHEAD + 1; + try (FSDataInputStream in = getBrittleFS().openFile(st.getPath()) + .withFileStatus(st) + .must(ASYNC_DRAIN_THRESHOLD, 1) + .build().get()) { + describe("Initiating unbuffer with async drain\n"); + for (int i = 0; i < ATTEMPTS; i++) { + describe("Starting read/unbuffer #%d", i); + in.seek(offset); + in.read(); + in.unbuffer(); + } + // verify the policy switched. + assertReadPolicy(in, S3AInputPolicy.Random); + // assert that the statistics are as expected + IOStatistics stats = in.getIOStatistics(); + verifyStatisticCounterValue(stats, + STREAM_READ_UNBUFFERED, + ATTEMPTS); + verifyStatisticCounterValue(stats, + STREAM_READ_ABORTED, + 0); + // there's always a policy of 1, so + // this value must be 1 + 1 + verifyStatisticCounterValue(stats, + STREAM_READ_SEEK_POLICY_CHANGED, + 2); + } + // filesystem statistic propagation + verifyStatisticCounterValue(brittleStats, + STREAM_READ_UNBUFFERED, + ATTEMPTS + originalUnbuffered); + } + + /** + * Lookup a counter, returning 0 if it is not defined. + * @param statistics stats to probe + * @param key counter key + * @return the value or 0 + */ + private static long lookupCounter( + final IOStatistics statistics, + final String key) { + Long counter = statistics.counters().get(key); + return counter == null ? 0 : counter; + } + + /** + * Assert that the read policy is as expected. + * @param in input stream + * @param policy read policy. + */ + private static void assertReadPolicy(final FSDataInputStream in, + final S3AInputPolicy policy) { + S3AInputStream inner = (S3AInputStream) in.getWrappedStream(); + Assertions.assertThat(inner.getInputPolicy()) + .describedAs("input policy of %s", inner) + .isEqualTo(policy); + } + + /** + * Test stream close performance/behavior with unbuffer + * aborting rather than draining. + */ + @Test + public void testUnbufferAborting() throws Throwable { + + describe("unbuffer aborting"); + FileStatus st = createTestFile(); + IOStatistics brittleStats = retrieveIOStatistics(getBrittleFS()); + long originalUnbuffered = + lookupCounter(brittleStats, STREAM_READ_UNBUFFERED); + long originalAborted = + lookupCounter(brittleStats, STREAM_READ_ABORTED); + + // open the file at the beginning with a whole file read policy, + // so even with s3a switching to random on unbuffer, + // this always does a full GET + try (FSDataInputStream in = getBrittleFS().openFile(st.getPath()) + .withFileStatus(st) + .must(ASYNC_DRAIN_THRESHOLD, 1) + .must(FS_OPTION_OPENFILE_READ_POLICY, + FS_OPTION_OPENFILE_READ_POLICY_WHOLE_FILE) + .build().get()) { + assertReadPolicy(in, S3AInputPolicy.Sequential); + + describe("Initiating unbuffer with async drain\n"); + for (int i = 0; i < ATTEMPTS; i++) { + describe("Starting read/unbuffer #%d", i); + in.read(); + in.unbuffer(); + // because the read policy is sequential, it doesn't change + assertReadPolicy(in, S3AInputPolicy.Sequential); + } + + // assert that the statistics are as expected + IOStatistics stats = in.getIOStatistics(); + verifyStatisticCounterValue(stats, + STREAM_READ_UNBUFFERED, + ATTEMPTS); + verifyStatisticCounterValue(stats, + STREAM_READ_ABORTED, + ATTEMPTS); + // there's always a policy of 1. + verifyStatisticCounterValue(stats, + STREAM_READ_SEEK_POLICY_CHANGED, + 1); + } + // look at FS statistics + verifyStatisticCounterValue(brittleStats, + STREAM_READ_UNBUFFERED, + ATTEMPTS + originalUnbuffered); + verifyStatisticCounterValue(brittleStats, + STREAM_READ_ABORTED, + ATTEMPTS + originalAborted); + } + + private FileStatus createTestFile() throws IOException { + byte[] data = dataset(FILE_SIZE, '0', 10); + S3AFileSystem fs = getFileSystem(); + + Path path = methodPath(); + ContractTestUtils.createFile(fs, path, true, data); + return fs.getFileStatus(path); + } + + +} diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/prefetch/MockS3ARemoteObject.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/prefetch/MockS3ARemoteObject.java new file mode 100644 index 0000000000000..6e2f547a22ec1 --- /dev/null +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/prefetch/MockS3ARemoteObject.java @@ -0,0 +1,114 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hadoop.fs.s3a.prefetch; + +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.util.concurrent.CompletableFuture; + +import com.amazonaws.services.s3.model.GetObjectRequest; +import com.amazonaws.services.s3.model.S3Object; + +import org.apache.hadoop.fs.impl.prefetch.Validate; +import org.apache.hadoop.fs.s3a.S3AInputStream; +import org.apache.hadoop.fs.s3a.statistics.impl.EmptyS3AStatisticsContext; +import org.apache.hadoop.util.functional.CallableRaisingIOE; + +/** + * A mock s3 file with some fault injection. + */ +class MockS3ARemoteObject extends S3ARemoteObject { + + private byte[] contents; + + // If true, throws IOException on open request just once. + // That allows test code to validate behavior related to retries. + private boolean throwExceptionOnOpen; + + private static final String BUCKET = "bucket"; + + private static final String KEY = "key"; + + MockS3ARemoteObject(int size) { + this(size, false); + } + + MockS3ARemoteObject(int size, boolean throwExceptionOnOpen) { + super( + S3APrefetchFakes.createReadContext(null, KEY, size, 1, 1), + S3APrefetchFakes.createObjectAttributes(BUCKET, KEY, size), + S3APrefetchFakes.createInputStreamCallbacks(BUCKET, KEY), + EmptyS3AStatisticsContext.EMPTY_INPUT_STREAM_STATISTICS, + S3APrefetchFakes.createChangeTracker(BUCKET, KEY, size) + ); + + this.throwExceptionOnOpen = throwExceptionOnOpen; + this.contents = new byte[size]; + for (int b = 0; b < size; b++) { + this.contents[b] = byteAtOffset(b); + } + } + + @Override + public InputStream openForRead(long offset, int size) throws IOException { + Validate.checkLessOrEqual(offset, "offset", size(), "size()"); + Validate.checkLessOrEqual(size, "size", size() - offset, "size() - offset"); + + if (throwExceptionOnOpen) { + throwExceptionOnOpen = false; + throw new IOException("Throwing because throwExceptionOnOpen is true "); + } + int bufSize = (int) Math.min(size, size() - offset); + return new ByteArrayInputStream(contents, (int) offset, bufSize); + } + + @Override + public void close(InputStream inputStream, int numRemainingBytes) { + // do nothing since we do not use a real S3 stream. + } + + public static byte byteAtOffset(int offset) { + return (byte) (offset % 128); + } + + public static S3AInputStream.InputStreamCallbacks createClient(String bucketName) { + return new S3AInputStream.InputStreamCallbacks() { + @Override + public S3Object getObject(GetObjectRequest request) { + return null; + } + + @Override + public CompletableFuture submit(CallableRaisingIOE operation) { + return null; + } + + @Override + public GetObjectRequest newGetRequest(String key) { + return new GetObjectRequest(bucketName, key); + } + + @Override + public void close() { + } + }; + } +} diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/prefetch/S3APrefetchFakes.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/prefetch/S3APrefetchFakes.java new file mode 100644 index 0000000000000..bab07f4f9ec83 --- /dev/null +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/prefetch/S3APrefetchFakes.java @@ -0,0 +1,413 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hadoop.fs.s3a.prefetch; + +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.net.URI; +import java.nio.ByteBuffer; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.Map; +import java.util.Random; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.TimeUnit; + +import com.amazonaws.services.s3.model.GetObjectRequest; +import com.amazonaws.services.s3.model.ObjectMetadata; +import com.amazonaws.services.s3.model.S3Object; +import com.amazonaws.services.s3.model.S3ObjectInputStream; + +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.impl.prefetch.BlockCache; +import org.apache.hadoop.fs.impl.prefetch.BlockData; +import org.apache.hadoop.fs.impl.prefetch.ExecutorServiceFuturePool; +import org.apache.hadoop.fs.impl.prefetch.SingleFilePerBlockCache; +import org.apache.hadoop.fs.impl.prefetch.Validate; +import org.apache.hadoop.fs.s3a.Invoker; +import org.apache.hadoop.fs.s3a.S3AEncryptionMethods; +import org.apache.hadoop.fs.s3a.S3AFileStatus; +import org.apache.hadoop.fs.s3a.S3AInputPolicy; +import org.apache.hadoop.fs.s3a.S3AInputStream; +import org.apache.hadoop.fs.s3a.S3AReadOpContext; +import org.apache.hadoop.fs.s3a.S3ObjectAttributes; +import org.apache.hadoop.fs.s3a.VectoredIOContext; +import org.apache.hadoop.fs.s3a.impl.ChangeDetectionPolicy; +import org.apache.hadoop.fs.s3a.impl.ChangeTracker; +import org.apache.hadoop.fs.s3a.statistics.S3AInputStreamStatistics; +import org.apache.hadoop.fs.s3a.statistics.S3AStatisticsContext; +import org.apache.hadoop.fs.s3a.statistics.impl.CountingChangeTracker; +import org.apache.hadoop.fs.s3a.statistics.impl.EmptyS3AStatisticsContext; +import org.apache.hadoop.io.retry.RetryPolicies; +import org.apache.hadoop.io.retry.RetryPolicy; +import org.apache.hadoop.util.functional.CallableRaisingIOE; + +import static org.apache.hadoop.fs.statistics.impl.IOStatisticsBinding.emptyStatisticsStore; + +/** + * Provides 'fake' implementations of S3ARemoteInputStream variants. + * + * These implementations avoid accessing the following real resources: + * -- S3 store + * -- local filesystem + * + * This arrangement allows thorough multi-threaded testing of those + * implementations without accessing external resources. It also helps + * avoid test flakiness introduced by external factors. + */ +public final class S3APrefetchFakes { + + private S3APrefetchFakes() { + } + + public static final String E_TAG = "eTag"; + + public static final String OWNER = "owner"; + + public static final String VERSION_ID = "v1"; + + public static final long MODIFICATION_TIME = 0L; + + public static final ChangeDetectionPolicy CHANGE_POLICY = + ChangeDetectionPolicy.createPolicy( + ChangeDetectionPolicy.Mode.None, + ChangeDetectionPolicy.Source.None, + false); + + public static S3AFileStatus createFileStatus(String key, long fileSize) { + org.apache.hadoop.fs.Path path = new org.apache.hadoop.fs.Path(key); + long blockSize = fileSize; + return new S3AFileStatus( + fileSize, MODIFICATION_TIME, path, blockSize, OWNER, E_TAG, VERSION_ID); + } + + public static S3ObjectAttributes createObjectAttributes( + String bucket, + String key, + long fileSize) { + + org.apache.hadoop.fs.Path path = new org.apache.hadoop.fs.Path(key); + String encryptionKey = ""; + + return new S3ObjectAttributes( + bucket, + path, + key, + S3AEncryptionMethods.NONE, + encryptionKey, + E_TAG, + VERSION_ID, + fileSize); + } + + public static S3AReadOpContext createReadContext( + ExecutorServiceFuturePool futurePool, + String key, + int fileSize, + int prefetchBlockSize, + int prefetchBlockCount) { + + S3AFileStatus fileStatus = createFileStatus(key, fileSize); + org.apache.hadoop.fs.Path path = new org.apache.hadoop.fs.Path(key); + FileSystem.Statistics statistics = new FileSystem.Statistics("s3a"); + S3AStatisticsContext statisticsContext = new EmptyS3AStatisticsContext(); + RetryPolicy retryPolicy = + RetryPolicies.retryUpToMaximumCountWithFixedSleep(3, 10, + TimeUnit.MILLISECONDS); + + return new S3AReadOpContext( + path, + new Invoker(retryPolicy, Invoker.LOG_EVENT), + statistics, + statisticsContext, + fileStatus, + new VectoredIOContext() + .setMinSeekForVectoredReads(1) + .setMaxReadSizeForVectoredReads(1) + .build(), + emptyStatisticsStore(), + futurePool, + prefetchBlockSize, + prefetchBlockCount) + .withChangeDetectionPolicy( + ChangeDetectionPolicy.createPolicy(ChangeDetectionPolicy.Mode.None, + ChangeDetectionPolicy.Source.ETag, false)) + .withInputPolicy(S3AInputPolicy.Normal); + } + + public static URI createUri(String bucket, String key) { + return URI.create(String.format("s3a://%s/%s", bucket, key)); + } + + public static ChangeTracker createChangeTracker( + String bucket, + String key, + long fileSize) { + + return new ChangeTracker( + createUri(bucket, key).toString(), + CHANGE_POLICY, + new CountingChangeTracker(), + createObjectAttributes(bucket, key, fileSize)); + } + + public static S3ObjectInputStream createS3ObjectInputStream(byte[] buffer) { + return new S3ObjectInputStream(new ByteArrayInputStream(buffer), null); + } + + public static S3AInputStream.InputStreamCallbacks createInputStreamCallbacks( + String bucket, + String key) { + + S3Object object = new S3Object() { + @Override + public S3ObjectInputStream getObjectContent() { + return createS3ObjectInputStream(new byte[8]); + } + + @Override + public ObjectMetadata getObjectMetadata() { + ObjectMetadata metadata = new ObjectMetadata(); + metadata.setHeader("ETag", E_TAG); + return metadata; + } + }; + + return new S3AInputStream.InputStreamCallbacks() { + @Override + public S3Object getObject(GetObjectRequest request) { + return object; + } + + @Override + public CompletableFuture submit(CallableRaisingIOE operation) { + return null; + } + + @Override + public GetObjectRequest newGetRequest(String key) { + return new GetObjectRequest(bucket, key); + } + + @Override + public void close() { + } + }; + } + + + public static S3ARemoteInputStream createInputStream( + Class clazz, + ExecutorServiceFuturePool futurePool, + String bucket, + String key, + int fileSize, + int prefetchBlockSize, + int prefetchBlockCount) { + + org.apache.hadoop.fs.Path path = new org.apache.hadoop.fs.Path(key); + + S3AFileStatus fileStatus = createFileStatus(key, fileSize); + S3ObjectAttributes s3ObjectAttributes = + createObjectAttributes(bucket, key, fileSize); + S3AReadOpContext s3AReadOpContext = createReadContext( + futurePool, + key, + fileSize, + prefetchBlockSize, + prefetchBlockCount); + + S3AInputStream.InputStreamCallbacks callbacks = + createInputStreamCallbacks(bucket, key); + S3AInputStreamStatistics stats = + s3AReadOpContext.getS3AStatisticsContext().newInputStreamStatistics(); + + if (clazz == FakeS3AInMemoryInputStream.class) { + return new FakeS3AInMemoryInputStream(s3AReadOpContext, + s3ObjectAttributes, callbacks, stats); + } else if (clazz == FakeS3ACachingInputStream.class) { + return new FakeS3ACachingInputStream(s3AReadOpContext, s3ObjectAttributes, + callbacks, stats); + } + + throw new RuntimeException("Unsupported class: " + clazz); + } + + public static FakeS3AInMemoryInputStream createS3InMemoryInputStream( + ExecutorServiceFuturePool futurePool, + String bucket, + String key, + int fileSize) { + + return (FakeS3AInMemoryInputStream) createInputStream( + FakeS3AInMemoryInputStream.class, futurePool, bucket, key, fileSize, 1, + 1); + } + + public static FakeS3ACachingInputStream createS3CachingInputStream( + ExecutorServiceFuturePool futurePool, + String bucket, + String key, + int fileSize, + int prefetchBlockSize, + int prefetchBlockCount) { + + return (FakeS3ACachingInputStream) createInputStream( + FakeS3ACachingInputStream.class, + futurePool, + bucket, + key, + fileSize, + prefetchBlockSize, + prefetchBlockCount); + } + + public static class FakeS3AInMemoryInputStream + extends S3AInMemoryInputStream { + + public FakeS3AInMemoryInputStream( + S3AReadOpContext context, + S3ObjectAttributes s3Attributes, + S3AInputStream.InputStreamCallbacks client, + S3AInputStreamStatistics streamStatistics) { + super(context, s3Attributes, client, streamStatistics); + } + + @Override + protected S3ARemoteObject getS3File() { + randomDelay(200); + return new MockS3ARemoteObject( + (int) this.getS3ObjectAttributes().getLen(), false); + } + } + + public static class FakeS3FilePerBlockCache extends SingleFilePerBlockCache { + + private final Map files; + + private final int readDelay; + + private final int writeDelay; + + public FakeS3FilePerBlockCache(int readDelay, int writeDelay) { + super(new EmptyS3AStatisticsContext().newInputStreamStatistics()); + this.files = new ConcurrentHashMap<>(); + this.readDelay = readDelay; + this.writeDelay = writeDelay; + } + + @Override + protected int readFile(Path path, ByteBuffer buffer) { + byte[] source = this.files.get(path); + randomDelay(this.readDelay); + buffer.put(source); + return source.length; + } + + @Override + protected void writeFile(Path path, ByteBuffer buffer) throws IOException { + Validate.checkPositiveInteger(buffer.limit(), "buffer.limit()"); + byte[] dest = new byte[buffer.limit()]; + randomDelay(this.writeDelay); + buffer.rewind(); + buffer.get(dest); + this.files.put(path, dest); + } + + private long fileCount = 0; + + @Override + protected Path getCacheFilePath() throws IOException { + fileCount++; + return Paths.get(Long.toString(fileCount)); + } + + @Override + public void close() throws IOException { + this.files.clear(); + } + } + + private static final Random RANDOM = new Random(); + + private static void randomDelay(int delay) { + try { + Thread.sleep(RANDOM.nextInt(delay)); + } catch (InterruptedException e) { + + } + } + + public static class FakeS3ACachingBlockManager + extends S3ACachingBlockManager { + + public FakeS3ACachingBlockManager( + ExecutorServiceFuturePool futurePool, + S3ARemoteObjectReader reader, + BlockData blockData, + int bufferPoolSize) { + super(futurePool, reader, blockData, bufferPoolSize, + new EmptyS3AStatisticsContext().newInputStreamStatistics()); + } + + @Override + public int read(ByteBuffer buffer, long offset, int size) + throws IOException { + randomDelay(100); + return this.getReader().read(buffer, offset, size); + } + + @Override + protected BlockCache createCache() { + final int readDelayMs = 50; + final int writeDelayMs = 200; + return new FakeS3FilePerBlockCache(readDelayMs, writeDelayMs); + } + } + + public static class FakeS3ACachingInputStream extends S3ACachingInputStream { + + public FakeS3ACachingInputStream( + S3AReadOpContext context, + S3ObjectAttributes s3Attributes, + S3AInputStream.InputStreamCallbacks client, + S3AInputStreamStatistics streamStatistics) { + super(context, s3Attributes, client, streamStatistics); + } + + @Override + protected S3ARemoteObject getS3File() { + randomDelay(200); + return new MockS3ARemoteObject( + (int) this.getS3ObjectAttributes().getLen(), false); + } + + @Override + protected S3ACachingBlockManager createBlockManager( + ExecutorServiceFuturePool futurePool, + S3ARemoteObjectReader reader, + BlockData blockData, + int bufferPoolSize) { + return new FakeS3ACachingBlockManager(futurePool, reader, blockData, + bufferPoolSize); + } + } +} diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/prefetch/TestS3ABlockManager.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/prefetch/TestS3ABlockManager.java new file mode 100644 index 0000000000000..c1b59d6f2e130 --- /dev/null +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/prefetch/TestS3ABlockManager.java @@ -0,0 +1,87 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hadoop.fs.s3a.prefetch; + +import java.io.IOException; +import java.nio.ByteBuffer; + +import org.junit.Test; + +import org.apache.hadoop.fs.impl.prefetch.BlockData; +import org.apache.hadoop.fs.impl.prefetch.BufferData; +import org.apache.hadoop.test.AbstractHadoopTestBase; + +import static org.apache.hadoop.test.LambdaTestUtils.intercept; +import static org.junit.Assert.assertEquals; + +public class TestS3ABlockManager extends AbstractHadoopTestBase { + + static final int FILE_SIZE = 12; + + static final int BLOCK_SIZE = 3; + + @Test + public void testArgChecks() throws Exception { + BlockData blockData = new BlockData(FILE_SIZE, BLOCK_SIZE); + MockS3ARemoteObject s3File = new MockS3ARemoteObject(FILE_SIZE, false); + S3ARemoteObjectReader reader = new S3ARemoteObjectReader(s3File); + + // Should not throw. + new S3ABlockManager(reader, blockData); + + // Verify it throws correctly. + intercept( + IllegalArgumentException.class, + "'reader' must not be null", + () -> new S3ABlockManager(null, blockData)); + + intercept( + IllegalArgumentException.class, + "'blockData' must not be null", + () -> new S3ABlockManager(reader, null)); + + intercept( + IllegalArgumentException.class, + "'blockNumber' must not be negative", + () -> new S3ABlockManager(reader, blockData).get(-1)); + + intercept( + IllegalArgumentException.class, + "'data' must not be null", + () -> new S3ABlockManager(reader, blockData).release(null)); + } + + @Test + public void testGet() throws IOException { + BlockData blockData = new BlockData(FILE_SIZE, BLOCK_SIZE); + MockS3ARemoteObject s3File = new MockS3ARemoteObject(FILE_SIZE, false); + S3ARemoteObjectReader reader = new S3ARemoteObjectReader(s3File); + S3ABlockManager blockManager = new S3ABlockManager(reader, blockData); + + for (int b = 0; b < blockData.getNumBlocks(); b++) { + BufferData data = blockManager.get(b); + ByteBuffer buffer = data.getBuffer(); + long startOffset = blockData.getStartOffset(b); + for (int i = 0; i < BLOCK_SIZE; i++) { + assertEquals(startOffset + i, buffer.get()); + } + } + } +} diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/prefetch/TestS3ACachingBlockManager.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/prefetch/TestS3ACachingBlockManager.java new file mode 100644 index 0000000000000..aecf8802beb85 --- /dev/null +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/prefetch/TestS3ACachingBlockManager.java @@ -0,0 +1,361 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hadoop.fs.s3a.prefetch; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; + +import org.junit.Test; + +import org.apache.hadoop.fs.impl.prefetch.BlockData; +import org.apache.hadoop.fs.impl.prefetch.BufferData; +import org.apache.hadoop.fs.impl.prefetch.ExecutorServiceFuturePool; +import org.apache.hadoop.fs.s3a.statistics.S3AInputStreamStatistics; +import org.apache.hadoop.fs.s3a.statistics.impl.EmptyS3AStatisticsContext; +import org.apache.hadoop.test.AbstractHadoopTestBase; + +import static org.apache.hadoop.test.LambdaTestUtils.intercept; +import static org.junit.Assert.assertEquals; + +public class TestS3ACachingBlockManager extends AbstractHadoopTestBase { + + static final int FILE_SIZE = 15; + + static final int BLOCK_SIZE = 2; + + static final int POOL_SIZE = 3; + + private final ExecutorService threadPool = Executors.newFixedThreadPool(4); + + private final ExecutorServiceFuturePool futurePool = + new ExecutorServiceFuturePool(threadPool); + + private final S3AInputStreamStatistics streamStatistics = + new EmptyS3AStatisticsContext().newInputStreamStatistics(); + + private final BlockData blockData = new BlockData(FILE_SIZE, BLOCK_SIZE); + + @Test + public void testArgChecks() throws Exception { + MockS3ARemoteObject s3File = new MockS3ARemoteObject(FILE_SIZE, false); + S3ARemoteObjectReader reader = new S3ARemoteObjectReader(s3File); + + // Should not throw. + S3ACachingBlockManager blockManager = + new S3ACachingBlockManager(futurePool, reader, blockData, POOL_SIZE, + streamStatistics); + + // Verify it throws correctly. + intercept( + NullPointerException.class, + () -> new S3ACachingBlockManager(null, reader, blockData, POOL_SIZE, + streamStatistics)); + + intercept( + IllegalArgumentException.class, + "'reader' must not be null", + () -> new S3ACachingBlockManager(futurePool, null, blockData, POOL_SIZE, + streamStatistics)); + + intercept( + IllegalArgumentException.class, + "'blockData' must not be null", + () -> new S3ACachingBlockManager(futurePool, reader, null, POOL_SIZE, + streamStatistics)); + + intercept( + IllegalArgumentException.class, + "'bufferPoolSize' must be a positive integer", + () -> new S3ACachingBlockManager(futurePool, reader, blockData, 0, + streamStatistics)); + + intercept( + IllegalArgumentException.class, + "'bufferPoolSize' must be a positive integer", + () -> new S3ACachingBlockManager(futurePool, reader, blockData, -1, + streamStatistics)); + + intercept(NullPointerException.class, + () -> new S3ACachingBlockManager(futurePool, reader, blockData, + POOL_SIZE, null)); + + intercept( + IllegalArgumentException.class, + "'blockNumber' must not be negative", + () -> blockManager.get(-1)); + + intercept( + IllegalArgumentException.class, + "'data' must not be null", + () -> blockManager.release(null)); + + intercept( + IllegalArgumentException.class, + "'blockNumber' must not be negative", + () -> blockManager.requestPrefetch(-1)); + + intercept( + IllegalArgumentException.class, + "'data' must not be null", + () -> blockManager.requestCaching(null)); + } + + /** + * Extends S3ACachingBlockManager so that we can inject asynchronous failures. + */ + private static final class BlockManagerForTesting + extends S3ACachingBlockManager { + + BlockManagerForTesting( + ExecutorServiceFuturePool futurePool, + S3ARemoteObjectReader reader, + BlockData blockData, + int bufferPoolSize, + S3AInputStreamStatistics streamStatistics) { + super(futurePool, reader, blockData, bufferPoolSize, streamStatistics); + } + + // If true, forces the next read operation to fail. + // Resets itself to false after one failure. + private boolean forceNextReadToFail; + + @Override + public int read(ByteBuffer buffer, long offset, int size) + throws IOException { + if (forceNextReadToFail) { + forceNextReadToFail = false; + throw new RuntimeException("foo"); + } else { + return super.read(buffer, offset, size); + } + } + + // If true, forces the next cache-put operation to fail. + // Resets itself to false after one failure. + private boolean forceNextCachePutToFail; + + @Override + protected void cachePut(int blockNumber, ByteBuffer buffer) + throws IOException { + if (forceNextCachePutToFail) { + forceNextCachePutToFail = false; + throw new RuntimeException("bar"); + } else { + super.cachePut(blockNumber, buffer); + } + } + } + + // @Ignore + @Test + public void testGet() throws Exception { + testGetHelper(false); + } + + // @Ignore + @Test + public void testGetFailure() throws Exception { + testGetHelper(true); + } + + private void testGetHelper(boolean forceReadFailure) throws Exception { + MockS3ARemoteObject s3File = new MockS3ARemoteObject(FILE_SIZE, true); + S3ARemoteObjectReader reader = new S3ARemoteObjectReader(s3File); + BlockManagerForTesting blockManager = + new BlockManagerForTesting(futurePool, reader, blockData, POOL_SIZE, + streamStatistics); + + for (int b = 0; b < blockData.getNumBlocks(); b++) { + // We simulate caching failure for all even numbered blocks. + boolean forceFailure = forceReadFailure && (b % 2 == 0); + + BufferData data = null; + + if (forceFailure) { + blockManager.forceNextReadToFail = true; + + intercept( + RuntimeException.class, + "foo", + () -> blockManager.get(3)); + } else { + data = blockManager.get(b); + + long startOffset = blockData.getStartOffset(b); + for (int i = 0; i < blockData.getSize(b); i++) { + assertEquals(startOffset + i, data.getBuffer().get()); + } + + blockManager.release(data); + } + + assertEquals(POOL_SIZE, blockManager.numAvailable()); + } + } + + // @Ignore + @Test + public void testPrefetch() throws IOException, InterruptedException { + testPrefetchHelper(false); + } + + // @Ignore + @Test + public void testPrefetchFailure() throws IOException, InterruptedException { + testPrefetchHelper(true); + } + + private void testPrefetchHelper(boolean forcePrefetchFailure) + throws IOException, InterruptedException { + MockS3ARemoteObject s3File = new MockS3ARemoteObject(FILE_SIZE, false); + S3ARemoteObjectReader reader = new S3ARemoteObjectReader(s3File); + BlockManagerForTesting blockManager = + new BlockManagerForTesting(futurePool, reader, blockData, POOL_SIZE, + streamStatistics); + assertInitialState(blockManager); + + int expectedNumErrors = 0; + int expectedNumSuccesses = 0; + + for (int b = 0; b < POOL_SIZE; b++) { + // We simulate caching failure for all odd numbered blocks. + boolean forceFailure = forcePrefetchFailure && (b % 2 == 1); + if (forceFailure) { + expectedNumErrors++; + blockManager.forceNextReadToFail = true; + } else { + expectedNumSuccesses++; + } + blockManager.requestPrefetch(b); + } + + assertEquals(0, blockManager.numCached()); + + blockManager.cancelPrefetches(); + waitForCaching(blockManager, expectedNumSuccesses); + assertEquals(expectedNumErrors, this.totalErrors(blockManager)); + assertEquals(expectedNumSuccesses, blockManager.numCached()); + } + + // @Ignore + @Test + public void testCachingOfPrefetched() + throws IOException, InterruptedException { + MockS3ARemoteObject s3File = new MockS3ARemoteObject(FILE_SIZE, false); + S3ARemoteObjectReader reader = new S3ARemoteObjectReader(s3File); + S3ACachingBlockManager blockManager = + new S3ACachingBlockManager(futurePool, reader, blockData, POOL_SIZE, + streamStatistics); + assertInitialState(blockManager); + + for (int b = 0; b < blockData.getNumBlocks(); b++) { + blockManager.requestPrefetch(b); + BufferData data = blockManager.get(b); + blockManager.requestCaching(data); + } + + waitForCaching(blockManager, blockData.getNumBlocks()); + assertEquals(blockData.getNumBlocks(), blockManager.numCached()); + assertEquals(0, this.totalErrors(blockManager)); + } + + // @Ignore + @Test + public void testCachingOfGet() throws IOException, InterruptedException { + testCachingOfGetHelper(false); + } + + // @Ignore + @Test + public void testCachingFailureOfGet() + throws IOException, InterruptedException { + testCachingOfGetHelper(true); + } + + public void testCachingOfGetHelper(boolean forceCachingFailure) + throws IOException, InterruptedException { + MockS3ARemoteObject s3File = new MockS3ARemoteObject(FILE_SIZE, false); + S3ARemoteObjectReader reader = new S3ARemoteObjectReader(s3File); + BlockManagerForTesting blockManager = + new BlockManagerForTesting(futurePool, reader, blockData, POOL_SIZE, + streamStatistics); + assertInitialState(blockManager); + + int expectedNumErrors = 0; + int expectedNumSuccesses = 0; + + for (int b = 0; b < blockData.getNumBlocks(); b++) { + // We simulate caching failure for all odd numbered blocks. + boolean forceFailure = forceCachingFailure && (b % 2 == 1); + if (forceFailure) { + expectedNumErrors++; + } else { + expectedNumSuccesses++; + } + + BufferData data = blockManager.get(b); + if (forceFailure) { + blockManager.forceNextCachePutToFail = true; + } + + blockManager.requestCaching(data); + waitForCaching(blockManager, expectedNumSuccesses); + assertEquals(expectedNumSuccesses, blockManager.numCached()); + + if (forceCachingFailure) { + assertEquals(expectedNumErrors, this.totalErrors(blockManager)); + } else { + assertEquals(0, this.totalErrors(blockManager)); + } + } + } + + private void waitForCaching( + S3ACachingBlockManager blockManager, + int expectedCount) + throws InterruptedException { + // Wait for async cache operation to be over. + int numTrys = 0; + int count; + do { + Thread.sleep(100); + count = blockManager.numCached(); + numTrys++; + if (numTrys > 600) { + String message = String.format( + "waitForCaching: expected: %d, actual: %d, read errors: %d, caching errors: %d", + expectedCount, count, blockManager.numReadErrors(), + blockManager.numCachingErrors()); + throw new IllegalStateException(message); + } + } + while (count < expectedCount); + } + + private int totalErrors(S3ACachingBlockManager blockManager) { + return blockManager.numCachingErrors() + blockManager.numReadErrors(); + } + + private void assertInitialState(S3ACachingBlockManager blockManager) { + assertEquals(0, blockManager.numCached()); + } +} diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/prefetch/TestS3ARemoteInputStream.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/prefetch/TestS3ARemoteInputStream.java new file mode 100644 index 0000000000000..4ab33ef6cd074 --- /dev/null +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/prefetch/TestS3ARemoteInputStream.java @@ -0,0 +1,279 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hadoop.fs.s3a.prefetch; + +import java.io.EOFException; +import java.io.IOException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; + +import org.junit.Test; + +import org.apache.hadoop.fs.FSExceptionMessages; +import org.apache.hadoop.fs.impl.prefetch.ExceptionAsserts; +import org.apache.hadoop.fs.impl.prefetch.ExecutorServiceFuturePool; +import org.apache.hadoop.fs.s3a.S3AInputStream; +import org.apache.hadoop.fs.s3a.S3AReadOpContext; +import org.apache.hadoop.fs.s3a.S3ObjectAttributes; +import org.apache.hadoop.fs.s3a.statistics.S3AInputStreamStatistics; +import org.apache.hadoop.test.AbstractHadoopTestBase; + +import static org.junit.Assert.assertEquals; + +/** + * Applies the same set of tests to both S3ACachingInputStream and S3AInMemoryInputStream. + */ +public class TestS3ARemoteInputStream extends AbstractHadoopTestBase { + + private static final int FILE_SIZE = 10; + + private final ExecutorService threadPool = Executors.newFixedThreadPool(4); + + private final ExecutorServiceFuturePool futurePool = + new ExecutorServiceFuturePool(threadPool); + + private final S3AInputStream.InputStreamCallbacks client = + MockS3ARemoteObject.createClient("bucket"); + + @Test + public void testArgChecks() throws Exception { + S3AReadOpContext readContext = + S3APrefetchFakes.createReadContext(futurePool, "key", 10, 10, 1); + S3ObjectAttributes attrs = + S3APrefetchFakes.createObjectAttributes("bucket", "key", 10); + S3AInputStreamStatistics stats = + readContext.getS3AStatisticsContext().newInputStreamStatistics(); + + // Should not throw. + new S3ACachingInputStream(readContext, attrs, client, stats); + + ExceptionAsserts.assertThrows( + NullPointerException.class, + () -> new S3ACachingInputStream(null, attrs, client, stats)); + + ExceptionAsserts.assertThrows( + NullPointerException.class, + () -> new S3ACachingInputStream(readContext, null, client, stats)); + + ExceptionAsserts.assertThrows( + NullPointerException.class, + () -> new S3ACachingInputStream(readContext, attrs, null, stats)); + + ExceptionAsserts.assertThrows( + NullPointerException.class, + () -> new S3ACachingInputStream(readContext, attrs, client, null)); + } + + @Test + public void testRead0SizedFile() throws Exception { + S3ARemoteInputStream inputStream = + S3APrefetchFakes.createS3InMemoryInputStream(futurePool, "bucket", + "key", 0); + testRead0SizedFileHelper(inputStream, 9); + + inputStream = + S3APrefetchFakes.createS3CachingInputStream(futurePool, "bucket", "key", + 0, 5, 2); + testRead0SizedFileHelper(inputStream, 5); + } + + private void testRead0SizedFileHelper(S3ARemoteInputStream inputStream, + int bufferSize) + throws Exception { + assertEquals(0, inputStream.available()); + assertEquals(-1, inputStream.read()); + assertEquals(-1, inputStream.read()); + + byte[] buffer = new byte[2]; + assertEquals(-1, inputStream.read(buffer)); + assertEquals(-1, inputStream.read()); + } + + @Test + public void testRead() throws Exception { + S3ARemoteInputStream inputStream = + S3APrefetchFakes.createS3InMemoryInputStream(futurePool, "bucket", + "key", FILE_SIZE); + testReadHelper(inputStream, FILE_SIZE); + + inputStream = + S3APrefetchFakes.createS3CachingInputStream(futurePool, "bucket", "key", + FILE_SIZE, 5, 2); + testReadHelper(inputStream, 5); + } + + private void testReadHelper(S3ARemoteInputStream inputStream, int bufferSize) + throws Exception { + assertEquals(bufferSize, inputStream.available()); + assertEquals(0, inputStream.read()); + assertEquals(1, inputStream.read()); + + byte[] buffer = new byte[2]; + assertEquals(2, inputStream.read(buffer)); + assertEquals(2, buffer[0]); + assertEquals(3, buffer[1]); + + assertEquals(4, inputStream.read()); + + buffer = new byte[10]; + int curPos = (int) inputStream.getPos(); + int expectedRemainingBytes = (int) (FILE_SIZE - curPos); + int readStartOffset = 2; + assertEquals( + expectedRemainingBytes, + inputStream.read(buffer, readStartOffset, expectedRemainingBytes)); + + for (int i = 0; i < expectedRemainingBytes; i++) { + assertEquals(curPos + i, buffer[readStartOffset + i]); + } + + assertEquals(-1, inputStream.read()); + Thread.sleep(100); + assertEquals(-1, inputStream.read()); + assertEquals(-1, inputStream.read()); + assertEquals(-1, inputStream.read(buffer)); + assertEquals(-1, inputStream.read(buffer, 1, 3)); + } + + @Test + public void testSeek() throws Exception { + S3ARemoteInputStream inputStream; + inputStream = + S3APrefetchFakes.createS3InMemoryInputStream(futurePool, "bucket", + "key", 9); + testSeekHelper(inputStream, 9, 9); + + inputStream = + S3APrefetchFakes.createS3CachingInputStream(futurePool, "bucket", "key", + 9, 5, 1); + testSeekHelper(inputStream, 5, 9); + } + + private void testSeekHelper(S3ARemoteInputStream inputStream, + int bufferSize, + int fileSize) + throws Exception { + assertEquals(0, inputStream.getPos()); + inputStream.seek(7); + assertEquals(7, inputStream.getPos()); + inputStream.seek(0); + + assertEquals(bufferSize, inputStream.available()); + for (int i = 0; i < fileSize; i++) { + assertEquals(i, inputStream.read()); + } + + for (int i = 0; i < fileSize; i++) { + inputStream.seek(i); + for (int j = i; j < fileSize; j++) { + assertEquals(j, inputStream.read()); + } + } + + // Test invalid seeks. + ExceptionAsserts.assertThrows( + EOFException.class, + FSExceptionMessages.NEGATIVE_SEEK, + () -> inputStream.seek(-1)); + } + + @Test + public void testRandomSeek() throws Exception { + S3ARemoteInputStream inputStream; + inputStream = + S3APrefetchFakes.createS3InMemoryInputStream(futurePool, "bucket", + "key", 9); + testRandomSeekHelper(inputStream, 9, 9); + + inputStream = + S3APrefetchFakes.createS3CachingInputStream(futurePool, "bucket", "key", + 9, 5, 1); + testRandomSeekHelper(inputStream, 5, 9); + } + + private void testRandomSeekHelper(S3ARemoteInputStream inputStream, + int bufferSize, + int fileSize) + throws Exception { + assertEquals(0, inputStream.getPos()); + inputStream.seek(7); + assertEquals(7, inputStream.getPos()); + inputStream.seek(0); + + assertEquals(bufferSize, inputStream.available()); + for (int i = 0; i < fileSize; i++) { + assertEquals(i, inputStream.read()); + } + + for (int i = 0; i < fileSize; i++) { + inputStream.seek(i); + for (int j = i; j < fileSize; j++) { + assertEquals(j, inputStream.read()); + } + + int seekFromEndPos = fileSize - i - 1; + inputStream.seek(seekFromEndPos); + for (int j = seekFromEndPos; j < fileSize; j++) { + assertEquals(j, inputStream.read()); + } + } + } + + @Test + public void testClose() throws Exception { + S3ARemoteInputStream inputStream = + S3APrefetchFakes.createS3InMemoryInputStream(futurePool, "bucket", + "key", 9); + testCloseHelper(inputStream, 9); + + inputStream = + S3APrefetchFakes.createS3CachingInputStream(futurePool, "bucket", "key", + 9, 5, 3); + testCloseHelper(inputStream, 5); + } + + private void testCloseHelper(S3ARemoteInputStream inputStream, int bufferSize) + throws Exception { + assertEquals(bufferSize, inputStream.available()); + assertEquals(0, inputStream.read()); + assertEquals(1, inputStream.read()); + + inputStream.close(); + + ExceptionAsserts.assertThrows( + IOException.class, + FSExceptionMessages.STREAM_IS_CLOSED, + () -> inputStream.available()); + + ExceptionAsserts.assertThrows( + IOException.class, + FSExceptionMessages.STREAM_IS_CLOSED, + () -> inputStream.read()); + + byte[] buffer = new byte[10]; + ExceptionAsserts.assertThrows( + IOException.class, + FSExceptionMessages.STREAM_IS_CLOSED, + () -> inputStream.read(buffer)); + + // Verify a second close() does not throw. + inputStream.close(); + } +} diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/prefetch/TestS3ARemoteObject.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/prefetch/TestS3ARemoteObject.java new file mode 100644 index 0000000000000..b3788aac80834 --- /dev/null +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/prefetch/TestS3ARemoteObject.java @@ -0,0 +1,88 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hadoop.fs.s3a.prefetch; + +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; + +import org.junit.Test; + +import org.apache.hadoop.fs.impl.prefetch.ExceptionAsserts; +import org.apache.hadoop.fs.impl.prefetch.ExecutorServiceFuturePool; +import org.apache.hadoop.fs.s3a.S3AInputStream; +import org.apache.hadoop.fs.s3a.S3AReadOpContext; +import org.apache.hadoop.fs.s3a.S3ObjectAttributes; +import org.apache.hadoop.fs.s3a.impl.ChangeTracker; +import org.apache.hadoop.fs.s3a.statistics.S3AInputStreamStatistics; +import org.apache.hadoop.test.AbstractHadoopTestBase; + +public class TestS3ARemoteObject extends AbstractHadoopTestBase { + + private final ExecutorService threadPool = Executors.newFixedThreadPool(1); + + private final ExecutorServiceFuturePool futurePool = + new ExecutorServiceFuturePool(threadPool); + + private final S3AInputStream.InputStreamCallbacks client = + MockS3ARemoteObject.createClient("bucket"); + + @Test + public void testArgChecks() throws Exception { + S3AReadOpContext readContext = + S3APrefetchFakes.createReadContext(futurePool, "key", 10, 10, 1); + S3ObjectAttributes attrs = + S3APrefetchFakes.createObjectAttributes("bucket", "key", 10); + S3AInputStreamStatistics stats = + readContext.getS3AStatisticsContext().newInputStreamStatistics(); + ChangeTracker changeTracker = + S3APrefetchFakes.createChangeTracker("bucket", "key", 10); + + // Should not throw. + new S3ARemoteObject(readContext, attrs, client, stats, changeTracker); + + ExceptionAsserts.assertThrows( + IllegalArgumentException.class, + "'context' must not be null", + () -> new S3ARemoteObject(null, attrs, client, stats, changeTracker)); + + ExceptionAsserts.assertThrows( + IllegalArgumentException.class, + "'s3Attributes' must not be null", + () -> new S3ARemoteObject(readContext, null, client, stats, + changeTracker)); + + ExceptionAsserts.assertThrows( + IllegalArgumentException.class, + "'client' must not be null", + () -> new S3ARemoteObject(readContext, attrs, null, stats, + changeTracker)); + + ExceptionAsserts.assertThrows( + IllegalArgumentException.class, + "'streamStatistics' must not be null", + () -> new S3ARemoteObject(readContext, attrs, client, null, + changeTracker)); + + ExceptionAsserts.assertThrows( + IllegalArgumentException.class, + "'changeTracker' must not be null", + () -> new S3ARemoteObject(readContext, attrs, client, stats, null)); + } +} diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/prefetch/TestS3ARemoteObjectReader.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/prefetch/TestS3ARemoteObjectReader.java new file mode 100644 index 0000000000000..db70c4f22bce9 --- /dev/null +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/prefetch/TestS3ARemoteObjectReader.java @@ -0,0 +1,112 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hadoop.fs.s3a.prefetch; + +import java.nio.ByteBuffer; + +import org.junit.Test; + +import org.apache.hadoop.test.AbstractHadoopTestBase; + +import static org.apache.hadoop.test.LambdaTestUtils.intercept; +import static org.junit.Assert.assertEquals; + +public class TestS3ARemoteObjectReader extends AbstractHadoopTestBase { + + private static final int FILE_SIZE = 9; + + private static final int BUFFER_SIZE = 2; + + private final S3ARemoteObject remoteObject = + new MockS3ARemoteObject(FILE_SIZE, false); + + @Test + public void testArgChecks() throws Exception { + // Should not throw. + S3ARemoteObjectReader reader = new S3ARemoteObjectReader(remoteObject); + + // Verify it throws correctly. + + intercept( + IllegalArgumentException.class, + "'remoteObject' must not be null", + () -> new S3ARemoteObjectReader(null)); + + intercept( + IllegalArgumentException.class, + "'buffer' must not be null", + () -> reader.read(null, 10, 2)); + + ByteBuffer buffer = ByteBuffer.allocate(BUFFER_SIZE); + + intercept( + IllegalArgumentException.class, + "'offset' (-1) must be within the range [0, 9]", + () -> reader.read(buffer, -1, 2)); + + intercept( + IllegalArgumentException.class, + "'offset' (11) must be within the range [0, 9]", + () -> reader.read(buffer, 11, 2)); + + intercept( + IllegalArgumentException.class, + "'size' must be a positive integer", + () -> reader.read(buffer, 1, 0)); + + intercept( + IllegalArgumentException.class, + "'size' must be a positive integer", + () -> reader.read(buffer, 1, -1)); + } + + @Test + public void testGetWithOffset() throws Exception { + for (int i = 0; i < FILE_SIZE; i++) { + testGetHelper(false, i); // no retry + testGetHelper(true, i); // with retry + } + } + + private void testGetHelper(boolean testWithRetry, long startOffset) + throws Exception { + int numBlocks = 0; + ByteBuffer buffer; + S3ARemoteObjectReader reader = + new S3ARemoteObjectReader( + new MockS3ARemoteObject(FILE_SIZE, testWithRetry)); + int remainingSize = FILE_SIZE - (int) startOffset; + for (int bufferSize = 0; bufferSize <= FILE_SIZE + 1; bufferSize++) { + buffer = ByteBuffer.allocate(bufferSize); + for (int readSize = 1; readSize <= FILE_SIZE; readSize++) { + buffer.clear(); + int numBytesRead = reader.read(buffer, startOffset, readSize); + int expectedNumBytesRead = Math.min(readSize, remainingSize); + expectedNumBytesRead = Math.min(bufferSize, expectedNumBytesRead); + assertEquals(expectedNumBytesRead, numBytesRead); + + byte[] bytes = buffer.array(); + for (int i = 0; i < expectedNumBytesRead; i++) { + assertEquals(startOffset + i, bytes[i]); + } + } + } + } +} diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/ILoadTestS3ABulkDeleteThrottling.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/ILoadTestS3ABulkDeleteThrottling.java index 6e51b7388691c..813eea8389f64 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/ILoadTestS3ABulkDeleteThrottling.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/ILoadTestS3ABulkDeleteThrottling.java @@ -45,6 +45,7 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.contract.ContractTestUtils; +import org.apache.hadoop.fs.store.audit.AuditSpan; import org.apache.hadoop.fs.s3a.S3AFileSystem; import org.apache.hadoop.fs.s3a.S3ATestUtils; import org.apache.hadoop.fs.s3a.auth.delegation.Csvout; @@ -144,26 +145,25 @@ public ILoadTestS3ABulkDeleteThrottling( @Override protected Configuration createScaleConfiguration() { Configuration conf = super.createScaleConfiguration(); - S3ATestUtils.disableFilesystemCaching(conf); - return conf; - } - @Override - public void setup() throws Exception { - final Configuration conf = getConf(); S3ATestUtils.removeBaseAndBucketOverrides(conf, EXPERIMENTAL_AWS_INTERNAL_THROTTLING, BULK_DELETE_PAGE_SIZE, - USER_AGENT_PREFIX); + USER_AGENT_PREFIX, + ENABLE_MULTI_DELETE); conf.setBoolean(EXPERIMENTAL_AWS_INTERNAL_THROTTLING, throttle); - Assertions.assertThat(pageSize) - .describedAs("page size") - .isGreaterThan(0); conf.setInt(BULK_DELETE_PAGE_SIZE, pageSize); conf.set(USER_AGENT_PREFIX, String.format("ILoadTestS3ABulkDeleteThrottling-%s-%04d", throttle, pageSize)); + S3ATestUtils.disableFilesystemCaching(conf); + return conf; + } + + @Override + public void setup() throws Exception { + final Configuration conf = getConf(); super.setup(); Assume.assumeTrue("multipart delete disabled", conf.getBoolean(ENABLE_MULTI_DELETE, true)); @@ -246,7 +246,7 @@ private File deleteFiles(final int requestCount, final ContractTestUtils.NanoTimer timer = new ContractTestUtils.NanoTimer(); Exception ex = null; - try { + try (AuditSpan span = span()) { fs.removeKeys(fileList, false); } catch (IOException e) { ex = e; diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/ITestS3AInputStreamPerformance.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/ITestS3AInputStreamPerformance.java index d73a938bcceea..eea70ced13c92 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/ITestS3AInputStreamPerformance.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/ITestS3AInputStreamPerformance.java @@ -28,6 +28,7 @@ import org.apache.hadoop.fs.s3a.S3AFileSystem; import org.apache.hadoop.fs.s3a.S3AInputPolicy; import org.apache.hadoop.fs.s3a.S3AInputStream; +import org.apache.hadoop.fs.s3a.S3ATestUtils; import org.apache.hadoop.fs.s3a.statistics.S3AInputStreamStatistics; import org.apache.hadoop.fs.statistics.IOStatistics; import org.apache.hadoop.fs.statistics.IOStatisticsSnapshot; @@ -56,6 +57,8 @@ import static org.apache.hadoop.fs.contract.ContractTestUtils.*; import static org.apache.hadoop.fs.s3a.Constants.*; import static org.apache.hadoop.fs.s3a.S3ATestUtils.assume; +import static org.apache.hadoop.fs.s3a.S3ATestUtils.getInputStreamStatistics; +import static org.apache.hadoop.fs.s3a.S3ATestUtils.getS3AInputStream; import static org.apache.hadoop.fs.statistics.IOStatisticAssertions.assertThatStatisticMinimum; import static org.apache.hadoop.fs.statistics.IOStatisticAssertions.lookupMaximumStatistic; import static org.apache.hadoop.fs.statistics.IOStatisticAssertions.lookupMeanStatistic; @@ -92,6 +95,14 @@ public class ITestS3AInputStreamPerformance extends S3AScaleTestBase { private boolean testDataAvailable = true; private String assumptionMessage = "test file"; + @Override + protected Configuration createScaleConfiguration() { + Configuration conf = super.createScaleConfiguration(); + S3ATestUtils.removeBaseAndBucketOverrides(conf, PREFETCH_ENABLED_KEY); + conf.setBoolean(PREFETCH_ENABLED_KEY, false); + return conf; + } + /** * Open the FS and the test data. The input stream is always set up here. * @throws IOException IO Problems. diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/S3AScaleTestBase.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/S3AScaleTestBase.java index d95b46b10d786..514c6cf886918 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/S3AScaleTestBase.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/S3AScaleTestBase.java @@ -19,19 +19,14 @@ package org.apache.hadoop.fs.s3a.scale; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FSDataInputStream; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.s3a.AbstractS3ATestBase; -import org.apache.hadoop.fs.s3a.S3AInputStream; import org.apache.hadoop.fs.s3a.S3ATestConstants; import org.apache.hadoop.fs.s3a.Statistic; -import org.apache.hadoop.fs.s3a.statistics.S3AInputStreamStatistics; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.io.InputStream; - import static org.apache.hadoop.fs.s3a.S3ATestUtils.*; import static org.apache.hadoop.fs.statistics.IOStatisticAssertions.lookupGaugeStatistic; @@ -154,34 +149,6 @@ protected int getTestTimeoutMillis() { return getTestTimeoutSeconds() * 1000; } - /** - * Get the input stream statistics of an input stream. - * Raises an exception if the inner stream is not an S3A input stream - * @param in wrapper - * @return the statistics for the inner stream - */ - protected S3AInputStreamStatistics getInputStreamStatistics( - FSDataInputStream in) { - return getS3AInputStream(in).getS3AStreamStatistics(); - } - - /** - * Get the inner stream of an input stream. - * Raises an exception if the inner stream is not an S3A input stream - * @param in wrapper - * @return the inner stream - * @throws AssertionError if the inner stream is of the wrong type - */ - protected S3AInputStream getS3AInputStream( - FSDataInputStream in) { - InputStream inner = in.getWrappedStream(); - if (inner instanceof S3AInputStream) { - return (S3AInputStream) inner; - } else { - throw new AssertionError("Not an S3AInputStream: " + inner); - } - } - /** * Get the gauge value of a statistic from the * IOStatistics of the filesystem. Raises an assertion if diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/select/ITestS3SelectLandsat.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/select/ITestS3SelectLandsat.java index 8b0578df11c01..51da971fb7063 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/select/ITestS3SelectLandsat.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/select/ITestS3SelectLandsat.java @@ -415,7 +415,7 @@ public void testSelectSeekFullLandsat() throws Throwable { long increment = 64 * _1KB; // seek forward, comparing bytes - for(offset = 32 * _1KB; offset < actualLen; offset += increment) { + for(offset = 32 * _1KB; offset < _1MB; offset += increment) { seek(seekStream, offset); assertEquals("Seek position in " + seekStream, offset, seekStream.getPos()); @@ -423,11 +423,6 @@ public void testSelectSeekFullLandsat() throws Throwable { assertEquals("byte at seek position", dataset[(int) seekStream.getPos()], seekStream.read()); } - for(; offset < len; offset += _1MB) { - seek(seekStream, offset); - assertEquals("Seek position in " + seekStream, - offset, seekStream.getPos()); - } // there's no knowledge of how much data is left, but with Gzip // involved there can be a lot. To keep the test duration down, // this test, unlike the simpler one, doesn't try to read past the diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/test/MinimalWriteOperationHelperCallbacks.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/test/MinimalWriteOperationHelperCallbacks.java new file mode 100644 index 0000000000000..ffba558d11fd0 --- /dev/null +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/test/MinimalWriteOperationHelperCallbacks.java @@ -0,0 +1,46 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.s3a.test; + +import com.amazonaws.services.s3.model.CompleteMultipartUploadRequest; +import com.amazonaws.services.s3.model.CompleteMultipartUploadResult; +import com.amazonaws.services.s3.model.SelectObjectContentRequest; +import com.amazonaws.services.s3.model.SelectObjectContentResult; + +import org.apache.hadoop.fs.s3a.WriteOperationHelper; + +/** + * Stub implementation of writeOperationHelper callbacks. + */ +public class MinimalWriteOperationHelperCallbacks + implements WriteOperationHelper.WriteOperationHelperCallbacks { + + @Override + public SelectObjectContentResult selectObjectContent(SelectObjectContentRequest request) { + return null; + } + + @Override + public CompleteMultipartUploadResult completeMultipartUpload( + CompleteMultipartUploadRequest request) { + return null; + } + +}; + diff --git a/hadoop-tools/hadoop-aws/src/test/resources/core-site.xml b/hadoop-tools/hadoop-aws/src/test/resources/core-site.xml index 1525f51d9d6a8..1533477229642 100644 --- a/hadoop-tools/hadoop-aws/src/test/resources/core-site.xml +++ b/hadoop-tools/hadoop-aws/src/test/resources/core-site.xml @@ -178,12 +178,22 @@ true + + + fs.iostatistics.thread.level.enabled + true + + + + fs.s3a.connection.request.timeout + 10s + + diff --git a/hadoop-tools/hadoop-aws/src/test/resources/log4j.properties b/hadoop-tools/hadoop-aws/src/test/resources/log4j.properties index c831999008fec..0ec8d52042807 100644 --- a/hadoop-tools/hadoop-aws/src/test/resources/log4j.properties +++ b/hadoop-tools/hadoop-aws/src/test/resources/log4j.properties @@ -55,6 +55,7 @@ log4j.logger.org.apache.hadoop.ipc.Server=WARN log4j.logger.org.apache.hadoop.fs.s3a=DEBUG #log4j.logger.org.apache.hadoop.fs.s3a.S3AUtils=INFO #log4j.logger.org.apache.hadoop.fs.s3a.Listing=INFO +log4j.logger.org.apache.hadoop.fs.s3a.SDKV2Upgrade=WARN # Log Committer classes #log4j.logger.org.apache.hadoop.fs.s3a.commit=DEBUG diff --git a/hadoop-tools/hadoop-azure/pom.xml b/hadoop-tools/hadoop-azure/pom.xml index 40aeec07026cc..c313fa28a3ab4 100644 --- a/hadoop-tools/hadoop-azure/pom.xml +++ b/hadoop-tools/hadoop-azure/pom.xml @@ -604,6 +604,7 @@ **/azurebfs/ITestAzureBlobFileSystemListStatus.java **/azurebfs/extensions/ITestAbfsDelegationTokens.java **/azurebfs/ITestSmallWriteOptimization.java + **/azurebfs/ITestAbfsStreamStatistics*.java **/azurebfs/services/ITestReadBufferManager.java **/azurebfs/commit/*.java @@ -647,6 +648,7 @@ **/azurebfs/extensions/ITestAbfsDelegationTokens.java **/azurebfs/ITestSmallWriteOptimization.java **/azurebfs/services/ITestReadBufferManager.java + **/azurebfs/ITestAbfsStreamStatistics*.java **/azurebfs/commit/*.java diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/oauth2/CustomTokenProviderAdapter.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/oauth2/CustomTokenProviderAdapter.java index a4ec35e2f69ef..889976041d46e 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/oauth2/CustomTokenProviderAdapter.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/oauth2/CustomTokenProviderAdapter.java @@ -22,6 +22,7 @@ import java.io.IOException; import java.net.URI; +import org.apache.hadoop.classification.VisibleForTesting; import org.apache.hadoop.util.Preconditions; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -138,4 +139,9 @@ public String getUserAgentSuffix() { String suffix = ExtensionHelper.getUserAgentSuffix(adaptee, ""); return suffix != null ? suffix : ""; } + + @VisibleForTesting + protected CustomTokenProviderAdaptee getCustomTokenProviderAdaptee() { + return adaptee; + } } diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsClient.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsClient.java index 0c0660c6f60bf..aa72ed64e6e5d 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsClient.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsClient.java @@ -1289,4 +1289,9 @@ public ListenableFuture submit(Runnable runnable) { public void addCallback(ListenableFuture future, FutureCallback callback) { Futures.addCallback(future, callback, executorService); } + + @VisibleForTesting + protected AccessTokenProvider getTokenProvider() { + return tokenProvider; + } } diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/AbstractAbfsIntegrationTest.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/AbstractAbfsIntegrationTest.java index 4a5507526c3a1..c5bf85a4f8190 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/AbstractAbfsIntegrationTest.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/AbstractAbfsIntegrationTest.java @@ -37,10 +37,12 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.azurebfs.constants.FSOperationType; import org.apache.hadoop.fs.azurebfs.contracts.exceptions.AzureBlobFileSystemException; +import org.apache.hadoop.fs.azurebfs.oauth2.AccessTokenProvider; import org.apache.hadoop.fs.azurebfs.security.AbfsDelegationTokenManager; import org.apache.hadoop.fs.azurebfs.services.AbfsClient; import org.apache.hadoop.fs.azurebfs.services.AbfsOutputStream; import org.apache.hadoop.fs.azurebfs.services.AuthType; +import org.apache.hadoop.fs.azurebfs.services.TestAbfsClient; import org.apache.hadoop.fs.azure.AzureNativeFileSystemStore; import org.apache.hadoop.fs.azure.NativeAzureFileSystem; import org.apache.hadoop.fs.azure.metrics.AzureFileSystemInstrumentation; @@ -149,6 +151,17 @@ protected boolean getIsNamespaceEnabled(AzureBlobFileSystem fs) return fs.getIsNamespaceEnabled(getTestTracingContext(fs, false)); } + public static TracingContext getSampleTracingContext(AzureBlobFileSystem fs, + boolean needsPrimaryReqId) { + String correlationId, fsId; + TracingHeaderFormat format; + correlationId = "test-corr-id"; + fsId = "test-filesystem-id"; + format = TracingHeaderFormat.ALL_ID_FORMAT; + return new TracingContext(correlationId, fsId, + FSOperationType.TEST_OP, needsPrimaryReqId, format, null); + } + public TracingContext getTestTracingContext(AzureBlobFileSystem fs, boolean needsPrimaryReqId) { String correlationId, fsId; @@ -167,7 +180,6 @@ public TracingContext getTestTracingContext(AzureBlobFileSystem fs, FSOperationType.TEST_OP, needsPrimaryReqId, format, null); } - @Before public void setup() throws Exception { //Create filesystem first to make sure getWasbFileSystem() can return an existing filesystem. @@ -241,6 +253,9 @@ public Hashtable call() throws Exception { } } + public AccessTokenProvider getAccessTokenProvider(final AzureBlobFileSystem fs) { + return TestAbfsClient.getAccessTokenProvider(fs.getAbfsStore().getClient()); + } public void loadConfiguredFileSystem() throws Exception { // disable auto-creation of filesystem diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAbfsRestOperationException.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAbfsRestOperationException.java index ce9415a81795c..3fe3557d501dc 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAbfsRestOperationException.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAbfsRestOperationException.java @@ -112,7 +112,10 @@ public void testWithDifferentCustomTokenFetchRetry(int numOfRetries) throws Exce final AzureBlobFileSystem fs1 = (AzureBlobFileSystem) FileSystem.newInstance(fs.getUri(), config); - RetryTestTokenProvider.ResetStatusToFirstTokenFetch(); + RetryTestTokenProvider retryTestTokenProvider + = RetryTestTokenProvider.getCurrentRetryTestProviderInstance( + getAccessTokenProvider(fs1)); + retryTestTokenProvider.resetStatusToFirstTokenFetch(); intercept(Exception.class, ()-> { @@ -120,10 +123,10 @@ public void testWithDifferentCustomTokenFetchRetry(int numOfRetries) throws Exce }); // Number of retries done should be as configured - Assert.assertTrue( - "Number of token fetch retries (" + RetryTestTokenProvider.reTryCount - + ") done, does not match with fs.azure.custom.token.fetch.retry.count configured (" + numOfRetries - + ")", RetryTestTokenProvider.reTryCount == numOfRetries); + Assert.assertEquals( + "Number of token fetch retries done does not match with fs.azure" + + ".custom.token.fetch.retry.count configured", numOfRetries, + retryTestTokenProvider.getRetryCount()); } @Test diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/commit/ITestAbfsRenameStageFailure.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/commit/ITestAbfsRenameStageFailure.java index 5547d081c963f..6b9830e8f33fc 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/commit/ITestAbfsRenameStageFailure.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/commit/ITestAbfsRenameStageFailure.java @@ -19,11 +19,13 @@ package org.apache.hadoop.fs.azurebfs.commit; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.azurebfs.AbstractAbfsIntegrationTest; +import org.apache.hadoop.fs.azurebfs.AzureBlobFileSystem; import org.apache.hadoop.fs.azurebfs.contract.ABFSContractTestBinding; import org.apache.hadoop.fs.azurebfs.contract.AbfsFileSystemContract; +import org.apache.hadoop.fs.azurebfs.contracts.exceptions.AzureBlobFileSystemException; import org.apache.hadoop.fs.contract.AbstractFSContract; import org.apache.hadoop.mapreduce.lib.output.committer.manifest.TestRenameStageFailure; - /** * Rename failure logic on ABFS. * This will go through the resilient rename operation. @@ -41,6 +43,11 @@ public ITestAbfsRenameStageFailure() throws Exception { binding = new ABFSContractTestBinding(); } + protected boolean isNamespaceEnabled() throws AzureBlobFileSystemException { + AzureBlobFileSystem fs = (AzureBlobFileSystem) getFileSystem(); + return fs.getAbfsStore().getIsNamespaceEnabled(AbstractAbfsIntegrationTest.getSampleTracingContext(fs, false)); + } + @Override public void setup() throws Exception { binding.setup(); @@ -58,8 +65,8 @@ protected AbstractFSContract createContract(final Configuration conf) { } @Override - protected boolean requireRenameResilience() { - return true; + protected boolean requireRenameResilience() throws AzureBlobFileSystemException { + return isNamespaceEnabled(); } @Override diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/oauth2/RetryTestTokenProvider.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/oauth2/RetryTestTokenProvider.java index 3566ebbaaaa2a..7427add29086c 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/oauth2/RetryTestTokenProvider.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/oauth2/RetryTestTokenProvider.java @@ -30,12 +30,12 @@ */ public class RetryTestTokenProvider implements CustomTokenProviderAdaptee { - // Need to track first token fetch otherwise will get counted as a retry too. - private static boolean isThisFirstTokenFetch = true; - public static int reTryCount = 0; + private static final Logger LOG = LoggerFactory.getLogger( + RetryTestTokenProvider.class); - private static final Logger LOG = LoggerFactory - .getLogger(RetryTestTokenProvider.class); + // Need to track first token fetch otherwise will get counted as a retry too. + private boolean isThisFirstTokenFetch = true; + private int retryCount = 0; @Override public void initialize(Configuration configuration, String accountName) @@ -43,9 +43,13 @@ public void initialize(Configuration configuration, String accountName) } - public static void ResetStatusToFirstTokenFetch() { + /** + * Clear earlier retry details and reset RetryTestTokenProvider instance to + * state of first access token fetch call. + */ + public void resetStatusToFirstTokenFetch() { isThisFirstTokenFetch = true; - reTryCount = 0; + retryCount = 0; } @Override @@ -53,7 +57,7 @@ public String getAccessToken() throws IOException { if (isThisFirstTokenFetch) { isThisFirstTokenFetch = false; } else { - reTryCount++; + retryCount++; } LOG.debug("RetryTestTokenProvider: Throw an exception in fetching tokens"); @@ -64,4 +68,13 @@ public String getAccessToken() throws IOException { public Date getExpiryTime() { return new Date(); } + + public static RetryTestTokenProvider getCurrentRetryTestProviderInstance( + AccessTokenProvider customTokenProvider) { + return (RetryTestTokenProvider) ((CustomTokenProviderAdapter) customTokenProvider).getCustomTokenProviderAdaptee(); + } + + public int getRetryCount() { + return retryCount; + } } diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/TestAbfsClient.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/TestAbfsClient.java index a725bf3175a5c..0a1dca7e7d8d7 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/TestAbfsClient.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/TestAbfsClient.java @@ -395,4 +395,8 @@ public static AbfsRestOperation getRestOp(AbfsRestOperationType type, url, requestHeaders); } + + public static AccessTokenProvider getAccessTokenProvider(AbfsClient client) { + return client.getTokenProvider(); + } } diff --git a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCp.java b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCp.java index 892bffa7db9bc..141f45d61f372 100644 --- a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCp.java +++ b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCp.java @@ -473,13 +473,18 @@ private static Configuration getDefaultConf() { return config; } - private synchronized void cleanup() { + /** + * Clean the staging folder created by distcp. + */ + protected synchronized void cleanup() { try { if (metaFolder != null) { - if (jobFS != null) { - jobFS.delete(metaFolder, true); + synchronized (this) { + if (jobFS != null) { + jobFS.delete(metaFolder, true); + } + metaFolder = null; } - metaFolder = null; } } catch (IOException e) { LOG.error("Unable to cleanup meta folder: " + metaFolder, e); diff --git a/hadoop-yarn-project/hadoop-yarn/bin/FederationStateStore/MySQL/FederationStateStoreStoredProcs.sql b/hadoop-yarn-project/hadoop-yarn/bin/FederationStateStore/MySQL/FederationStateStoreStoredProcs.sql index eae882e4a48dd..9434ed3848805 100644 --- a/hadoop-yarn-project/hadoop-yarn/bin/FederationStateStore/MySQL/FederationStateStoreStoredProcs.sql +++ b/hadoop-yarn-project/hadoop-yarn/bin/FederationStateStore/MySQL/FederationStateStoreStoredProcs.sql @@ -159,4 +159,53 @@ BEGIN FROM policies WHERE queue = queue_IN; END // +CREATE PROCEDURE sp_addReservationHomeSubCluster( + IN reservationId_IN varchar(128), IN homeSubCluster_IN varchar(256), + OUT storedHomeSubCluster_OUT varchar(256), OUT rowCount_OUT int) +BEGIN + INSERT INTO reservationsHomeSubCluster + (reservationId,homeSubCluster) + (SELECT reservationId_IN, homeSubCluster_IN + FROM applicationsHomeSubCluster + WHERE reservationId = reservationId_IN + HAVING COUNT(*) = 0 ); + SELECT ROW_COUNT() INTO rowCount_OUT; + SELECT homeSubCluster INTO storedHomeSubCluster_OUT + FROM reservationsHomeSubCluster + WHERE applicationId = reservationId_IN; +END // + +CREATE PROCEDURE sp_getReservationHomeSubCluster( + IN reservationId_IN varchar(128), + OUT homeSubCluster_OUT varchar(256)) +BEGIN + SELECT homeSubCluster INTO homeSubCluster_OUT + FROM reservationsHomeSubCluster + WHERE reservationId = reservationId_IN; +END // + +CREATE PROCEDURE sp_getReservationsHomeSubCluster() +BEGIN + SELECT reservationId, homeSubCluster + FROM reservationsHomeSubCluster; +END // + +CREATE PROCEDURE sp_updateReservationHomeSubCluster( + IN reservationId_IN varchar(128), + IN homeSubCluster_IN varchar(256), OUT rowCount_OUT int) +BEGIN + UPDATE reservationsHomeSubCluster + SET homeSubCluster = homeSubCluster_IN + WHERE reservationId = reservationId_IN; + SELECT ROW_COUNT() INTO rowCount_OUT; +END // + +CREATE PROCEDURE sp_deleteReservationHomeSubCluster( + IN reservationId_IN varchar(128), OUT rowCount_OUT int) +BEGIN + DELETE FROM reservationsHomeSubCluster + WHERE reservationId = reservationId_IN; + SELECT ROW_COUNT() INTO rowCount_OUT; +END // + DELIMITER ; diff --git a/hadoop-yarn-project/hadoop-yarn/bin/FederationStateStore/MySQL/FederationStateStoreTables.sql b/hadoop-yarn-project/hadoop-yarn/bin/FederationStateStore/MySQL/FederationStateStoreTables.sql index 3a255f06bc3f8..6a3188bab6eab 100644 --- a/hadoop-yarn-project/hadoop-yarn/bin/FederationStateStore/MySQL/FederationStateStoreTables.sql +++ b/hadoop-yarn-project/hadoop-yarn/bin/FederationStateStore/MySQL/FederationStateStoreTables.sql @@ -46,3 +46,9 @@ CREATE TABLE policies( params varbinary(32768), CONSTRAINT pk_queue PRIMARY KEY (queue) ); + +CREATE TABLE reservationsHomeSubCluster ( + reservationId varchar(128) NOT NULL, + homeSubCluster varchar(256) NOT NULL, + CONSTRAINT pk_reservationId PRIMARY KEY (reservationId) +); \ No newline at end of file diff --git a/hadoop-yarn-project/hadoop-yarn/bin/FederationStateStore/MySQL/dropStoreProcedures.sql b/hadoop-yarn-project/hadoop-yarn/bin/FederationStateStore/MySQL/dropStoreProcedures.sql index f24f3fb22b55c..a2f0b882b3ff6 100644 --- a/hadoop-yarn-project/hadoop-yarn/bin/FederationStateStore/MySQL/dropStoreProcedures.sql +++ b/hadoop-yarn-project/hadoop-yarn/bin/FederationStateStore/MySQL/dropStoreProcedures.sql @@ -45,3 +45,13 @@ DROP PROCEDURE sp_setPolicyConfiguration; DROP PROCEDURE sp_getPolicyConfiguration; DROP PROCEDURE sp_getPoliciesConfigurations; + +DROP PROCEDURE sp_addReservationHomeSubCluster; + +DROP PROCEDURE sp_getReservationHomeSubCluster; + +DROP PROCEDURE sp_getReservationsHomeSubCluster; + +DROP PROCEDURE sp_deleteReservationHomeSubCluster; + +DROP PROCEDURE sp_updateReservationHomeSubCluster; diff --git a/hadoop-yarn-project/hadoop-yarn/bin/FederationStateStore/MySQL/dropTables.sql b/hadoop-yarn-project/hadoop-yarn/bin/FederationStateStore/MySQL/dropTables.sql index ea6567b028b81..d29f8652c153e 100644 --- a/hadoop-yarn-project/hadoop-yarn/bin/FederationStateStore/MySQL/dropTables.sql +++ b/hadoop-yarn-project/hadoop-yarn/bin/FederationStateStore/MySQL/dropTables.sql @@ -25,3 +25,5 @@ DROP TABLE applicationsHomeSubCluster; DROP TABLE membership; DROP TABLE policies; + +DROP TABLE reservationsHomeSubCluster; diff --git a/hadoop-yarn-project/hadoop-yarn/bin/FederationStateStore/SQLServer/FederationStateStoreStoreProcs.sql b/hadoop-yarn-project/hadoop-yarn/bin/FederationStateStore/SQLServer/FederationStateStoreStoreProcs.sql index 66d6f0e203558..ab17aae4f88d9 100644 --- a/hadoop-yarn-project/hadoop-yarn/bin/FederationStateStore/SQLServer/FederationStateStoreStoreProcs.sql +++ b/hadoop-yarn-project/hadoop-yarn/bin/FederationStateStore/SQLServer/FederationStateStoreStoreProcs.sql @@ -508,4 +508,181 @@ AS BEGIN ) WITH log END CATCH END; +GO + +IF OBJECT_ID ( '[sp_addApplicationHomeSubCluster]', 'P' ) IS NOT NULL + DROP PROCEDURE [sp_addApplicationHomeSubCluster]; +GO + +CREATE PROCEDURE [dbo].[sp_addReservationHomeSubCluster] + @reservationId VARCHAR(128), + @homeSubCluster VARCHAR(256), + @storedHomeSubCluster VARCHAR(256) OUTPUT, + @rowCount int OUTPUT +AS BEGIN + DECLARE @errorMessage nvarchar(4000) + + BEGIN TRY + BEGIN TRAN + -- If application to sub-cluster map doesn't exist, insert it. + -- Otherwise don't change the current mapping. + IF NOT EXISTS (SELECT TOP 1 * + FROM [dbo].[reservationsHomeSubCluster] + WHERE [reservationId] = @reservationId) + + INSERT INTO [dbo].[reservationsHomeSubCluster] ( + [reservationId], + [homeSubCluster]) + VALUES ( + @reservationId, + @homeSubCluster); + -- End of the IF block + + SELECT @rowCount = @@ROWCOUNT; + + SELECT @storedHomeSubCluster = [homeSubCluster] + FROM [dbo].[reservationsHomeSubCluster] + WHERE [reservationId] = @reservationId; + + COMMIT TRAN + END TRY + + BEGIN CATCH + ROLLBACK TRAN + + SET @errorMessage = dbo.func_FormatErrorMessage(ERROR_MESSAGE(), ERROR_LINE()) + + /* raise error and terminate the execution */ + RAISERROR(@errorMessage, --- Error Message + 1, -- Severity + -1 -- State + ) WITH log + END CATCH +END; +GO + +IF OBJECT_ID ( '[sp_updateReservationHomeSubCluster]', 'P' ) IS NOT NULL + DROP PROCEDURE [sp_updateReservationHomeSubCluster]; +GO + +CREATE PROCEDURE [dbo].[sp_updateReservationHomeSubCluster] + @reservationId VARCHAR(128), + @homeSubCluster VARCHAR(256), + @rowCount int OUTPUT +AS BEGIN + DECLARE @errorMessage nvarchar(4000) + + BEGIN TRY + BEGIN TRAN + + UPDATE [dbo].[reservationsHomeSubCluster] + SET [homeSubCluster] = @homeSubCluster + WHERE [reservationId] = @reservationId; + SELECT @rowCount = @@ROWCOUNT; + + COMMIT TRAN + END TRY + + BEGIN CATCH + ROLLBACK TRAN + + SET @errorMessage = dbo.func_FormatErrorMessage(ERROR_MESSAGE(), ERROR_LINE()) + + /* raise error and terminate the execution */ + RAISERROR(@errorMessage, --- Error Message + 1, -- Severity + -1 -- State + ) WITH log + END CATCH +END; +GO + +IF OBJECT_ID ( '[sp_getReservationsHomeSubCluster]', 'P' ) IS NOT NULL + DROP PROCEDURE [sp_getReservationsHomeSubCluster]; +GO + +CREATE PROCEDURE [dbo].[sp_getReservationsHomeSubCluster] +AS BEGIN + DECLARE @errorMessage nvarchar(4000) + + BEGIN TRY + SELECT [reservationId], [homeSubCluster], [createTime] + FROM [dbo].[reservationsHomeSubCluster] + END TRY + + BEGIN CATCH + SET @errorMessage = dbo.func_FormatErrorMessage(ERROR_MESSAGE(), ERROR_LINE()) + + /* raise error and terminate the execution */ + RAISERROR(@errorMessage, --- Error Message + 1, -- Severity + -1 -- State + ) WITH log + END CATCH +END; +GO + +IF OBJECT_ID ( '[sp_getReservationHomeSubCluster]', 'P' ) IS NOT NULL + DROP PROCEDURE [sp_getReservationHomeSubCluster]; +GO + +CREATE PROCEDURE [dbo].[sp_getReservationHomeSubCluster] + @reservationId VARCHAR(128), + @homeSubCluster VARCHAR(256) OUTPUT +AS BEGIN + DECLARE @errorMessage nvarchar(4000) + + BEGIN TRY + + SELECT @homeSubCluster = [homeSubCluster] + FROM [dbo].[reservationsHomeSubCluster] + WHERE [reservationId] = @reservationId; + + END TRY + + BEGIN CATCH + + SET @errorMessage = dbo.func_FormatErrorMessage(ERROR_MESSAGE(), ERROR_LINE()) + + /* raise error and terminate the execution */ + RAISERROR(@errorMessage, --- Error Message + 1, -- Severity + -1 -- State + ) WITH log + END CATCH +END; +GO + +IF OBJECT_ID ( '[sp_deleteReservationHomeSubCluster]', 'P' ) IS NOT NULL + DROP PROCEDURE [sp_deleteReservationHomeSubCluster]; +GO + +CREATE PROCEDURE [dbo].[sp_deleteReservationHomeSubCluster] + @reservationId VARCHAR(128), + @rowCount int OUTPUT +AS BEGIN + DECLARE @errorMessage nvarchar(4000) + + BEGIN TRY + BEGIN TRAN + + DELETE FROM [dbo].[reservationsHomeSubCluster] + WHERE [reservationId] = @reservationId; + SELECT @rowCount = @@ROWCOUNT; + + COMMIT TRAN + END TRY + + BEGIN CATCH + ROLLBACK TRAN + + SET @errorMessage = dbo.func_FormatErrorMessage(ERROR_MESSAGE(), ERROR_LINE()) + + /* raise error and terminate the execution */ + RAISERROR(@errorMessage, --- Error Message + 1, -- Severity + -1 -- State + ) WITH log + END CATCH +END; GO \ No newline at end of file diff --git a/hadoop-yarn-project/hadoop-yarn/bin/FederationStateStore/SQLServer/FederationStateStoreTables.sql b/hadoop-yarn-project/hadoop-yarn/bin/FederationStateStore/SQLServer/FederationStateStoreTables.sql index c16091ccf1461..84f28acc17461 100644 --- a/hadoop-yarn-project/hadoop-yarn/bin/FederationStateStore/SQLServer/FederationStateStoreTables.sql +++ b/hadoop-yarn-project/hadoop-yarn/bin/FederationStateStore/SQLServer/FederationStateStoreTables.sql @@ -124,3 +124,35 @@ ELSE PRINT 'Table policies exists, no operation required...' GO GO + +IF NOT EXISTS ( SELECT * FROM [FederationStateStore].sys.tables + WHERE name = 'reservationsHomeSubCluster' + AND schema_id = SCHEMA_ID('dbo')) + BEGIN + PRINT 'Table reservationsHomeSubCluster does not exist, create it...' + + SET ANSI_NULLS ON + + SET QUOTED_IDENTIFIER ON + + SET ANSI_PADDING ON + + CREATE TABLE [dbo].[reservationsHomeSubCluster]( + reservationId VARCHAR(128) COLLATE Latin1_General_100_BIN2 NOT NULL, + homeSubCluster VARCHAR(256) NOT NULL, + createTime DATETIME2 NOT NULL CONSTRAINT ts_createAppTime DEFAULT GETUTCDATE(), + + CONSTRAINT [pk_reservationId] PRIMARY KEY + ( + [reservationId] + ) + ) + + SET ANSI_PADDING OFF + + PRINT 'Table reservationsHomeSubCluster created.' + END +ELSE + PRINT 'Table reservationsHomeSubCluster exists, no operation required...' + GO +GO \ No newline at end of file diff --git a/hadoop-yarn-project/hadoop-yarn/dev-support/jdiff/Apache_Hadoop_YARN_API_3.2.4.xml b/hadoop-yarn-project/hadoop-yarn/dev-support/jdiff/Apache_Hadoop_YARN_API_3.2.4.xml new file mode 100644 index 0000000000000..98c7645154996 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/dev-support/jdiff/Apache_Hadoop_YARN_API_3.2.4.xml @@ -0,0 +1,25332 @@ + + + + + + + + + + + + + + + + + + + + The interface used by clients to obtain a new {@link ApplicationId} for + submitting new applications.

    + +

    The ResourceManager responds with a new, monotonically + increasing, {@link ApplicationId} which is used by the client to submit + a new application.

    + +

    The ResourceManager also responds with details such + as maximum resource capabilities in the cluster as specified in + {@link GetNewApplicationResponse}.

    + + @param request request to get a new ApplicationId + @return response containing the new ApplicationId to be used + to submit an application + @throws YarnException + @throws IOException + @see #submitApplication(SubmitApplicationRequest)]]> + + + + + + + + The interface used by clients to submit a new application to the + ResourceManager.

    + +

    The client is required to provide details such as queue, + {@link Resource} required to run the ApplicationMaster, + the equivalent of {@link ContainerLaunchContext} for launching + the ApplicationMaster etc. via the + {@link SubmitApplicationRequest}.

    + +

    Currently the ResourceManager sends an immediate (empty) + {@link SubmitApplicationResponse} on accepting the submission and throws + an exception if it rejects the submission. However, this call needs to be + followed by {@link #getApplicationReport(GetApplicationReportRequest)} + to make sure that the application gets properly submitted - obtaining a + {@link SubmitApplicationResponse} from ResourceManager doesn't guarantee + that RM 'remembers' this application beyond failover or restart. If RM + failover or RM restart happens before ResourceManager saves the + application's state successfully, the subsequent + {@link #getApplicationReport(GetApplicationReportRequest)} will throw + a {@link ApplicationNotFoundException}. The Clients need to re-submit + the application with the same {@link ApplicationSubmissionContext} when + it encounters the {@link ApplicationNotFoundException} on the + {@link #getApplicationReport(GetApplicationReportRequest)} call.

    + +

    During the submission process, it checks whether the application + already exists. If the application exists, it will simply return + SubmitApplicationResponse

    + +

    In secure mode,the ResourceManager verifies access to + queues etc. before accepting the application submission.

    + + @param request request to submit a new application + @return (empty) response on accepting the submission + @throws YarnException + @throws IOException + @see #getNewApplication(GetNewApplicationRequest)]]> +     +     + + + + + + The interface used by clients to request the + ResourceManager to fail an application attempt.

    + +

    The client, via {@link FailApplicationAttemptRequest} provides the + {@link ApplicationAttemptId} of the attempt to be failed.

    + +

    In secure mode,the ResourceManager verifies access to the + application, queue etc. before failing the attempt.

    + +

    Currently, the ResourceManager returns an empty response + on success and throws an exception on rejecting the request.

    + + @param request request to fail an attempt + @return ResourceManager returns an empty response + on success and throws an exception on rejecting the request + @throws YarnException + @throws IOException + @see #getQueueUserAcls(GetQueueUserAclsInfoRequest)]]> +     +     + + + + + + The interface used by clients to request the + ResourceManager to abort submitted application.

    + +

    The client, via {@link KillApplicationRequest} provides the + {@link ApplicationId} of the application to be aborted.

    + +

    In secure mode,the ResourceManager verifies access to the + application, queue etc. before terminating the application.

    + +

    Currently, the ResourceManager returns an empty response + on success and throws an exception on rejecting the request.

    + + @param request request to abort a submitted application + @return ResourceManager returns an empty response + on success and throws an exception on rejecting the request + @throws YarnException + @throws IOException + @see #getQueueUserAcls(GetQueueUserAclsInfoRequest)]]> +     +     + + + + + + The interface used by clients to get metrics about the cluster from + the ResourceManager.

    + +

    The ResourceManager responds with a + {@link GetClusterMetricsResponse} which includes the + {@link YarnClusterMetrics} with details such as number of current + nodes in the cluster.

    + + @param request request for cluster metrics + @return cluster metrics + @throws YarnException + @throws IOException]]> +     +     + + + + + + The interface used by clients to get a report of all nodes + in the cluster from the ResourceManager.

    + +

    The ResourceManager responds with a + {@link GetClusterNodesResponse} which includes the + {@link NodeReport} for all the nodes in the cluster.

    + + @param request request for report on all nodes + @return report on all nodes + @throws YarnException + @throws IOException]]> +     +     + + + + + + The interface used by clients to get information about queues + from the ResourceManager.

    + +

    The client, via {@link GetQueueInfoRequest}, can ask for details such + as used/total resources, child queues, running applications etc.

    + +

    In secure mode,the ResourceManager verifies access before + providing the information.

    + + @param request request to get queue information + @return queue information + @throws YarnException + @throws IOException]]> +     +     + + + + + + The interface used by clients to get information about queue + acls for current user from the ResourceManager. +

    + +

    The ResourceManager responds with queue acls for all + existing queues.

    + + @param request request to get queue acls for current user + @return queue acls for current user + @throws YarnException + @throws IOException]]> +     +     + + + + + + + + + + + + + + The interface used by clients to obtain a new {@link ReservationId} for + submitting new reservations.

    + +

    The ResourceManager responds with a new, unique, + {@link ReservationId} which is used by the client to submit + a new reservation.

    + + @param request to get a new ReservationId + @return response containing the new ReservationId to be used + to submit a new reservation + @throws YarnException if the reservation system is not enabled. + @throws IOException on IO failures. + @see #submitReservation(ReservationSubmissionRequest)]]> +     +     + + + + + + + The interface used by clients to submit a new reservation to the + {@code ResourceManager}. +

    + +

    + The client packages all details of its request in a + {@link ReservationSubmissionRequest} object. This contains information + about the amount of capacity, temporal constraints, and concurrency needs. + Furthermore, the reservation might be composed of multiple stages, with + ordering dependencies among them. +

    + +

    + In order to respond, a new admission control component in the + {@code ResourceManager} performs an analysis of the resources that have + been committed over the period of time the user is requesting, verify that + the user requests can be fulfilled, and that it respect a sharing policy + (e.g., {@code CapacityOverTimePolicy}). Once it has positively determined + that the ReservationSubmissionRequest is satisfiable the + {@code ResourceManager} answers with a + {@link ReservationSubmissionResponse} that include a non-null + {@link ReservationId}. Upon failure to find a valid allocation the response + is an exception with the reason. + + On application submission the client can use this {@link ReservationId} to + obtain access to the reserved resources. +

    + +

    + The system guarantees that during the time-range specified by the user, the + reservationID will be corresponding to a valid reservation. The amount of + capacity dedicated to such queue can vary overtime, depending of the + allocation that has been determined. But it is guaranteed to satisfy all + the constraint expressed by the user in the + {@link ReservationSubmissionRequest}. +

    + + @param request the request to submit a new Reservation + @return response the {@link ReservationId} on accepting the submission + @throws YarnException if the request is invalid or reservation cannot be + created successfully + @throws IOException]]> +     +     + + + + + + + The interface used by clients to update an existing Reservation. This is + referred to as a re-negotiation process, in which a user that has + previously submitted a Reservation. +

    + +

    + The allocation is attempted by virtually substituting all previous + allocations related to this Reservation with new ones, that satisfy the new + {@link ReservationUpdateRequest}. Upon success the previous allocation is + substituted by the new one, and on failure (i.e., if the system cannot find + a valid allocation for the updated request), the previous allocation + remains valid. + + The {@link ReservationId} is not changed, and applications currently + running within this reservation will automatically receive the resources + based on the new allocation. +

    + + @param request to update an existing Reservation (the ReservationRequest + should refer to an existing valid {@link ReservationId}) + @return response empty on successfully updating the existing reservation + @throws YarnException if the request is invalid or reservation cannot be + updated successfully + @throws IOException]]> +     +     + + + + + + + The interface used by clients to remove an existing Reservation. + + Upon deletion of a reservation applications running with this reservation, + are automatically downgraded to normal jobs running without any dedicated + reservation. +

    + + @param request to remove an existing Reservation (the ReservationRequest + should refer to an existing valid {@link ReservationId}) + @return response empty on successfully deleting the existing reservation + @throws YarnException if the request is invalid or reservation cannot be + deleted successfully + @throws IOException]]> +     +     + + + + + + + The interface used by clients to get the list of reservations in a plan. + The reservationId will be used to search for reservations to list if it is + provided. Otherwise, it will select active reservations within the + startTime and endTime (inclusive). +

    + + @param request to list reservations in a plan. Contains fields to select + String queue, ReservationId reservationId, long startTime, + long endTime, and a bool includeReservationAllocations. + + queue: Required. Cannot be null or empty. Refers to the + reservable queue in the scheduler that was selected when + creating a reservation submission + {@link ReservationSubmissionRequest}. + + reservationId: Optional. If provided, other fields will + be ignored. + + startTime: Optional. If provided, only reservations that + end after the startTime will be selected. This defaults + to 0 if an invalid number is used. + + endTime: Optional. If provided, only reservations that + start on or before endTime will be selected. This defaults + to Long.MAX_VALUE if an invalid number is used. + + includeReservationAllocations: Optional. Flag that + determines whether the entire reservation allocations are + to be returned. Reservation allocations are subject to + change in the event of re-planning as described by + {@code ReservationDefinition}. + + @return response that contains information about reservations that are + being searched for. + @throws YarnException if the request is invalid + @throws IOException on IO failures]]> +     +     + + + + + + + The interface used by client to get node to labels mappings in existing cluster +

    + + @param request + @return node to labels mappings + @throws YarnException + @throws IOException]]> +     +     + + + + + + + The interface used by client to get labels to nodes mappings + in existing cluster +

    + + @param request + @return labels to nodes mappings + @throws YarnException + @throws IOException]]> +     +     + + + + + + + The interface used by client to get node labels in the cluster +

    + + @param request to get node labels collection of this cluster + @return node labels collection of this cluster + @throws YarnException + @throws IOException]]> +     +     + + + + + + + The interface used by client to set priority of an application. +

    + @param request to set priority of an application + @return an empty response + @throws YarnException + @throws IOException]]> +     +     + + + + + + The interface used by clients to request the + ResourceManager to signal a container. For example, + the client can send command OUTPUT_THREAD_DUMP to dump threads of the + container.

    + +

    The client, via {@link SignalContainerRequest} provides the + id of the container and the signal command.

    + +

    In secure mode,the ResourceManager verifies access to the + application before signaling the container. + The user needs to have MODIFY_APP permission.

    + +

    Currently, the ResourceManager returns an empty response + on success and throws an exception on rejecting the request.

    + + @param request request to signal a container + @return ResourceManager returns an empty response + on success and throws an exception on rejecting the request + @throws YarnException + @throws IOException]]> +     +     + + + + + + + The interface used by client to set ApplicationTimeouts of an application. + The UpdateApplicationTimeoutsRequest should have timeout value with + absolute time with ISO8601 format yyyy-MM-dd'T'HH:mm:ss.SSSZ. +

    + Note: If application timeout value is less than or equal to current + time then update application throws YarnException. + @param request to set ApplicationTimeouts of an application + @return a response with updated timeouts. + @throws YarnException if update request has empty values or application is + in completing states. + @throws IOException on IO failures]]> +     +     + + + + + + + The interface used by clients to get all the resource profiles that are + available on the ResourceManager. +

    + @param request request to get all the resource profiles + @return Response containing a map of the profile name to Resource + capabilities + @throws YARNFeatureNotEnabledException if resource-profile is disabled + @throws YarnException if any error happens inside YARN + @throws IOException in case of other errors]]> +     +     + + + + + + + The interface to get the details for a specific resource profile. +

    + @param request request to get the details of a resource profile + @return Response containing the details for a particular resource profile + @throws YARNFeatureNotEnabledException if resource-profile is disabled + @throws YarnException if any error happens inside YARN + @throws IOException in case of other errors]]> +     +     + + + + + + + The interface to get the details for a specific resource profile. +

    + @param request request to get the details of a resource profile + @return Response containing the details for a particular resource profile + @throws YarnException if any error happens inside YARN + @throws IOException in case of other errors]]> +     +     + + + + + + + The interface used by client to get attributes to nodes mappings + available in ResourceManager. +

    + + @param request request to get details of attributes to nodes mapping. + @return Response containing the details of attributes to nodes mappings. + @throws YarnException if any error happens inside YARN + @throws IOException incase of other errors]]> +     +     + + + + + + + The interface used by client to get node attributes available in + ResourceManager. +

    + + @param request request to get node attributes collection of this cluster. + @return Response containing node attributes collection. + @throws YarnException if any error happens inside YARN. + @throws IOException incase of other errors.]]> +     +     + + + + + + + The interface used by client to get node to attributes mappings. + in existing cluster. +

    + + @param request request to get nodes to attributes mapping. + @return nodes to attributes mappings. + @throws YarnException if any error happens inside YARN. + @throws IOException]]> +     +     + + The protocol between clients and the ResourceManager + to submit/abort jobs and to get information on applications, cluster metrics, + nodes, queues and ACLs.

    ]]> +     + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + The protocol between clients and the ApplicationHistoryServer to + get the information of completed applications etc. +

    ]]> +     +     + + + + + + + + + + The interface used by a new ApplicationMaster to register with + the ResourceManager. +

    + +

    + The ApplicationMaster needs to provide details such as RPC + Port, HTTP tracking url etc. as specified in + {@link RegisterApplicationMasterRequest}. +

    + +

    + The ResourceManager responds with critical details such as + maximum resource capabilities in the cluster as specified in + {@link RegisterApplicationMasterResponse}. +

    + +

    + Re-register is only allowed for Unmanaged Application Master + (UAM) HA, with + {@link org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext#getKeepContainersAcrossApplicationAttempts()} + set to true. +

    + + @param request registration request + @return registration respose + @throws YarnException + @throws IOException + @throws InvalidApplicationMasterRequestException The exception is thrown + when an ApplicationMaster tries to register more then once. + @see RegisterApplicationMasterRequest + @see RegisterApplicationMasterResponse]]> +     +     + + + + + + The interface used by an ApplicationMaster to notify the + ResourceManager about its completion (success or failed).

    + +

    The ApplicationMaster has to provide details such as + final state, diagnostics (in case of failures) etc. as specified in + {@link FinishApplicationMasterRequest}.

    + +

    The ResourceManager responds with + {@link FinishApplicationMasterResponse}.

    + + @param request completion request + @return completion response + @throws YarnException + @throws IOException + @see FinishApplicationMasterRequest + @see FinishApplicationMasterResponse]]> +     +     + + + + + + + The main interface between an ApplicationMaster and the + ResourceManager. +

    + +

    + The ApplicationMaster uses this interface to provide a list of + {@link ResourceRequest} and returns unused {@link Container} allocated to + it via {@link AllocateRequest}. Optionally, the + ApplicationMaster can also blacklist resources which + it doesn't want to use. +

    + +

    + This also doubles up as a heartbeat to let the + ResourceManager know that the ApplicationMaster + is alive. Thus, applications should periodically make this call to be kept + alive. The frequency depends on + {@link YarnConfiguration#RM_AM_EXPIRY_INTERVAL_MS} which defaults to + {@link YarnConfiguration#DEFAULT_RM_AM_EXPIRY_INTERVAL_MS}. +

    + +

    + The ResourceManager responds with list of allocated + {@link Container}, status of completed containers and headroom information + for the application. +

    + +

    + The ApplicationMaster can use the available headroom + (resources) to decide how to utilized allocated resources and make informed + decisions about future resource requests. +

    + + @param request + allocation request + @return allocation response + @throws YarnException + @throws IOException + @throws InvalidApplicationMasterRequestException + This exception is thrown when an ApplicationMaster calls allocate + without registering first. + @throws InvalidResourceBlacklistRequestException + This exception is thrown when an application provides an invalid + specification for blacklist of resources. + @throws InvalidResourceRequestException + This exception is thrown when a {@link ResourceRequest} is out of + the range of the configured lower and upper limits on the + resources. + @see AllocateRequest + @see AllocateResponse]]> +     +     + + The protocol between a live instance of ApplicationMaster + and the ResourceManager.

    + +

    This is used by the ApplicationMaster to register/unregister + and to request and obtain resources in the cluster from the + ResourceManager.

    ]]> +     +     + + + + + + + + + + The interface used by clients to claim a resource with the + SharedCacheManager. The client uses a checksum to identify the + resource and an {@link ApplicationId} to identify which application will be + using the resource. +

    + +

    + The SharedCacheManager responds with whether or not the + resource exists in the cache. If the resource exists, a Path + to the resource in the shared cache is returned. If the resource does not + exist, the response is empty. +

    + + @param request request to claim a resource in the shared cache + @return response indicating if the resource is already in the cache + @throws YarnException + @throws IOException]]> +     +     + + + + + + + The interface used by clients to release a resource with the + SharedCacheManager. This method is called once an application + is no longer using a claimed resource in the shared cache. The client uses + a checksum to identify the resource and an {@link ApplicationId} to + identify which application is releasing the resource. +

    + +

    + Note: This method is an optimization and the client is not required to call + it for correctness. +

    + +

    + Currently the SharedCacheManager sends an empty response. +

    + + @param request request to release a resource in the shared cache + @return (empty) response on releasing the resource + @throws YarnException + @throws IOException]]> +     +     + + + The protocol between clients and the SharedCacheManager to claim + and release resources in the shared cache. +

    ]]> +     +     + + + + + + + + + + The ApplicationMaster provides a list of + {@link StartContainerRequest}s to a NodeManager to + start {@link Container}s allocated to it using this interface. +

    + +

    + The ApplicationMaster has to provide details such as allocated + resource capability, security tokens (if enabled), command to be executed + to start the container, environment for the process, necessary + binaries/jar/shared-objects etc. via the {@link ContainerLaunchContext} in + the {@link StartContainerRequest}. +

    + +

    + The NodeManager sends a response via + {@link StartContainersResponse} which includes a list of + {@link Container}s of successfully launched {@link Container}s, a + containerId-to-exception map for each failed {@link StartContainerRequest} in + which the exception indicates errors from per container and a + allServicesMetaData map between the names of auxiliary services and their + corresponding meta-data. Note: None-container-specific exceptions will + still be thrown by the API method itself. +

    +

    + The ApplicationMaster can use + {@link #getContainerStatuses(GetContainerStatusesRequest)} to get updated + statuses of the to-be-launched or launched containers. +

    + + @param request + request to start a list of containers + @return response including conatinerIds of all successfully launched + containers, a containerId-to-exception map for failed requests and + a allServicesMetaData map. + @throws YarnException + @throws IOException]]> +     +     + + + + + + + The ApplicationMaster requests a NodeManager to + stop a list of {@link Container}s allocated to it using this + interface. +

    + +

    + The ApplicationMaster sends a {@link StopContainersRequest} + which includes the {@link ContainerId}s of the containers to be stopped. +

    + +

    + The NodeManager sends a response via + {@link StopContainersResponse} which includes a list of {@link ContainerId} + s of successfully stopped containers, a containerId-to-exception map for + each failed request in which the exception indicates errors from per + container. Note: None-container-specific exceptions will still be thrown by + the API method itself. ApplicationMaster can use + {@link #getContainerStatuses(GetContainerStatusesRequest)} to get updated + statuses of the containers. +

    + + @param request + request to stop a list of containers + @return response which includes a list of containerIds of successfully + stopped containers, a containerId-to-exception map for failed + requests. + @throws YarnException + @throws IOException]]> +     +     + + + + + + + The API used by the ApplicationMaster to request for current + statuses of Containers from the NodeManager. +

    + +

    + The ApplicationMaster sends a + {@link GetContainerStatusesRequest} which includes the {@link ContainerId}s + of all containers whose statuses are needed. +

    + +

    + The NodeManager responds with + {@link GetContainerStatusesResponse} which includes a list of + {@link ContainerStatus} of the successfully queried containers and a + containerId-to-exception map for each failed request in which the exception + indicates errors from per container. Note: None-container-specific + exceptions will still be thrown by the API method itself. +

    + + @param request + request to get ContainerStatuses of containers with + the specified ContainerIds + @return response containing the list of ContainerStatus of the + successfully queried containers and a containerId-to-exception map + for failed requests. + + @throws YarnException + @throws IOException]]> +     +     + + + + + + + The API used by the ApplicationMaster to request for + resource increase of running containers on the NodeManager. +

    + + @param request + request to increase resource of a list of containers + @return response which includes a list of containerIds of containers + whose resource has been successfully increased and a + containerId-to-exception map for failed requests. + + @throws YarnException + @throws IOException]]> +     +     + + + + + + + The API used by the ApplicationMaster to request for + resource update of running containers on the NodeManager. +

    + + @param request + request to update resource of a list of containers + @return response which includes a list of containerIds of containers + whose resource has been successfully updated and a + containerId-to-exception map for failed requests. + + @throws YarnException Exception specific to YARN + @throws IOException IOException thrown from NodeManager]]> +     +     + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + The protocol between an ApplicationMaster and a + NodeManager to start/stop and increase resource of containers + and to get status of running containers.

    + +

    If security is enabled the NodeManager verifies that the + ApplicationMaster has truly been allocated the container + by the ResourceManager and also verifies all interactions such + as stopping the container or obtaining status information for the container. +

    ]]> +     +     + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + response id used to track duplicate responses. + @return response id]]> + + + + + + response id used to track duplicate responses. + @param id response id]]> + + + + + current progress of application. + @return current progress of application]]> + + + + + + current progress of application + @param progress current progress of application]]> + + + + + ResourceRequest to update the + ResourceManager about the application's resource requirements. + @return the list of ResourceRequest + @see ResourceRequest]]> + + + + + + ResourceRequest to update the + ResourceManager about the application's resource requirements. + @param resourceRequests list of ResourceRequest to update the + ResourceManager about the application's + resource requirements + @see ResourceRequest]]> + + + + + ContainerId of containers being + released by the ApplicationMaster. + @return list of ContainerId of containers being + released by the ApplicationMaster]]> + + + + + + ContainerId of containers being + released by the ApplicationMaster + @param releaseContainers list of ContainerId of + containers being released by the + ApplicationMaster]]> + + + + + ResourceBlacklistRequest being sent by the + ApplicationMaster. + @return the ResourceBlacklistRequest being sent by the + ApplicationMaster + @see ResourceBlacklistRequest]]> + + + + + + ResourceBlacklistRequest to inform the + ResourceManager about the blacklist additions and removals + per the ApplicationMaster. + + @param resourceBlacklistRequest the ResourceBlacklistRequest + to inform the ResourceManager about + the blacklist additions and removals + per the ApplicationMaster + @see ResourceBlacklistRequest]]> + + + + + ApplicationMaster. + @return list of {@link UpdateContainerRequest} + being sent by the + ApplicationMaster.]]> + + + + + + ResourceManager about the containers that need to be + updated. + @param updateRequests list of UpdateContainerRequest for + containers to be updated]]> + + + + + ApplicationMaster. + @return list of {@link SchedulingRequest} being sent by the + ApplicationMaster.]]> + + + + + + ResourceManager about the application's resource requirements + (potentially including allocation tags and placement constraints). + @param schedulingRequests list of {@link SchedulingRequest} to update + the ResourceManager about the application's resource + requirements.]]> + + + + + + + + + + + + + + + + + The core request sent by the ApplicationMaster to the + ResourceManager to obtain resources in the cluster.

    + +

    The request includes: +

      +
    • A response id to track duplicate responses.
      • +
    • Progress information.
      • +
    • + A list of {@link ResourceRequest} to inform the + ResourceManager about the application's + resource requirements. +
      • +
    • + A list of unused {@link Container} which are being returned. +
      • +
    • + A list of {@link UpdateContainerRequest} to inform + the ResourceManager about the change in + requirements of running containers. +
      • +
    + + @see ApplicationMasterProtocol#allocate(AllocateRequest)]]> +     +     + + + + + + + responseId of the request. + @see AllocateRequest#setResponseId(int) + @param responseId responseId of the request + @return {@link AllocateRequestBuilder}]]> + + + + + + progress of the request. + @see AllocateRequest#setProgress(float) + @param progress progress of the request + @return {@link AllocateRequestBuilder}]]> + + + + + + askList of the request. + @see AllocateRequest#setAskList(List) + @param askList askList of the request + @return {@link AllocateRequestBuilder}]]> + + + + + + releaseList of the request. + @see AllocateRequest#setReleaseList(List) + @param releaseList releaseList of the request + @return {@link AllocateRequestBuilder}]]> + + + + + + resourceBlacklistRequest of the request. + @see AllocateRequest#setResourceBlacklistRequest( + ResourceBlacklistRequest) + @param resourceBlacklistRequest + resourceBlacklistRequest of the request + @return {@link AllocateRequestBuilder}]]> + + + + + + updateRequests of the request. + @see AllocateRequest#setUpdateRequests(List) + @param updateRequests updateRequests of the request + @return {@link AllocateRequestBuilder}]]> + + + + + + schedulingRequests of the request. + @see AllocateRequest#setSchedulingRequests(List) + @param schedulingRequests SchedulingRequest of the request + @return {@link AllocateRequestBuilder}]]> + + + + + + trackingUrl of the request. + @see AllocateRequest#setTrackingUrl(String) + @param trackingUrl new tracking url + @return {@link AllocateRequestBuilder}]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + ResourceManager needs the + ApplicationMaster to take some action then it will send an + AMCommand to the ApplicationMaster. See AMCommand + for details on commands and actions for them. + @return AMCommand if the ApplicationMaster should + take action, null otherwise + @see AMCommand]]> + + + + + last response id. + @return last response id]]> + + + + + newly allocated Container by the + ResourceManager. + @return list of newly allocated Container]]> + + + + + available headroom for resources in the cluster for the + application. + @return limit of available headroom for resources in the cluster for the + application]]> + + + + + completed containers' statuses. + @return the list of completed containers' statuses]]> + + + + + updated NodeReports. Updates could + be changes in health, availability etc of the nodes. + @return The delta of updated nodes since the last response]]> + + + + + + + + + + + The message is a snapshot of the resources the RM wants back from the AM. + While demand persists, the RM will repeat its request; applications should + not interpret each message as a request for additional + resources on top of previous messages. Resources requested consistently + over some duration may be forcibly killed by the RM. + + @return A specification of the resources to reclaim from this AM.]]> + + + + + + 1) AM is receiving first container on underlying NodeManager.
    + OR
    + 2) NMToken master key rolled over in ResourceManager and AM is getting new + container on the same underlying NodeManager. +

    + AM will receive one NMToken per NM irrespective of the number of containers + issued on same NM. AM is expected to store these tokens until issued a + new token for the same NM. + @return list of NMTokens required for communicating with NM]]> + + + + + ResourceManager. + @return list of newly increased containers]]> + + + + + + + + + + + + + + + + + + + + + + + + + + UpdateContainerError for + containers updates requests that were in error]]> + + + + + ResourceManager from previous application attempts which + have not been reported to the Application Master yet. +
    + These containers were recovered by the RM after the application master + had already registered. This may happen after RM restart when some NMs get + delayed in connecting to the RM and reporting the active containers. + Since they were not reported in the registration + response, they are reported in the response to the AM heartbeat. + + @return the list of running containers as viewed by + ResourceManager from previous application attempts.]]> +     +     + + + + + + + ResourceManager the + ApplicationMaster during resource negotiation. +

    + The response, includes: +

      +
    • Response ID to track duplicate responses.
      • +
    • + An AMCommand sent by ResourceManager to let the + {@code ApplicationMaster} take some actions (resync, shutdown etc.). +
      • +
    • A list of newly allocated {@link Container}.
      • +
    • A list of completed {@link Container}s' statuses.
      • +
    • + The available headroom for resources in the cluster for the + application. +
      • +
    • A list of nodes whose status has been updated.
      • +
    • The number of available nodes in a cluster.
      • +
    • A description of resources requested back by the cluster
      • +
    • AMRMToken, if AMRMToken has been rolled over
      • +
    • + A list of {@link Container} representing the containers + whose resource has been increased. +
      • +
    • + A list of {@link Container} representing the containers + whose resource has been decreased. +
      • +
    + + @see ApplicationMasterProtocol#allocate(AllocateRequest)]]> +     +     + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Note: {@link NMToken} will be used for authenticating communication with + {@code NodeManager}. + @return the list of container tokens to be used for authorization during + container resource update. + @see NMToken]]> + + + + + + AllocateResponse.getUpdatedContainers. + The token contains the container id and resource capability required for + container resource update. + @param containersToUpdate the list of container tokens to be used + for container resource increase.]]> + + + + The request sent by Application Master to the + Node Manager to change the resource quota of a container.

    + + @see ContainerManagementProtocol#updateContainer(ContainerUpdateRequest)]]> +     +     + + + + + + + + + + + + + + + + + + + + + + The response sent by the NodeManager to the + ApplicationMaster when asked to update container resource. +

    + + @see ContainerManagementProtocol#updateContainer(ContainerUpdateRequest)]]> +     +     + + + + + + + + + + + ApplicationAttemptId of the attempt to be failed. + @return ApplicationAttemptId of the attempt.]]> + + + + + + + The request sent by the client to the ResourceManager + to fail an application attempt.

    + +

    The request includes the {@link ApplicationAttemptId} of the attempt to + be failed.

    + + @see ApplicationClientProtocol#failApplicationAttempt(FailApplicationAttemptRequest)]]> +     +     + + + + + + + The response sent by the ResourceManager to the client + failing an application attempt.

    + +

    Currently it's empty.

    + + @see ApplicationClientProtocol#failApplicationAttempt(FailApplicationAttemptRequest)]]> +     +     + + + + + + + + + + + + + final state of the ApplicationMaster. + @return final state of the ApplicationMaster]]> + + + + + + final state of the ApplicationMaster + @param finalState final state of the ApplicationMaster]]> + + + + + diagnostic information on application failure. + @return diagnostic information on application failure]]> + + + + + + diagnostic information on application failure. + @param diagnostics diagnostic information on application failure]]> + + + + + tracking URL for the ApplicationMaster. + This url if contains scheme then that will be used by resource manager + web application proxy otherwise it will default to http. + @return tracking URLfor the ApplicationMaster]]> + + + + + + final tracking URLfor the ApplicationMaster. + This is the web-URL to which ResourceManager or web-application proxy will + redirect client/users once the application is finished and the + ApplicationMaster is gone. +

    + If the passed url has a scheme then that will be used by the + ResourceManager and web-application proxy, otherwise the scheme will + default to http. +

    +

    + Empty, null, "N/A" strings are all valid besides a real URL. In case an url + isn't explicitly passed, it defaults to "N/A" on the ResourceManager. +

    + + @param url + tracking URLfor the ApplicationMaster]]> + + + + + The final request includes details such: +

      +
    • Final state of the {@code ApplicationMaster}
      • +
    • + Diagnostic information in case of failure of the + {@code ApplicationMaster} +
      • +
    • Tracking URL
      • +
    + + @see ApplicationMasterProtocol#finishApplicationMaster(FinishApplicationMasterRequest)]]> +     +     + + + + + + + + + + + + ResourceManager to a + ApplicationMaster on it's completion. +

    + The response, includes: +

      +
    • A flag which indicates that the application has successfully unregistered + with the RM and the application can safely stop.
      • +
    +

    + Note: The flag indicates whether the application has successfully + unregistered and is safe to stop. The application may stop after the flag is + true. If the application stops before the flag is true then the RM may retry + the application. + + @see ApplicationMasterProtocol#finishApplicationMaster(FinishApplicationMasterRequest)]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + ApplicationAttemptId of an application attempt. + + @return ApplicationAttemptId of an application attempt]]> + + + + + + ApplicationAttemptId of an application attempt + + @param applicationAttemptId + ApplicationAttemptId of an application attempt]]> + + + + + The request sent by a client to the ResourceManager to get an + {@link ApplicationAttemptReport} for an application attempt. +

    + +

    + The request should include the {@link ApplicationAttemptId} of the + application attempt. +

    + + @see ApplicationAttemptReport + @see ApplicationHistoryProtocol#getApplicationAttemptReport(GetApplicationAttemptReportRequest)]]> +     +     + + + + + + + + + + + ApplicationAttemptReport for the application attempt. + + @return ApplicationAttemptReport for the application attempt]]> + + + + + + ApplicationAttemptReport for the application attempt. + + @param applicationAttemptReport + ApplicationAttemptReport for the application attempt]]> + + + + + The response sent by the ResourceManager to a client requesting + an application attempt report. +

    + +

    + The response includes an {@link ApplicationAttemptReport} which has the + details about the particular application attempt +

    + + @see ApplicationAttemptReport + @see ApplicationHistoryProtocol#getApplicationAttemptReport(GetApplicationAttemptReportRequest)]]> +     +     + + + + + + + + + + + ApplicationId of an application + + @return ApplicationId of an application]]> + + + + + + ApplicationId of an application + + @param applicationId + ApplicationId of an application]]> + + + + + The request from clients to get a list of application attempt reports of an + application from the ResourceManager. +

    + + @see ApplicationHistoryProtocol#getApplicationAttempts(GetApplicationAttemptsRequest)]]> +     +     + + + + + + + + + + + ApplicationReport of an application. + + @return a list of ApplicationReport of an application]]> + + + + + + ApplicationReport of an application. + + @param applicationAttempts + a list of ApplicationReport of an application]]> + + + + + The response sent by the ResourceManager to a client requesting + a list of {@link ApplicationAttemptReport} for application attempts. +

    + +

    + The ApplicationAttemptReport for each application includes the + details of an application attempt. +

    + + @see ApplicationAttemptReport + @see ApplicationHistoryProtocol#getApplicationAttempts(GetApplicationAttemptsRequest)]]> +     +     + + + + + + + + + + + ApplicationId of the application. + @return ApplicationId of the application]]> + + + + + + ApplicationId of the application + @param applicationId ApplicationId of the application]]> + + + + The request sent by a client to the ResourceManager to + get an {@link ApplicationReport} for an application.

    + +

    The request should include the {@link ApplicationId} of the + application.

    + + @see ApplicationClientProtocol#getApplicationReport(GetApplicationReportRequest) + @see ApplicationReport]]> +     +     + + + + + + + + ApplicationReport for the application. + @return ApplicationReport for the application]]> + + + + The response sent by the ResourceManager to a client + requesting an application report.

    + +

    The response includes an {@link ApplicationReport} which has details such + as user, queue, name, host on which the ApplicationMaster is + running, RPC port, tracking URL, diagnostics, start time etc.

    + + @see ApplicationClientProtocol#getApplicationReport(GetApplicationReportRequest)]]> +     +     + + + + + + + + + + + + + + + + + + + + The request from clients to get a report of Applications matching the + giving application types in the cluster from the + ResourceManager. +

    + + @see ApplicationClientProtocol#getApplications(GetApplicationsRequest) + +

    Setting any of the parameters to null, would just disable that + filter

    + + @param scope {@link ApplicationsRequestScope} to filter by + @param users list of users to filter by + @param queues list of scheduler queues to filter by + @param applicationTypes types of applications + @param applicationTags application tags to filter by + @param applicationStates application states to filter by + @param startRange range of application start times to filter by + @param finishRange range of application finish times to filter by + @param limit number of applications to limit to + @return {@link GetApplicationsRequest} to be used with + {@link ApplicationClientProtocol#getApplications(GetApplicationsRequest)}]]> +     +     + + + + + The request from clients to get a report of Applications matching the + giving application types in the cluster from the + ResourceManager. +

    + + @param scope {@link ApplicationsRequestScope} to filter by + @see ApplicationClientProtocol#getApplications(GetApplicationsRequest) + @return a report of Applications in {@link GetApplicationsRequest}]]> +     +     + + + + + The request from clients to get a report of Applications matching the + giving application types in the cluster from the + ResourceManager. +

    + + + @see ApplicationClientProtocol#getApplications(GetApplicationsRequest) + @return a report of Applications in {@link GetApplicationsRequest}]]> +     +     + + + + + The request from clients to get a report of Applications matching the + giving application states in the cluster from the + ResourceManager. +

    + + + @see ApplicationClientProtocol#getApplications(GetApplicationsRequest) + @return a report of Applications in {@link GetApplicationsRequest}]]> +     +     + + + + + + The request from clients to get a report of Applications matching the + giving and application types and application types in the cluster from the + ResourceManager. +

    + + + @see ApplicationClientProtocol#getApplications(GetApplicationsRequest) + @return a report of Applications in GetApplicationsRequest]]> +     +     + + + + + + + + + + + + The request from clients to get a report of Applications + in the cluster from the ResourceManager.

    + + @see ApplicationClientProtocol#getApplications(GetApplicationsRequest)]]> +     +     + + + + + + + + ApplicationReport for applications. + @return ApplicationReport for applications]]> + + + + The response sent by the ResourceManager to a client + requesting an {@link ApplicationReport} for applications.

    + +

    The ApplicationReport for each application includes details + such as user, queue, name, host on which the ApplicationMaster + is running, RPC port, tracking URL, diagnostics, start time etc.

    + + @see ApplicationReport + @see ApplicationClientProtocol#getApplications(GetApplicationsRequest)]]> +     +     + + + + + + + + + + + + + + + + + + + + + + + + The request from clients to get node to attribute value mapping for all or + given set of Node AttributeKey's in the cluster from the + ResourceManager. +

    + + @see ApplicationClientProtocol#getAttributesToNodes + (GetAttributesToNodesRequest)]]> +     +     + + + + + + + + + + + + + + + + + + + The response sent by the ResourceManager to a client requesting + node to attribute value mapping for all or given set of Node AttributeKey's. +

    + + @see ApplicationClientProtocol#getAttributesToNodes + (GetAttributesToNodesRequest)]]> +     +     + + + + + + + + + The request sent by clients to get cluster metrics from the + ResourceManager.

    + +

    Currently, this is empty.

    + + @see ApplicationClientProtocol#getClusterMetrics(GetClusterMetricsRequest)]]> +     +     + + + + + + + + YarnClusterMetrics for the cluster. + @return YarnClusterMetrics for the cluster]]> + + + + ResourceManager to a client + requesting cluster metrics. + + @see YarnClusterMetrics + @see ApplicationClientProtocol#getClusterMetrics(GetClusterMetricsRequest)]]> + + + + + + + + + + + + + + + The request from clients to get node attributes in the cluster from the + ResourceManager. +

    + + @see ApplicationClientProtocol#getClusterNodeAttributes + (GetClusterNodeAttributesRequest)]]> +     +     + + + + + + + + + + + + + + + + + + + + + + + + + The response sent by the ResourceManager to a client requesting + a node attributes in cluster. +

    + + @see ApplicationClientProtocol#getClusterNodeAttributes + (GetClusterNodeAttributesRequest)]]> +     +     + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + The request from clients to get a report of all nodes + in the cluster from the ResourceManager.

    + + The request will ask for all nodes in the given {@link NodeState}s. + + @see ApplicationClientProtocol#getClusterNodes(GetClusterNodesRequest)]]> +     +     + + + + + + + + NodeReport for all nodes in the cluster. + @return NodeReport for all nodes in the cluster]]> + + + + The response sent by the ResourceManager to a client + requesting a {@link NodeReport} for all nodes.

    + +

    The NodeReport contains per-node information such as + available resources, number of containers, tracking url, rack name, health + status etc. + + @see NodeReport + @see ApplicationClientProtocol#getClusterNodes(GetClusterNodesRequest)]]> + + + + + + + + + + + + + ContainerId of the Container. + + @return ContainerId of the Container]]> + + + + + + ContainerId of the container + + @param containerId + ContainerId of the container]]> + + + + + The request sent by a client to the ResourceManager to get an + {@link ContainerReport} for a container. +

    ]]> +     +     + + + + + + + + + + + ContainerReport for the container. + + @return ContainerReport for the container]]> + + + + + + + + The response sent by the ResourceManager to a client requesting + a container report. +

    + +

    + The response includes a {@link ContainerReport} which has details of a + container. +

    ]]> +     +     + + + + + + + + + + + ApplicationAttemptId of an application attempt. + + @return ApplicationAttemptId of an application attempt]]> + + + + + + ApplicationAttemptId of an application attempt + + @param applicationAttemptId + ApplicationAttemptId of an application attempt]]> + + + + + The request from clients to get a list of container reports, which belong to + an application attempt from the ResourceManager. +

    + + @see ApplicationHistoryProtocol#getContainers(GetContainersRequest)]]> +     +     + + + + + + + + + + + ContainerReport for all the containers of an + application attempt. + + @return a list of ContainerReport for all the containers of an + application attempt]]> + + + + + + ContainerReport for all the containers of an + application attempt. + + @param containers + a list of ContainerReport for all the containers of + an application attempt]]> + + + + + The response sent by the ResourceManager to a client requesting + a list of {@link ContainerReport} for containers. +

    + +

    + The ContainerReport for each container includes the container + details. +

    + + @see ContainerReport + @see ApplicationHistoryProtocol#getContainers(GetContainersRequest)]]> +     +     + + + + + + + + + + + ContainerIds of containers for which to obtain + the ContainerStatus. + + @return the list of ContainerIds of containers for which to + obtain the ContainerStatus.]]> + + + + + + ContainerIds of containers for which to obtain + the ContainerStatus + + @param containerIds + a list of ContainerIds of containers for which to + obtain the ContainerStatus]]> + + + + ApplicationMaster to the + NodeManager to get {@link ContainerStatus} of requested + containers. + + @see ContainerManagementProtocol#getContainerStatuses(GetContainerStatusesRequest)]]> + + + + + + + + + + ContainerStatuses of the requested containers. + + @return ContainerStatuses of the requested containers.]]> + + + + + + + + + NodeManager to the + ApplicationMaster when asked to obtain the + ContainerStatus of requested containers. + + @see ContainerManagementProtocol#getContainerStatuses(GetContainerStatusesRequest)]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + The request sent by clients to get a new {@link ApplicationId} for + submitting an application.

    + +

    Currently, this is empty.

    + + @see ApplicationClientProtocol#getNewApplication(GetNewApplicationRequest)]]> +     +     + + + + + + + + new ApplicationId allocated by the + ResourceManager. + @return new ApplicationId allocated by the + ResourceManager]]> + + + + + ResourceManager in the cluster. + @return maximum capability of allocated resources in the cluster]]> + + + + The response sent by the ResourceManager to the client for + a request to get a new {@link ApplicationId} for submitting applications.

    + +

    Clients can submit an application with the returned + {@link ApplicationId}.

    + + @see ApplicationClientProtocol#getNewApplication(GetNewApplicationRequest)]]> +     +     + + + + + + + + + The request sent by clients to get a new {@code ReservationId} for + submitting an reservation.

    + + {@code ApplicationClientProtocol#getNewReservation(GetNewReservationRequest)}]]> +     +     + + + + + + + + + + + + The response sent by the ResourceManager to the client for + a request to get a new {@link ReservationId} for submitting reservations.

    + +

    Clients can submit an reservation with the returned + {@link ReservationId}.

    + + {@code ApplicationClientProtocol#getNewReservation(GetNewReservationRequest)}]]> +     +     + + + + + + + + + + + + + + + + + + + + + + The request from clients to get nodes to attributes mapping + in the cluster from the ResourceManager. +

    + + @see ApplicationClientProtocol#getNodesToAttributes + (GetNodesToAttributesRequest)]]> +     +     + + + + + + + + + + + + + + + + + + + The response sent by the ResourceManager to a client requesting + nodes to attributes mapping. +

    + + @see ApplicationClientProtocol#getNodesToAttributes + (GetNodesToAttributesRequest)]]> +     +     + + + + + + + + + + + + + + queue name for which to get queue information. + @return queue name for which to get queue information]]> + + + + + + queue name for which to get queue information + @param queueName queue name for which to get queue information]]> + + + + + active applications required? + @return true if applications' information is to be included, + else false]]> + + + + + + active applications? + @param includeApplications fetch information about active + applications?]]> + + + + + child queues required? + @return true if information about child queues is required, + else false]]> + + + + + + child queues? + @param includeChildQueues fetch information about child queues?]]> + + + + + child queue hierarchy required? + @return true if information about entire hierarchy is + required, false otherwise]]> + + + + + + child queue hierarchy? + @param recursive fetch information on the entire child queue + hierarchy?]]> + + + + The request sent by clients to get queue information + from the ResourceManager.

    + + @see ApplicationClientProtocol#getQueueInfo(GetQueueInfoRequest)]]> +     +     + + + + + + + + QueueInfo for the specified queue. + @return QueueInfo for the specified queue]]> + + + + + The response includes a {@link QueueInfo} which has details such as + queue name, used/total capacities, running applications, child queues etc. + + @see QueueInfo + @see ApplicationClientProtocol#getQueueInfo(GetQueueInfoRequest)]]> + + + + + + + + + + + The request sent by clients to the ResourceManager to + get queue acls for the current user.

    + +

    Currently, this is empty.

    + + @see ApplicationClientProtocol#getQueueUserAcls(GetQueueUserAclsInfoRequest)]]> +     +     + + + + + + + + QueueUserACLInfo per queue for the user. + @return QueueUserACLInfo per queue for the user]]> + + + + The response sent by the ResourceManager to clients + seeking queue acls for the user.

    + +

    The response contains a list of {@link QueueUserACLInfo} which + provides information about {@link QueueACL} per queue.

    + + @see QueueACL + @see QueueUserACLInfo + @see ApplicationClientProtocol#getQueueUserAcls(GetQueueUserAclsInfoRequest)]]> +     +     + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Note: {@link NMToken} will be used for authenticating communication with + {@code NodeManager}. + @return the list of container tokens to be used for authorization during + container resource increase. + @see NMToken]]> + + + + + + AllocateResponse.getIncreasedContainers. + The token contains the container id and resource capability required for + container resource increase. + @param containersToIncrease the list of container tokens to be used + for container resource increase.]]> + + + + The request sent by Application Master to the + Node Manager to change the resource quota of a container.

    + + @see ContainerManagementProtocol#increaseContainersResource(IncreaseContainersResourceRequest)]]> +     +     + + + + + + + + + + + + + + + + + + The response sent by the NodeManager to the + ApplicationMaster when asked to increase container resource. +

    + + @see ContainerManagementProtocol#increaseContainersResource(IncreaseContainersResourceRequest)]]> +     +     + + + + + + + + + + + ApplicationId of the application to be aborted. + @return ApplicationId of the application to be aborted]]> + + + + + + + + diagnostics to which the application is being killed. + @return diagnostics to which the application is being killed]]> + + + + + + diagnostics to which the application is being killed. + @param diagnostics diagnostics to which the application is being + killed]]> + + + + The request sent by the client to the ResourceManager + to abort a submitted application.

    + +

    The request includes the {@link ApplicationId} of the application to be + aborted.

    + + @see ApplicationClientProtocol#forceKillApplication(KillApplicationRequest)]]> +     +     + + + + + + + + + + + + ResourceManager to the client aborting + a submitted application. +

    + The response, includes: +

      +
    • + A flag which indicates that the process of killing the application is + completed or not. +
      • +
    + Note: user is recommended to wait until this flag becomes true, otherwise if + the ResourceManager crashes before the process of killing the + application is completed, the ResourceManager may retry this + application on recovery. + + @see ApplicationClientProtocol#forceKillApplication(KillApplicationRequest)]]> +     +     + + + + + + + + + + + + ApplicationId of the application to be moved. + @return ApplicationId of the application to be moved]]> + + + + + + ApplicationId of the application to be moved. + @param appId ApplicationId of the application to be moved]]> + + + + + + + + + + + + + + + The request sent by the client to the ResourceManager + to move a submitted application to a different queue.

    + +

    The request includes the {@link ApplicationId} of the application to be + moved and the queue to place it in.

    + + @see ApplicationClientProtocol#moveApplicationAcrossQueues(MoveApplicationAcrossQueuesRequest)]]> +     +     + + + + + + + + The response sent by the ResourceManager to the client moving + a submitted application to a different queue. +

    +

    + A response without exception means that the move has completed successfully. +

    + + @see ApplicationClientProtocol#moveApplicationAcrossQueues(MoveApplicationAcrossQueuesRequest)]]> +     +     + + + + + + + + + + + RegisterApplicationMasterRequest. + If port, trackingUrl is not used, use the following default value: +
      +
    • port: -1
      • +
    • trackingUrl: null
      • +
    + The port is allowed to be any integer larger than or equal to -1. + @return the new instance of RegisterApplicationMasterRequest]]> +     +     + + + host on which the ApplicationMaster is + running. + @return host on which the ApplicationMaster is running]]> + + + + + + host on which the ApplicationMaster is + running. + @param host host on which the ApplicationMaster + is running]]> + + + + + RPC port on which the {@code ApplicationMaster} is + responding. + @return the RPC port on which the {@code ApplicationMaster} + is responding]]> + + + + + + RPC port on which the {@code ApplicationMaster} is + responding. + @param port RPC port on which the {@code ApplicationMaster} + is responding]]> + + + + + tracking URL for the ApplicationMaster. + This url if contains scheme then that will be used by resource manager + web application proxy otherwise it will default to http. + @return tracking URL for the ApplicationMaster]]> + + + + + + tracking URLfor the ApplicationMaster while + it is running. This is the web-URL to which ResourceManager or + web-application proxy will redirect client/users while the application and + the ApplicationMaster are still running. +

    + If the passed url has a scheme then that will be used by the + ResourceManager and web-application proxy, otherwise the scheme will + default to http. +

    +

    + Empty, null, "N/A" strings are all valid besides a real URL. In case an url + isn't explicitly passed, it defaults to "N/A" on the ResourceManager. +

    + + @param trackingUrl + tracking URLfor the ApplicationMaster]]> + + + + + PlacementConstraint associated with the tags, i.e., each + {@link org.apache.hadoop.yarn.api.records.SchedulingRequest} that has those + tags will be placed taking into account the corresponding constraint. + + @return A map of Placement Constraints.]]> + + + + + + PlacementConstraint associated with the tags. + For example: + Map < + <hb_regionserver> -> node_anti_affinity, + <hb_regionserver, hb_master> -> rack_affinity, + ... + > + @param placementConstraints Placement Constraint Mapping.]]> + + + + + The registration includes details such as: +

      +
    • Hostname on which the AM is running.
      • +
    • RPC Port
      • +
    • Tracking URL
      • +
    + + @see ApplicationMasterProtocol#registerApplicationMaster(RegisterApplicationMasterRequest)]]> +     +     + + + + + + + + ResourceManager in the cluster. + @return maximum capability of allocated resources in the cluster]]> + + + + + ApplicationACLs for the application. + @return all the ApplicationACLs]]> + + + + + Get ClientToAMToken master key.

    +

    The ClientToAMToken master key is sent to ApplicationMaster + by ResourceManager via {@link RegisterApplicationMasterResponse} + , used to verify corresponding ClientToAMToken.

    + @return ClientToAMToken master key]]> +     +     + + + + + + + + + Get the queue that the application was placed in.

    + @return the queue that the application was placed in.]]> + + + + + + Set the queue that the application was placed in.

    ]]> + + + + + + Get the list of running containers as viewed by + ResourceManager from previous application attempts. +

    + + @return the list of running containers as viewed by + ResourceManager from previous application attempts + @see RegisterApplicationMasterResponse#getNMTokensFromPreviousAttempts()]]> +     +     + + + + + + + + + + + + + + + + + + + + + + + The response contains critical details such as: +
      +
    • Maximum capability for allocated resources in the cluster.
      • +
    • {@code ApplicationACL}s for the application.
      • +
    • ClientToAMToken master key.
      • +
    + + @see ApplicationMasterProtocol#registerApplicationMaster(RegisterApplicationMasterRequest)]]> +     +     + + + + + + + + + + + + + + + + ContainerId of the container to re-initialize. + + @return ContainerId of the container to re-initialize.]]> + + + + + ContainerLaunchContext to re-initialize the container + with. + + @return ContainerLaunchContext of to re-initialize the + container with.]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + ApplicationId of the resource to be released. + + @return ApplicationId]]> + + + + + + ApplicationId of the resource to be released. + + @param id ApplicationId]]> + + + + + key of the resource to be released. + + @return key]]> + + + + + + key of the resource to be released. + + @param key unique identifier for the resource]]> + + + + The request from clients to release a resource in the shared cache.

    ]]> +     +     + + + + + + + + The response to clients from the SharedCacheManager when + releasing a resource in the shared cache. +

    + +

    + Currently, this is empty. +

    ]]> +     +     + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + The response sent by the ResourceManager to a client on + reservation submission.

    + +

    Currently, this is empty.

    + + {@code ApplicationClientProtocol#submitReservation( + ReservationSubmissionRequest)}]]> +     +     + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + ContainerId of the container to localize resources. + + @return ContainerId of the container to localize resources.]]> + + + + + LocalResource required by the container. + + @return all LocalResource required by the container]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + ContainerId of the container to signal. + @return ContainerId of the container to signal.]]> + + + + + + ContainerId of the container to signal.]]> + + + + + SignalContainerCommand of the signal request. + @return SignalContainerCommand of the signal request.]]> + + + + + + SignalContainerCommand of the signal request.]]> + + + + The request sent by the client to the ResourceManager + or by the ApplicationMaster to the NodeManager + to signal a container. + @see SignalContainerCommand

    ]]> +     +     + + + + + + + The response sent by the ResourceManager to the client + signalling a container.

    + +

    Currently it's empty.

    + + @see ApplicationClientProtocol#signalToContainer(SignalContainerRequest)]]> +     +     + + + + + + + + + + + + ContainerLaunchContext for the container to be started + by the NodeManager. + + @return ContainerLaunchContext for the container to be started + by the NodeManager]]> + + + + + + ContainerLaunchContext for the container to be started + by the NodeManager + @param context ContainerLaunchContext for the container to be + started by the NodeManager]]> + + + + + + Note: {@link NMToken} will be used for authenticating communication with + {@code NodeManager}. + @return the container token to be used for authorization during starting + container. + @see NMToken + @see ContainerManagementProtocol#startContainers(StartContainersRequest)]]> + + + + + + + The request sent by the ApplicationMaster to the + NodeManager to start a container.

    + +

    The ApplicationMaster has to provide details such as + allocated resource capability, security tokens (if enabled), command + to be executed to start the container, environment for the process, + necessary binaries/jar/shared-objects etc. via the + {@link ContainerLaunchContext}.

    + + @see ContainerManagementProtocol#startContainers(StartContainersRequest)]]> +     +     + + + + + + + + + + + + + + + + + + + + + + The request which contains a list of {@link StartContainerRequest} sent by + the ApplicationMaster to the NodeManager to + start containers. +

    + +

    + In each {@link StartContainerRequest}, the ApplicationMaster has + to provide details such as allocated resource capability, security tokens (if + enabled), command to be executed to start the container, environment for the + process, necessary binaries/jar/shared-objects etc. via the + {@link ContainerLaunchContext}. +

    + + @see ContainerManagementProtocol#startContainers(StartContainersRequest)]]> +     +     + + + + + + + + ContainerId s of the containers that are + started successfully. + + @return the list of ContainerId s of the containers that are + started successfully. + @see ContainerManagementProtocol#startContainers(StartContainersRequest)]]> + + + + + + + + + + + Get the meta-data from all auxiliary services running on the + NodeManager. +

    +

    + The meta-data is returned as a Map between the auxiliary service names and + their corresponding per service meta-data as an opaque blob + ByteBuffer +

    + +

    + To be able to interpret the per-service meta-data, you should consult the + documentation for the Auxiliary-service configured on the NodeManager +

    + + @return a Map between the names of auxiliary services and their + corresponding meta-data]]> +     +     + + + The response sent by the NodeManager to the + ApplicationMaster when asked to start an allocated + container. +

    + + @see ContainerManagementProtocol#startContainers(StartContainersRequest)]]> +     +     + + + + + + + + + + + ContainerIds of the containers to be stopped. + @return ContainerIds of containers to be stopped]]> + + + + + + ContainerIds of the containers to be stopped. + @param containerIds ContainerIds of the containers to be stopped]]> + + + + The request sent by the ApplicationMaster to the + NodeManager to stop containers.

    + + @see ContainerManagementProtocol#stopContainers(StopContainersRequest)]]> +     +     + + + + + + + + + + + + + + + + + + The response sent by the NodeManager to the + ApplicationMaster when asked to stop allocated + containers. +

    + + @see ContainerManagementProtocol#stopContainers(StopContainersRequest)]]> +     +     + + + + + + + + + + + ApplicationSubmissionContext for the application. + @return ApplicationSubmissionContext for the application]]> + + + + + + ApplicationSubmissionContext for the application. + @param context ApplicationSubmissionContext for the + application]]> + + + + The request sent by a client to submit an application to the + ResourceManager.

    + +

    The request, via {@link ApplicationSubmissionContext}, contains + details such as queue, {@link Resource} required to run the + ApplicationMaster, the equivalent of + {@link ContainerLaunchContext} for launching the + ApplicationMaster etc. + + @see ApplicationClientProtocol#submitApplication(SubmitApplicationRequest)]]> + + + + + + + + + The response sent by the ResourceManager to a client on + application submission.

    + +

    Currently, this is empty.

    + + @see ApplicationClientProtocol#submitApplication(SubmitApplicationRequest)]]> +     +     + + + + + + + + + + + + ApplicationId of the application. + + @return ApplicationId of the application]]> + + + + + + ApplicationId of the application. + + @param applicationId ApplicationId of the application]]> + + + + + Priority of the application to be set. + + @return Priority of the application to be set.]]> + + + + + + Priority of the application. + + @param priority Priority of the application]]> + + + + + The request sent by the client to the ResourceManager to set or + update the application priority. +

    +

    + The request includes the {@link ApplicationId} of the application and + {@link Priority} to be set for an application +

    + + @see ApplicationClientProtocol#updateApplicationPriority(UpdateApplicationPriorityRequest)]]> +     +     + + + + + + + + + + + Priority of the application to be set. + @return Updated Priority of the application.]]> + + + + + + Priority of the application. + + @param priority Priority of the application]]> + + + + + The response sent by the ResourceManager to the client on update + the application priority. +

    +

    + A response without exception means that the move has completed successfully. +

    + + @see ApplicationClientProtocol#updateApplicationPriority(UpdateApplicationPriorityRequest)]]> +     +     + + + + + + + + + + + + ApplicationId of the application. + @return ApplicationId of the application]]> + + + + + + ApplicationId of the application. + @param applicationId ApplicationId of the application]]> + + + + + ApplicationTimeouts of the application. Timeout value is + in ISO8601 standard with format yyyy-MM-dd'T'HH:mm:ss.SSSZ. + @return all ApplicationTimeouts of the application.]]> + + + + + + ApplicationTimeouts for the application. Timeout value + is absolute. Timeout value should meet ISO8601 format. Support ISO8601 + format is yyyy-MM-dd'T'HH:mm:ss.SSSZ. All pre-existing Map entries + are cleared before adding the new Map. + @param applicationTimeouts ApplicationTimeoutss for the + application]]> + + + + + The request sent by the client to the ResourceManager to set or + update the application timeout. +

    +

    + The request includes the {@link ApplicationId} of the application and timeout + to be set for an application +

    ]]> +     +     + + + + + + + + + + ApplicationTimeouts of the application. Timeout value is + in ISO8601 standard with format yyyy-MM-dd'T'HH:mm:ss.SSSZ. + @return all ApplicationTimeouts of the application.]]> + + + + + + ApplicationTimeouts for the application. Timeout value + is absolute. Timeout value should meet ISO8601 format. Support ISO8601 + format is yyyy-MM-dd'T'HH:mm:ss.SSSZ. All pre-existing Map entries + are cleared before adding the new Map. + @param applicationTimeouts ApplicationTimeoutss for the + application]]> + + + + + The response sent by the ResourceManager to the client on update + application timeout. +

    +

    + A response without exception means that the update has completed + successfully. +

    ]]> +     +     + + + + + + + + ApplicationId of the resource to be used. + + @return ApplicationId]]> + + + + + + ApplicationId of the resource to be used. + + @param id ApplicationId]]> + + + + + key of the resource to be used. + + @return key]]> + + + + + + key of the resource to be used. + + @param key unique identifier for the resource]]> + + + + + The request from clients to the SharedCacheManager that claims a + resource in the shared cache. +

    ]]> +     +     + + + + + + + + Path corresponding to the requested resource in the + shared cache. + + @return String A Path if the resource exists in the shared + cache, null otherwise]]> + + + + + + Path corresponding to a resource in the shared cache. + + @param p A Path corresponding to a resource in the shared + cache]]> + + + + + The response from the SharedCacheManager to the client that indicates whether + a requested resource exists in the cache. +

    ]]> +     +     + +     + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + ApplicationId of the ApplicationAttempId. + @return ApplicationId of the ApplicationAttempId]]> + + + + + attempt id of the Application. + @return attempt id of the Application]]> + + + + + + + + + + + + + + + + + + + + + ApplicationAttemptId denotes the particular attempt + of an ApplicationMaster for a given {@link ApplicationId}.

    + +

    Multiple attempts might be needed to run an application to completion due + to temporal failures of the ApplicationMaster such as hardware + failures, connectivity issues etc. on the node on which it was scheduled.

    ]]> +     +     + + + + + + + + + + + + + + + + + + YarnApplicationAttemptState of the application attempt. + + @return YarnApplicationAttemptState of the application attempt]]> + + + + + RPC port of this attempt ApplicationMaster. + + @return RPC port of this attempt ApplicationMaster]]> + + + + + host on which this attempt of + ApplicationMaster is running. + + @return host on which this attempt of + ApplicationMaster is running]]> + + + + + diagnositic information of the application attempt in case + of errors. + + @return diagnositic information of the application attempt in case + of errors]]> + + + + + tracking url for the application attempt. + + @return tracking url for the application attempt]]> + + + + + original tracking url for the application attempt. + + @return original tracking url for the application attempt]]> + + + + + ApplicationAttemptId of this attempt of the + application + + @return ApplicationAttemptId of the attempt]]> + + + + + ContainerId of AMContainer for this attempt + + @return ContainerId of the attempt]]> + + + + + + + finish time of the application. + + @return finish time of the application]]> + + + + + It includes details such as: +
      +
    • {@link ApplicationAttemptId} of the application.
      • +
    • Host on which the ApplicationMaster of this attempt is + running.
      • +
    • RPC port of the ApplicationMaster of this attempt.
      • +
    • Tracking URL.
      • +
    • Diagnostic information in case of errors.
      • +
    • {@link YarnApplicationAttemptState} of the application attempt.
      • +
    • {@link ContainerId} of the master Container.
      • +
    ]]> +     +     + + + + + + + + + + + + + ApplicationId + which is unique for all applications started by a particular instance + of the ResourceManager. + @return short integer identifier of the ApplicationId]]> + + + + + start time of the ResourceManager which is + used to generate globally unique ApplicationId. + @return start time of the ResourceManager]]> + + + + + + + + + + + + + + + + + + + + + ApplicationId represents the globally unique + identifier for an application.

    + +

    The globally unique nature of the identifier is achieved by using the + cluster timestamp i.e. start-time of the + ResourceManager along with a monotonically increasing counter + for the application.

    ]]> +     +     + + + + + + + + ApplicationId of the application. + @return ApplicationId of the application]]> + + + + + ApplicationAttemptId of the current + attempt of the application + @return ApplicationAttemptId of the attempt]]> + + + + + user who submitted the application. + @return user who submitted the application]]> + + + + + queue to which the application was submitted. + @return queue to which the application was submitted]]> + + + + + name of the application. + @return name of the application]]> + + + + + host on which the ApplicationMaster + is running. + @return host on which the ApplicationMaster + is running]]> + + + + + RPC port of the ApplicationMaster. + @return RPC port of the ApplicationMaster]]> + + + + + client token for communicating with the + ApplicationMaster. +

    + ClientToAMToken is the security token used by the AMs to verify + authenticity of any client. +

    + +

    + The ResourceManager, provides a secure token (via + {@link ApplicationReport#getClientToAMToken()}) which is verified by the + ApplicationMaster when the client directly talks to an AM. +

    + @return client token for communicating with the + ApplicationMaster]]> +     +     + + + YarnApplicationState of the application. + @return YarnApplicationState of the application]]> + + + + + diagnositic information of the application in case of + errors. + @return diagnositic information of the application in case + of errors]]> + + + + + tracking url for the application. + @return tracking url for the application]]> + + + + + start time of the application. + @return start time of the application]]> + + + + + + + + + finish time of the application. + @return finish time of the application]]> + + + + + final finish status of the application. + @return final finish status of the application]]> + + + + + + + + + + + + + + + + + + + + + + + + + + The AMRM token is required for AM to RM scheduling operations. For + managed Application Masters YARN takes care of injecting it. For unmanaged + Applications Masters, the token must be obtained via this method and set + in the {@link org.apache.hadoop.security.UserGroupInformation} of the + current user. +

    + The AMRM token will be returned only if all the following conditions are + met: +

      +
    • the requester is the owner of the ApplicationMaster
      • +
    • the application master is an unmanaged ApplicationMaster
      • +
    • the application master is in ACCEPTED state
      • +
    + Else this method returns NULL. + + @return the AM to RM token if available.]]> +     +     + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + It includes details such as: +
      +
    • {@link ApplicationId} of the application.
      • +
    • Applications user.
      • +
    • Application queue.
      • +
    • Application name.
      • +
    • Host on which the ApplicationMaster is running.
      • +
    • RPC port of the ApplicationMaster.
      • +
    • Tracking URL.
      • +
    • {@link YarnApplicationState} of the application.
      • +
    • Diagnostic information in case of errors.
      • +
    • Start time of the application.
      • +
    • Client {@link Token} of the application (if security is enabled).
      • +
    + + @see ApplicationClientProtocol#getApplicationReport(org.apache.hadoop.yarn.api.protocolrecords.GetApplicationReportRequest)]]> +     +     + + + + + + + + + + + + + Resource. -1 for invalid/inaccessible reports. + @return the used Resource]]> + + + + + Resource. -1 for invalid/inaccessible reports. + @return the reserved Resource]]> + + + + + Resource. -1 for invalid/inaccessible reports. + @return the needed Resource]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + ApplicationId of the submitted application. + @return ApplicationId of the submitted application]]> + + + + + + ApplicationId of the submitted application. + @param applicationId ApplicationId of the submitted + application]]> + + + + + name. + @return application name]]> + + + + + + name. + @param applicationName application name]]> + + + + + queue to which the application is being submitted. + @return queue to which the application is being submitted]]> + + + + + + queue to which the application is being submitted + @param queue queue to which the application is being submitted]]> + + + + + Priority of the application. + @return Priority of the application]]> + + + + + ContainerLaunchContext to describe the + Container with which the ApplicationMaster is + launched. + @return ContainerLaunchContext for the + ApplicationMaster container]]> + + + + + + ContainerLaunchContext to describe the + Container with which the ApplicationMaster is + launched. + @param amContainer ContainerLaunchContext for the + ApplicationMaster container]]> + + + + + YarnApplicationState. + Such apps will not be retried by the RM on app attempt failure. + The default value is false. + @return true if the AM is not managed by the RM]]> + + + + + + + + + + + + + + + + + + + + + + ApplicationMaster for this + application. Please note this will be DEPRECATED, use getResource + in getAMContainerResourceRequest instead. + + @return the resource required by the ApplicationMaster for + this application.]]> + + + + + + ApplicationMaster for this + application. + + @param resource the resource required by the ApplicationMaster + for this application.]]> + + + + + + + + + + + + + + + + + + + + + + + For managed AM, if the flag is true, running containers will not be killed + when application attempt fails and these containers will be retrieved by + the new application attempt on registration via + {@link ApplicationMasterProtocol#registerApplicationMaster(RegisterApplicationMasterRequest)}. +

    +

    + For unmanaged AM, if the flag is true, RM allows re-register and returns + the running containers in the same attempt back to the UAM for HA. +

    + + @param keepContainers the flag which indicates whether to keep containers + across application attempts.]]> +     +     + + + + + + + + + + + + + + + + + + + + + + + + + getResource and getPriority of + ApplicationSubmissionContext. + + Number of containers and Priority will be ignored. + + @return ResourceRequest of the AM container + @deprecated See {@link #getAMContainerResourceRequests()}]]> + + + + + + + + + + + getAMContainerResourceRequest and its behavior. + + Number of containers and Priority will be ignored. + + @return List of ResourceRequests of the AM container]]> + + + + + + + + + + + + + + + + + + + + + + LogAggregationContext of the application + + @return LogAggregationContext of the application]]> + + + + + + LogAggregationContext for the application + + @param logAggregationContext + for the application]]> + + + + + + + + + + + + + + + + ApplicationTimeouts of the application. Timeout value is + in seconds. + @return all ApplicationTimeouts of the application.]]> + + + + + + ApplicationTimeouts for the application in seconds. + All pre-existing Map entries are cleared before adding the new Map. +

    + Note: If application timeout value is less than or equal to zero + then application submission will throw an exception. +

    + @param applicationTimeouts ApplicationTimeoutss for the + application]]> +     +     + + + + + + + + + + + + + + It includes details such as: +
      +
    • {@link ApplicationId} of the application.
      • +
    • Application user.
      • +
    • Application name.
      • +
    • {@link Priority} of the application.
      • +
    • + {@link ContainerLaunchContext} of the container in which the + ApplicationMaster is executed. +
      • +
    • + maxAppAttempts. The maximum number of application attempts. + It should be no larger than the global number of max attempts in the + YARN configuration. +
      • +
    • + attemptFailuresValidityInterval. The default value is -1. + when attemptFailuresValidityInterval in milliseconds is set to + {@literal >} 0, the failure number will no take failures which happen + out of the validityInterval into failure count. If failure count + reaches to maxAppAttempts, the application will be failed. +
      • +
    • Optional, application-specific {@link LogAggregationContext}
      • +
    + + @see ContainerLaunchContext + @see ApplicationClientProtocol#submitApplication(org.apache.hadoop.yarn.api.protocolrecords.SubmitApplicationRequest)]]> +     +     + + + + + + + + + + + + + + + + + + + + + + + + expiryTime for given timeout type. + @return expiryTime in ISO8601 standard with format + yyyy-MM-dd'T'HH:mm:ss.SSSZ.]]> + + + + + + expiryTime for given timeout type. + @param expiryTime in ISO8601 standard with format + yyyy-MM-dd'T'HH:mm:ss.SSSZ.]]> + + + + + Remaining Time of an application for given timeout type. + @return Remaining Time in seconds.]]> + + + + + + Remaining Time of an application for given timeout type. + @param remainingTime in seconds.]]> + + + + +
  • {@link ApplicationTimeoutType} of the timeout type.
    • +
  • Expiry time in ISO8601 standard with format + yyyy-MM-dd'T'HH:mm:ss.SSSZ or "UNLIMITED".
    • +
  • Remaining time in seconds.
    • + + The possible values for {ExpiryTime, RemainingTimeInSeconds} are +
      +
    • {UNLIMITED,-1} : Timeout is not configured for given timeout type + (LIFETIME).
      • +
    • {ISO8601 date string, 0} : Timeout is configured and application has + completed.
      • +
    • {ISO8601 date string, greater than zero} : Timeout is configured and + application is RUNNING. Application will be timed out after configured + value.
      • +
    ]]> +     +     + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Resource allocated to the container. + @return Resource allocated to the container]]> + + + + + Priority at which the Container was + allocated. + @return Priority at which the Container was + allocated]]> + + + + + ContainerToken for the container. +

    ContainerToken is the security token used by the framework + to verify authenticity of any Container.

    + +

    The ResourceManager, on container allocation provides a + secure token which is verified by the NodeManager on + container launch.

    + +

    Applications do not need to care about ContainerToken, they + are transparently handled by the framework - the allocated + Container includes the ContainerToken.

    + + @see ApplicationMasterProtocol#allocate(org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest) + @see ContainerManagementProtocol#startContainers(org.apache.hadoop.yarn.api.protocolrecords.StartContainersRequest) + + @return ContainerToken for the container]]> +     +     + + + ID corresponding to the original {@code + ResourceRequest{@link #getAllocationRequestId()}}s which is satisfied by + this allocated {@code Container}. +

    + The scheduler may return multiple {@code AllocateResponse}s corresponding + to the same ID as and when scheduler allocates {@code Container}s. + Applications can continue to completely ignore the returned ID in + the response and use the allocation for any of their outstanding requests. +

    + + @return the ID corresponding to the original allocation request + which is satisfied by this allocation.]]> + + + + + The {@code ResourceManager} is the sole authority to allocate any + {@code Container} to applications. The allocated {@code Container} + is always on a single node and has a unique {@link ContainerId}. It has + a specific amount of {@link Resource} allocated. +

    + It includes details such as: +

      +
    • {@link ContainerId} for the container, which is globally unique.
      • +
    • + {@link NodeId} of the node on which it is allocated. +
      • +
    • HTTP uri of the node.
      • +
    • {@link Resource} allocated to the container.
      • +
    • {@link Priority} at which the container was allocated.
      • +
    • + Container {@link Token} of the container, used to securely verify + authenticity of the allocation. +
      • +
    + + Typically, an {@code ApplicationMaster} receives the {@code Container} + from the {@code ResourceManager} during resource-negotiation and then + talks to the {@code NodeManager} to start/stop containers. + + @see ApplicationMasterProtocol#allocate(org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest) + @see ContainerManagementProtocol#startContainers(org.apache.hadoop.yarn.api.protocolrecords.StartContainersRequest) + @see ContainerManagementProtocol#stopContainers(org.apache.hadoop.yarn.api.protocolrecords.StopContainersRequest)]]> +     +     + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + ApplicationAttemptId of the application to which the + Container was assigned. +

    + Note: If containers are kept alive across application attempts via + {@link ApplicationSubmissionContext#setKeepContainersAcrossApplicationAttempts(boolean)} + the ContainerId does not necessarily contain the current + running application attempt's ApplicationAttemptId This + container can be allocated by previously exited application attempt and + managed by the current running attempt thus have the previous application + attempt's ApplicationAttemptId. +

    + + @return ApplicationAttemptId of the application to which the + Container was assigned]]> +     +     + + + ContainerId, + which doesn't include epoch. Note that this method will be marked as + deprecated, so please use getContainerId instead. + @return lower 32 bits of identifier of the ContainerId]]> + + + + + ContainerId. Upper 24 bits are + reserved as epoch of cluster, and lower 40 bits are reserved as + sequential number of containers. + @return identifier of the ContainerId]]> + + + + + + + + + + + + + + + + + + + + + + + + ContainerId represents a globally unique identifier + for a {@link Container} in the cluster.

    ]]> +     +     + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + LocalResource required by the container. + @return all LocalResource required by the container]]> + + + + + + LocalResource required by the container. All pre-existing + Map entries are cleared before adding the new Map + @param localResources LocalResource required by the container]]> + + + + + + Get application-specific binary service data. This is a map keyed + by the name of each {@link AuxiliaryService} that is configured on a + NodeManager and value correspond to the application specific data targeted + for the keyed {@link AuxiliaryService}. +

    + +

    + This will be used to initialize this application on the specific + {@link AuxiliaryService} running on the NodeManager by calling + {@link AuxiliaryService#initializeApplication(ApplicationInitializationContext)} +

    + + @return application-specific binary service data]]> +     +     + + + + + Set application-specific binary service data. This is a map keyed + by the name of each {@link AuxiliaryService} that is configured on a + NodeManager and value correspond to the application specific data targeted + for the keyed {@link AuxiliaryService}. All pre-existing Map entries are + preserved. +

    + + @param serviceData + application-specific binary service data]]> +     +     + + + environment variables for the container. + @return environment variables for the container]]> + + + + + + environment variables for the container. All pre-existing Map + entries are cleared before adding the new Map + @param environment environment variables for the container]]> + + + + + commands for launching the container. + @return the list of commands for launching the container]]> + + + + + + commands for launching the container. All + pre-existing List entries are cleared before adding the new List + @param commands the list of commands for launching the container]]> + + + + + ApplicationACLs for the application. + @return all the ApplicationACLs]]> + + + + + + ApplicationACLs for the application. All pre-existing + Map entries are cleared before adding the new Map + @param acls ApplicationACLs for the application]]> + + + + + ContainerRetryContext to relaunch container. + @return ContainerRetryContext to relaunch container.]]> + + + + + + ContainerRetryContext to relaunch container. + @param containerRetryContext ContainerRetryContext to + relaunch container.]]> + + + + + It includes details such as: +
      +
    • {@link ContainerId} of the container.
      • +
    • {@link Resource} allocated to the container.
      • +
    • User to whom the container is allocated.
      • +
    • Security tokens (if security is enabled).
      • +
    • + {@link LocalResource} necessary for running the container such + as binaries, jar, shared-objects, side-files etc. +
      • +
    • Optional, application-specific binary service data.
      • +
    • Environment variables for the launched process.
      • +
    • Command to launch the container.
      • +
    • Retry strategy when container exits with failure.
      • +
    + + @see ContainerManagementProtocol#startContainers(org.apache.hadoop.yarn.api.protocolrecords.StartContainersRequest)]]> +     +     + + + + + + + + ContainerId of the container. + + @return ContainerId of the container.]]> + + + + + + + + Resource of the container. + + @return allocated Resource of the container.]]> + + + + + + + + NodeId where container is running. + + @return allocated NodeId where container is running.]]> + + + + + + + + Priority of the container. + + @return allocated Priority of the container.]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + ContainerState of the container. + + @return final ContainerState of the container.]]> + + + + + + + + exit status of the container. + + @return final exit status of the container.]]> + + + + + + + + + + + + + + + + + + It includes details such as: +
      +
    • {@link ContainerId} of the container.
      • +
    • Allocated Resources to the container.
      • +
    • Assigned Node id.
      • +
    • Assigned Priority.
      • +
    • Creation Time.
      • +
    • Finish Time.
      • +
    • Container Exit Status.
      • +
    • {@link ContainerState} of the container.
      • +
    • Diagnostic information in case of errors.
      • +
    • Log URL.
      • +
    • nodeHttpAddress
      • +
    ]]> +     +     + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + It provides details such as: +
      +
    • + {@link ContainerRetryPolicy} : + - NEVER_RETRY(DEFAULT value): no matter what error code is when container + fails to run, just do not retry. + - RETRY_ON_ALL_ERRORS: no matter what error code is, when container fails + to run, just retry. + - RETRY_ON_SPECIFIC_ERROR_CODES: when container fails to run, do retry if + the error code is one of errorCodes, otherwise do not retry. + + Note: if error code is 137(SIGKILL) or 143(SIGTERM), it will not retry + because it is usually killed on purpose. +
      • +
    • + maxRetries specifies how many times to retry if need to retry. + If the value is -1, it means retry forever. +
      • +
    • retryInterval specifies delaying some time before relaunch + container, the unit is millisecond.
      • +
    • + failuresValidityInterval: default value is -1. + When failuresValidityInterval in milliseconds is set to {@literal >} 0, + the failure number will not take failures which happen out of the + failuresValidityInterval into failure count. If failure count + reaches to maxRetries, the container will be failed. +
      • +
    ]]> +     +     + + + + + + + + + + Retry policy for relaunching a Container.

    ]]> +     +     + + + + + + + + + + State of a Container.

    ]]> +     +     + + + + + + + + ContainerId of the container. + @return ContainerId of the container]]> + + + + + ExecutionType of the container. + @return ExecutionType of the container]]> + + + + + ContainerState of the container. + @return ContainerState of the container]]> + + + + + Get the exit status for the container.

    + +

    Note: This is valid only for completed containers i.e. containers + with state {@link ContainerState#COMPLETE}. + Otherwise, it returns an ContainerExitStatus.INVALID. +

    + +

    Containers killed by the framework, either due to being released by + the application or being 'lost' due to node failures etc. have a special + exit code of ContainerExitStatus.ABORTED.

    + +

    When threshold number of the nodemanager-local-directories or + threshold number of the nodemanager-log-directories become bad, then + container is not launched and is exited with ContainersExitStatus.DISKS_FAILED. +

    + + @return exit status for the container]]> +     +     + + + diagnostic messages for failed containers. + @return diagnostic messages for failed containers]]> + + + + + Resource allocated to the container. + @return Resource allocated to the container]]> + + + + + + + + + + + + + + + It provides details such as: +
      +
    • {@code ContainerId} of the container.
      • +
    • {@code ExecutionType} of the container.
      • +
    • {@code ContainerState} of the container.
      • +
    • Exit status of a completed container.
      • +
    • Diagnostic message for a failed container.
      • +
    • {@link Resource} allocated to the container.
      • +
    ]]> +     +     + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + The execution types are the following: +
      +
    • {@link #GUARANTEED} - this container is guaranteed to start its + execution, once the corresponding start container request is received by + an NM. +
    • {@link #OPPORTUNISTIC} - the execution of this container may not start + immediately at the NM that receives the corresponding start container + request (depending on the NM's available resources). Moreover, it may be + preempted if it blocks a GUARANTEED container from being executed. +
    ]]> +     +     + + + + + + + + + + + + + + + + + + + ExecutionType of the requested container. + + @param execType + ExecutionType of the requested container]]> + + + + + ExecutionType. + + @return ExecutionType.]]> + + + + + + + + + + + ResourceRequest. + Defaults to false. + @return whether ExecutionType request should be strictly honored]]> + + + + + + + + + ExecutionType as well as flag that explicitly asks the + configuredScheduler to return Containers of exactly the Execution Type + requested.]]> + + + + + + + + + + + + Application.]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + location of the resource to be localized. + @return location of the resource to be localized]]> + + + + + + location of the resource to be localized. + @param resource location of the resource to be localized]]> + + + + + size of the resource to be localized. + @return size of the resource to be localized]]> + + + + + + size of the resource to be localized. + @param size size of the resource to be localized]]> + + + + + timestamp of the resource to be localized, used + for verification. + @return timestamp of the resource to be localized]]> + + + + + + timestamp of the resource to be localized, used + for verification. + @param timestamp timestamp of the resource to be localized]]> + + + + + LocalResourceType of the resource to be localized. + @return LocalResourceType of the resource to be localized]]> + + + + + + LocalResourceType of the resource to be localized. + @param type LocalResourceType of the resource to be localized]]> + + + + + LocalResourceVisibility of the resource to be + localized. + @return LocalResourceVisibility of the resource to be + localized]]> + + + + + + LocalResourceVisibility of the resource to be + localized. + @param visibility LocalResourceVisibility of the resource to be + localized]]> + + + + + pattern that should be used to extract entries from the + archive (only used when type is PATTERN). + @return pattern that should be used to extract entries from the + archive.]]> + + + + + + pattern that should be used to extract entries from the + archive (only used when type is PATTERN). + @param pattern pattern that should be used to extract entries + from the archive.]]> + + + + + + + + + + + shouldBeUploadedToSharedCache + of this request]]> + + + + LocalResource represents a local resource required to + run a container.

    + +

    The NodeManager is responsible for localizing the resource + prior to launching the container.

    + +

    Applications can specify {@link LocalResourceType} and + {@link LocalResourceVisibility}.

    + + @see LocalResourceType + @see LocalResourceVisibility + @see ContainerLaunchContext + @see ApplicationSubmissionContext + @see ContainerManagementProtocol#startContainers(org.apache.hadoop.yarn.api.protocolrecords.StartContainersRequest)]]> +     +     + + + + + + + + + + type + of a resource localized by the {@code NodeManager}. +

    + The type can be one of: +

      +
    • + {@link #FILE} - Regular file i.e. uninterpreted bytes. +
      • +
    • + {@link #ARCHIVE} - Archive, which is automatically unarchived by the + NodeManager. +
      • +
    • + {@link #PATTERN} - A hybrid between {@link #ARCHIVE} and {@link #FILE}. +
      • +
    + + @see LocalResource + @see ContainerLaunchContext + @see ApplicationSubmissionContext + @see ContainerManagementProtocol#startContainers(org.apache.hadoop.yarn.api.protocolrecords.StartContainersRequest)]]> +     +     + + + + + + + + + + visibility + of a resource localized by the {@code NodeManager}. +

    + The visibility can be one of: +

      +
    • {@link #PUBLIC} - Shared by all users on the node.
      • +
    • + {@link #PRIVATE} - Shared among all applications of the + same user on the node. +
      • +
    • + {@link #APPLICATION} - Shared only among containers of the + same application on the node. +
      • +
    + + @see LocalResource + @see ContainerLaunchContext + @see ApplicationSubmissionContext + @see ContainerManagementProtocol#startContainers(org.apache.hadoop.yarn.api.protocolrecords.StartContainersRequest)]]> +     +     + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + It includes details such as: +
      +
    • + includePattern. It uses Java Regex to filter the log files + which match the defined include pattern and those log files + will be uploaded when the application finishes. +
      • +
    • + excludePattern. It uses Java Regex to filter the log files + which match the defined exclude pattern and those log files + will not be uploaded when application finishes. If the log file + name matches both the include and the exclude pattern, this file + will be excluded eventually. +
      • +
    • + rolledLogsIncludePattern. It uses Java Regex to filter the log files + which match the defined include pattern and those log files + will be aggregated in a rolling fashion. +
      • +
    • + rolledLogsExcludePattern. It uses Java Regex to filter the log files + which match the defined exclude pattern and those log files + will not be aggregated in a rolling fashion. If the log file + name matches both the include and the exclude pattern, this file + will be excluded eventually. +
      • +
    • + policyClassName. The policy class name that implements + ContainerLogAggregationPolicy. At runtime, nodemanager will the policy + if a given container's log should be aggregated based on the + ContainerType and other runtime state such as exit code by calling + ContainerLogAggregationPolicy#shouldDoLogAggregation. + This is useful when the app only wants to aggregate logs of a subset of + containers. Here are the available policies. Please make sure to specify + the canonical name by prefixing org.apache.hadoop.yarn.server. + nodemanager.containermanager.logaggregation. + to the class simple name below. + NoneContainerLogAggregationPolicy: skip aggregation for all containers. + AllContainerLogAggregationPolicy: aggregate all containers. + AMOrFailedContainerLogAggregationPolicy: aggregate application master + or failed containers. + FailedOrKilledContainerLogAggregationPolicy: aggregate failed or killed + containers + FailedContainerLogAggregationPolicy: aggregate failed containers + AMOnlyLogAggregationPolicy: aggregate application master containers + SampleContainerLogAggregationPolicy: sample logs of successful worker + containers, in addition to application master and failed/killed + containers. + If it isn't specified, it will use the cluster-wide default policy + defined by configuration yarn.nodemanager.log-aggregation.policy.class. + The default value of yarn.nodemanager.log-aggregation.policy.class is + AllContainerLogAggregationPolicy. +
      • +
    • + policyParameters. The parameters passed to the policy class via + ContainerLogAggregationPolicy#parseParameters during the policy object + initialization. This is optional. Some policy class might use parameters + to adjust its settings. It is up to policy class to define the scheme of + parameters. + For example, SampleContainerLogAggregationPolicy supports the format of + "SR:0.5,MIN:50", which means sample rate of 50% beyond the first 50 + successful worker containers. +
      • +
    + + @see ApplicationSubmissionContext]]> +     +     + + + + + + + + NodeManager for which the NMToken + is used to authenticate. + @return the {@link NodeId} of the NodeManager for which the + NMToken is used to authenticate.]]> + + + + + + + + NodeManager + @return the {@link Token} used for authenticating with NodeManager]]> + + + + + + + + + + + + The NMToken is used for authenticating communication with + NodeManager

    +

    It is issued by ResourceMananger when ApplicationMaster + negotiates resource with ResourceManager and + validated on NodeManager side.

    + @see AllocateResponse#getNMTokens()]]> +     +     + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Node Attribute is a kind of a label which represents one of the + attribute/feature of a Node. Its different from node partition label as + resource guarantees across the queues will not be maintained for these type + of labels. +

    +

    + A given Node can be mapped with any kind of attribute, few examples are + HAS_SSD=true, JAVA_VERSION=JDK1.8, OS_TYPE=WINDOWS. +

    +

    + Its not compulsory for all the attributes to have value, empty string is the + default value of the NodeAttributeType.STRING +

    +

    + Node Attribute Prefix is used as namespace to segregate the attributes. +

    ]]> +     +     + + + + + + + + + + + + + + + + + + + + + + + + + Node Attribute Info describes a NodeAttribute. +

    ]]> +     +     + + + + + + + + + + + + + + + + + + + + + + + + + Node AttributeKey uniquely identifies a given Node Attribute. Node Attribute + is identified based on attribute prefix and name. +

    +

    + Node Attribute Prefix is used as namespace to segregate the attributes. +

    ]]> +     +     + + + + + + + + + + + + + + + + + + + + + + + Type of a node Attribute. +

    + Based on this attribute expressions and values will be evaluated.]]> +     +     + + + + + + + + + + + + + hostname of the node. + @return hostname of the node]]> + + + + + port for communicating with the node. + @return port for communicating with the node]]> + + + + + + + + + + + + + + + + + + + NodeId is the unique identifier for a node.

    + +

    It includes the hostname and port to uniquely + identify the node. Thus, it is unique across restarts of any + NodeManager.

    ]]> +     +     + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + NodeId of the node. + @return NodeId of the node]]> + + + + + NodeState of the node. + @return NodeState of the node]]> + + + + + http address of the node. + @return http address of the node]]> + + + + + rack name for the node. + @return rack name for the node]]> + + + + + used Resource on the node. + @return used Resource on the node]]> + + + + + total Resource on the node. + @return total Resource on the node]]> + + + + + diagnostic health report of the node. + @return diagnostic health report of the node]]> + + + + + last timestamp at which the health report was received. + @return last timestamp at which the health report was received]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + It includes details such as: +
      +
    • {@link NodeId} of the node.
      • +
    • HTTP Tracking URL of the node.
      • +
    • Rack name for the node.
      • +
    • Used {@link Resource} on the node.
      • +
    • Total available {@link Resource} of the node.
      • +
    • Number of running containers on the node.
      • +
    + + @see ApplicationClientProtocol#getClusterNodes(org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodesRequest)]]> +     +     + + + + + + + + + + + + + + + + State of a Node.

    ]]> +     +     + + + + + + + + + + + + + + + + + + + + + + Mapping of Attribute Value to a Node. +

    ]]> +     +     + + + + + + + + + + + + ResourceManager. + @see PreemptionContract + @see StrictPreemptionContract]]> + + + + + + + + + + ApplicationMaster about resources requested back by the + ResourceManager. + @see AllocateRequest#setAskList(List)]]> + + + + + ApplicationMaster that may be reclaimed by the + ResourceManager. If the AM prefers a different set of + containers, then it may checkpoint or kill containers matching the + description in {@link #getResourceRequest}. + @return Set of containers at risk if the contract is not met.]]> + + + + ResourceManager. + The ApplicationMaster (AM) can satisfy this request according + to its own priorities to prevent containers from being forcibly killed by + the platform. + @see PreemptionMessage]]> + + + + + + + + + + ResourceManager]]> + + + + + + + + + + The AM should decode both parts of the message. The {@link + StrictPreemptionContract} specifies particular allocations that the RM + requires back. The AM can checkpoint containers' state, adjust its execution + plan to move the computation, or take no action and hope that conditions that + caused the RM to ask for the container will change. +

    + In contrast, the {@link PreemptionContract} also includes a description of + resources with a set of containers. If the AM releases containers matching + that profile, then the containers enumerated in {@link + PreemptionContract#getContainers()} may not be killed. +

    + Each preemption message reflects the RM's current understanding of the + cluster state, so a request to return N containers may not + reflect containers the AM is releasing, recently exited containers the RM has + yet to learn about, or new containers allocated before the message was + generated. Conversely, an RM may request a different profile of containers in + subsequent requests. +

    + The policy enforced by the RM is part of the scheduler. Generally, only + containers that have been requested consistently should be killed, but the + details are not specified.]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + The ACL is one of: +

      +
    • + {@link #SUBMIT_APPLICATIONS} - ACL to submit applications to the queue. +
      • +
    • {@link #ADMINISTER_QUEUE} - ACL to administer the queue.
      • +
    + + @see QueueInfo + @see ApplicationClientProtocol#getQueueUserAcls(org.apache.hadoop.yarn.api.protocolrecords.GetQueueUserAclsInfoRequest)]]> +     +     + + + + + + + + name of the queue. + @return name of the queue]]> + + + + + configured capacity of the queue. + @return configured capacity of the queue]]> + + + + + maximum capacity of the queue. + @return maximum capacity of the queue]]> + + + + + current capacity of the queue. + @return current capacity of the queue]]> + + + + + child queues of the queue. + @return child queues of the queue]]> + + + + + running applications of the queue. + @return running applications of the queue]]> + + + + + QueueState of the queue. + @return QueueState of the queue]]> + + + + + accessible node labels of the queue. + @return accessible node labels of the queue]]> + + + + + default node label expression of the queue, this takes + affect only when the ApplicationSubmissionContext and + ResourceRequest don't specify their + NodeLabelExpression. + + @return default node label expression of the queue]]> + + + + + + + + queue stats for the queue + + @return queue stats of the queue]]> + + + + + + + + + + + preemption status of the queue. + @return if property is not in proto, return null; + otherwise, return preemption status of the queue]]> + + + + + + + + + + + + + + + It includes information such as: +
      +
    • Queue name.
      • +
    • Capacity of the queue.
      • +
    • Maximum capacity of the queue.
      • +
    • Current capacity of the queue.
      • +
    • Child queues.
      • +
    • Running applications.
      • +
    • {@link QueueState} of the queue.
      • +
    • {@link QueueConfigurations} of the queue.
      • +
    + + @see QueueState + @see QueueConfigurations + @see ApplicationClientProtocol#getQueueInfo(org.apache.hadoop.yarn.api.protocolrecords.GetQueueInfoRequest)]]> +     +     + + + + + + + + + + + A queue is in one of: +
      +
    • {@link #RUNNING} - normal state.
      • +
    • {@link #STOPPED} - not accepting new application submissions.
      • +
    • + {@link #DRAINING} - not accepting new application submissions + and waiting for applications finish. +
      • +
    + + @see QueueInfo + @see ApplicationClientProtocol#getQueueInfo(org.apache.hadoop.yarn.api.protocolrecords.GetQueueInfoRequest)]]> +     +     + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + queue name of the queue. + @return queue name of the queue]]> + + + + + QueueACL for the given user. + @return list of QueueACL for the given user]]> + + + + QueueUserACLInfo provides information {@link QueueACL} for + the given user.

    + + @see QueueACL + @see ApplicationClientProtocol#getQueueUserAcls(org.apache.hadoop.yarn.api.protocolrecords.GetQueueUserAclsInfoRequest)]]> +     +     + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + The ACL is one of: +
      +
    • + {@link #ADMINISTER_RESERVATIONS} - ACL to create, list, update and + delete reservations. +
      • +
    • {@link #LIST_RESERVATIONS} - ACL to list reservations.
      • +
    • {@link #SUBMIT_RESERVATIONS} - ACL to create reservations.
      • +
    + Users can always list, update and delete their own reservations.]]> +     +     + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + It includes: +
      +
    • Duration of the reservation.
      • +
    • Acceptance time of the duration.
      • +
    • + List of {@link ResourceAllocationRequest}, which includes the time + interval, and capability of the allocation. + {@code ResourceAllocationRequest} represents an allocation + made for a reservation for the current state of the queue. This can be + changed for reasons such as re-planning, but will always be subject to + the constraints of the user contract as described by + {@link ReservationDefinition} +
      • +
    • {@link ReservationId} of the reservation.
      • +
    • {@link ReservationDefinition} used to make the reservation.
      • +
    + + @see ResourceAllocationRequest + @see ReservationId + @see ReservationDefinition]]> +     +     + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + start time of the {@code ResourceManager} which is used to + generate globally unique {@link ReservationId}. + + @return start time of the {@code ResourceManager}]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + {@link ReservationId} represents the globally unique identifier for + a reservation. +

    + +

    + The globally unique nature of the identifier is achieved by using the + cluster timestamp i.e. start-time of the {@code ResourceManager} + along with a monotonically increasing counter for the reservation. +

    ]]> +     +     + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + It includes: +
      +
    • {@link Resource} required for each request.
      • +
    • + Number of containers, of above specifications, which are required by the + application. +
      • +
    • Concurrency that indicates the gang size of the request.
      • +
    ]]> +     +     + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + memory of the resource. Note - while memory has + never had a unit specified, all YARN configurations have specified memory + in MB. The assumption has been that the daemons and applications are always + using the same units. With the introduction of the ResourceInformation + class we have support for units - so this function will continue to return + memory but in the units of MB + + @return memory(in MB) of the resource]]> + + + + + memory of the resource. Note - while memory has + never had a unit specified, all YARN configurations have specified memory + in MB. The assumption has been that the daemons and applications are always + using the same units. With the introduction of the ResourceInformation + class we have support for units - so this function will continue to return + memory but in the units of MB + + @return memory of the resource]]> + + + + + + memory of the resource. Note - while memory has + never had a unit specified, all YARN configurations have specified memory + in MB. The assumption has been that the daemons and applications are always + using the same units. With the introduction of the ResourceInformation + class we have support for units - so this function will continue to set + memory but the assumption is that the value passed is in units of MB. + + @param memory memory(in MB) of the resource]]> + + + + + + memory of the resource. + @param memory memory of the resource]]> + + + + + number of virtual cpu cores of the resource. + + Virtual cores are a unit for expressing CPU parallelism. A node's capacity + should be configured with virtual cores equal to its number of physical + cores. A container should be requested with the number of cores it can + saturate, i.e. the average number of threads it expects to have runnable + at a time. + + @return num of virtual cpu cores of the resource]]> + + + + + + number of virtual cpu cores of the resource. + + Virtual cores are a unit for expressing CPU parallelism. A node's capacity + should be configured with virtual cores equal to its number of physical + cores. A container should be requested with the number of cores it can + saturate, i.e. the average number of threads it expects to have runnable + at a time. + + @param vCores number of virtual cpu cores of the resource]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Resource models a set of computer resources in the + cluster.

    + +

    Currently it models both memory and CPU.

    + +

    The unit for memory is megabytes. CPU is modeled with virtual cores + (vcores), a unit for expressing parallelism. A node's capacity should + be configured with virtual cores equal to its number of physical cores. A + container should be requested with the number of cores it can saturate, i.e. + the average number of threads it expects to have runnable at a time.

    + +

    Virtual cores take integer values and thus currently CPU-scheduling is + very coarse. A complementary axis for CPU requests that represents + processing power will likely be added in the future to enable finer-grained + resource configuration.

    + +

    Typically, applications request Resource of suitable + capability to run their component tasks.

    + + @see ResourceRequest + @see ApplicationMasterProtocol#allocate(org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest)]]> +     +     + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + It includes: +
      +
    • StartTime of the allocation.
      • +
    • EndTime of the allocation.
      • +
    • {@link Resource} reserved for the allocation.
      • +
    + + @see Resource]]> +     +     + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + blacklist of resources + for the application. + + @see ResourceRequest + @see ApplicationMasterProtocol#allocate(org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest)]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + host/rack string represents an arbitrary + host name. + + @param hostName host/rack on which the allocation is desired + @return whether the given host/rack string represents an arbitrary + host name]]> + + + + + Priority of the request. + @return Priority of the request]]> + + + + + + Priority of the request + @param priority Priority of the request]]> + + + + + host/rack) on which the allocation + is desired. + + A special value of * signifies that any resource + (host/rack) is acceptable. + + @return resource (e.g. host/rack) on which the allocation + is desired]]> + + + + + + host/rack) on which the allocation + is desired. + + A special value of * signifies that any resource name + (e.g. host/rack) is acceptable. + + @param resourceName (e.g. host/rack) on which the + allocation is desired]]> + + + + + + + + + + + + + + + + ResourceRequest. Defaults to true. + + @return whether locality relaxation is enabled with this + ResourceRequest.]]> + + + + + + ExecutionTypeRequest of the requested container. + + @param execSpec + ExecutionTypeRequest of the requested container]]> + + + + + ResourceRequest. Defaults to true. + + @return whether locality relaxation is enabled with this + ResourceRequest.]]> + + + + + + For a request at a network hierarchy level, set whether locality can be relaxed + to that level and beyond.

    + +

    If the flag is off on a rack-level ResourceRequest, + containers at that request's priority will not be assigned to nodes on that + request's rack unless requests specifically for those nodes have also been + submitted.

    + +

    If the flag is off on an {@link ResourceRequest#ANY}-level + ResourceRequest, containers at that request's priority will + only be assigned on racks for which specific requests have also been + submitted.

    + +

    For example, to request a container strictly on a specific node, the + corresponding rack-level and any-level requests should have locality + relaxation set to false. Similarly, to request a container strictly on a + specific rack, the corresponding any-level request should have locality + relaxation set to false.

    + + @param relaxLocality whether locality relaxation is enabled with this + ResourceRequest.]]> + + + + + + + + + + + + + + + + ID corresponding to this allocation request. This + ID is an identifier for different {@code ResourceRequest}s from the same + application. The allocated {@code Container}(s) received as part of the + {@code AllocateResponse} response will have the ID corresponding to the + original {@code ResourceRequest} for which the RM made the allocation. +

    + The scheduler may return multiple {@code AllocateResponse}s corresponding + to the same ID as and when scheduler allocates {@code Container}(s). + Applications can continue to completely ignore the returned ID in + the response and use the allocation for any of their outstanding requests. +

    + If one wishes to replace an entire {@code ResourceRequest} corresponding to + a specific ID, they can simply cancel the corresponding {@code + ResourceRequest} and submit a new one afresh. + + @return the ID corresponding to this allocation request.]]> + + + + + + ID corresponding to this allocation request. This + ID is an identifier for different {@code ResourceRequest}s from the same + application. The allocated {@code Container}(s) received as part of the + {@code AllocateResponse} response will have the ID corresponding to the + original {@code ResourceRequest} for which the RM made the allocation. +

    + The scheduler may return multiple {@code AllocateResponse}s corresponding + to the same ID as and when scheduler allocates {@code Container}(s). + Applications can continue to completely ignore the returned ID in + the response and use the allocation for any of their outstanding requests. +

    + If one wishes to replace an entire {@code ResourceRequest} corresponding to + a specific ID, they can simply cancel the corresponding {@code + ResourceRequest} and submit a new one afresh. +

    + If the ID is not set, scheduler will continue to work as previously and all + allocated {@code Container}(s) will have the default ID, -1. + + @param allocationRequestID the ID corresponding to this allocation + request.]]> + + + + + + Resource capability of the request. + @param capability Resource capability of the request]]> + + + + + Resource capability of the request. + @return Resource capability of the request]]> + + + + + + + + + + + + + + + + + + It includes: +

      +
    • {@link Priority} of the request.
      • +
    • + The name of the host or rack on which the allocation is + desired. A special value of * signifies that + any host/rack is acceptable to the application. +
      • +
    • {@link Resource} required for each request.
      • +
    • + Number of containers, of above specifications, which are required + by the application. +
      • +
    • + A boolean relaxLocality flag, defaulting to {@code true}, + which tells the {@code ResourceManager} if the application wants + locality to be loose (i.e. allows fall-through to rack or any) + or strict (i.e. specify hard constraint on resource allocation). +
      • +
    + + @see Resource + @see ApplicationMasterProtocol#allocate(org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest)]]> +     +     + + + + + + + priority of the request. + @see ResourceRequest#setPriority(Priority) + @param priority priority of the request + @return {@link ResourceRequestBuilder}]]> + + + + + + resourceName of the request. + @see ResourceRequest#setResourceName(String) + @param resourceName resourceName of the request + @return {@link ResourceRequestBuilder}]]> + + + + + + capability of the request. + @see ResourceRequest#setCapability(Resource) + @param capability capability of the request + @return {@link ResourceRequestBuilder}]]> + + + + + + numContainers of the request. + @see ResourceRequest#setNumContainers(int) + @param numContainers numContainers of the request + @return {@link ResourceRequestBuilder}]]> + + + + + + relaxLocality of the request. + @see ResourceRequest#setRelaxLocality(boolean) + @param relaxLocality relaxLocality of the request + @return {@link ResourceRequestBuilder}]]> + + + + + + nodeLabelExpression of the request. + @see ResourceRequest#setNodeLabelExpression(String) + @param nodeLabelExpression + nodeLabelExpression of the request + @return {@link ResourceRequestBuilder}]]> + + + + + + executionTypeRequest of the request. + @see ResourceRequest#setExecutionTypeRequest( + ExecutionTypeRequest) + @param executionTypeRequest + executionTypeRequest of the request + @return {@link ResourceRequestBuilder}]]> + + + + + + executionTypeRequest of the request with 'ensure + execution type' flag set to true. + @see ResourceRequest#setExecutionTypeRequest( + ExecutionTypeRequest) + @param executionType executionType of the request. + @return {@link ResourceRequestBuilder}]]> + + + + + + allocationRequestId of the request. + @see ResourceRequest#setAllocationRequestId(long) + @param allocationRequestId + allocationRequestId of the request + @return {@link ResourceRequestBuilder}]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + virtual memory. + + @return virtual memory in MB]]> + + + + + + virtual memory. + + @param vmem virtual memory in MB]]> + + + + + physical memory. + + @return physical memory in MB]]> + + + + + + physical memory. + + @param pmem physical memory in MB]]> + + + + + CPU utilization. + + @return CPU utilization normalized to 1 CPU]]> + + + + + + CPU utilization. + + @param cpu CPU utilization normalized to 1 CPU]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + ResourceUtilization models the utilization of a set of computer + resources in the cluster. +

    ]]> +     +     + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + allocationRequestId of the request. + + @see SchedulingRequest#setAllocationRequestId(long) + @param allocationRequestId allocationRequestId of the + request + @return {@link SchedulingRequest.SchedulingRequestBuilder}]]> + + + + + + priority of the request. + + @param priority priority of the request + @return {@link SchedulingRequest.SchedulingRequestBuilder} + @see SchedulingRequest#setPriority(Priority)]]> + + + + + + executionType of the request. + + @see SchedulingRequest#setExecutionType(ExecutionTypeRequest) + @param executionType executionType of the request + @return {@link SchedulingRequest.SchedulingRequestBuilder}]]> + + + + + + allocationTags of the request. + + @see SchedulingRequest#setAllocationTags(Set) + @param allocationTags allocationsTags of the request + @return {@link SchedulingRequest.SchedulingRequestBuilder}]]> + + + + + + executionType of the request. + + @see SchedulingRequest#setResourceSizing(ResourceSizing) + @param resourceSizing resourceSizing of the request + @return {@link SchedulingRequest.SchedulingRequestBuilder}]]> + + + + + + placementConstraintExpression of the request. + + @see SchedulingRequest#setPlacementConstraint( + PlacementConstraint) + @param placementConstraintExpression placementConstraints of + the request + @return {@link SchedulingRequest.SchedulingRequestBuilder}]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + ApplicationMaster that may be reclaimed by the + ResourceManager. + @return the set of {@link ContainerId} to be preempted.]]> + + + + ApplicationMaster (AM) + may attempt to checkpoint work or adjust its execution plan to accommodate + it. In contrast to {@link PreemptionContract}, the AM has no flexibility in + selecting which resources to return to the cluster. + @see PreemptionMessage]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Token is the security entity used by the framework + to verify authenticity of any resource.

    ]]> +     +     + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + ContainerId of the container. + @return ContainerId of the container]]> + + + + + + + + + + + ContainerUpdateType of the container. + @return ContainerUpdateType of the container.]]> + + + + + + ContainerUpdateType of the container. + @param updateType of the Container]]> + + + + + ContainerId of the container. + @return ContainerId of the container]]> + + + + + + ContainerId of the container. + @param containerId ContainerId of the container]]> + + + + + ExecutionType of the container. + @return ExecutionType of the container]]> + + + + + + ExecutionType of the container. + @param executionType ExecutionType of the container]]> + + + + + + Resource capability of the request. + @param capability Resource capability of the request]]> + + + + + Resource capability of the request. + @return Resource capability of the request]]> + + + + + + + + + + + + It includes: +
      +
    • version for the container.
      • +
    • {@link ContainerId} for the container.
      • +
    • + {@link Resource} capability of the container after the update request + is completed. +
      • +
    • + {@link ExecutionType} of the container after the update request is + completed. +
      • +
    + + Update rules: +
      +
    • + Currently only ONE aspect of the container can be updated per request + (user can either update Capability OR ExecutionType in one request.. + not both). +
      • +
    • + There must be only 1 update request per container in an allocate call. +
      • +
    • + If a new update request is sent for a container (in a subsequent allocate + call) before the first one is satisfied by the Scheduler, it will + overwrite the previous request. +
      • +
    + @see ApplicationMasterProtocol#allocate(org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest)]]> +     +     + + + + + + + + + + + + + + + ContainerUpdateType. + @return ContainerUpdateType]]> + + + + + + ContainerUpdateType. + @param updateType ContainerUpdateType]]> + + + + + Container. + @return Container]]> + + + + + + Container. + @param container Container]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + URL represents a serializable {@link java.net.URL}.

    ]]> +     +     + + + + + + + + + + RMAppAttempt.]]> + + + + + + + + + + + + ApplicationMaster.]]> + + + + + + + + + + NodeManagers in the cluster. + @return number of NodeManagers in the cluster]]> + + + + + DecommissionedNodeManagers in the cluster. + + @return number of DecommissionedNodeManagers in the cluster]]> + + + + + ActiveNodeManagers in the cluster. + + @return number of ActiveNodeManagers in the cluster]]> + + + + + LostNodeManagers in the cluster. + + @return number of LostNodeManagers in the cluster]]> + + + + + UnhealthyNodeManagers in the cluster. + + @return number of UnhealthyNodeManagers in the cluster]]> + + + + + RebootedNodeManagers in the cluster. + + @return number of RebootedNodeManagers in the cluster]]> + + + + YarnClusterMetrics represents cluster metrics.

    + +

    Currently only number of NodeManagers is provided.

    ]]> +     +     + +     + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + This class contains the information about a timeline domain, which is used + to a user to host a number of timeline entities, isolating them from others'. + The user can also define the reader and writer users/groups for the the + domain, which is used to control the access to its entities. +

    + +

    + The reader and writer users/groups pattern that the user can supply is the + same as what AccessControlList takes. +

    ]]> +     +     + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + The class that contains the the meta information of some conceptual entity + and its related events. The entity can be an application, an application + attempt, a container or whatever the user-defined object. +

    + +

    + Primary filters will be used to index the entities in + TimelineStore, such that users should carefully choose the + information they want to store as the primary filters. The remaining can be + stored as other information. +

    ]]> +     +     + + + + + + + + + + + + + ApplicationId of the + TimelineEntityGroupId. + + @return ApplicationId of the + TimelineEntityGroupId]]> + + + + + + + + timelineEntityGroupId. + + @return timelineEntityGroupId]]> + + + + + + + + + + + + + + + + + + + TimelineEntityGroupId is an abstract way for + timeline service users to represent #a group of related timeline data. + For example, all entities that represents one data flow DAG execution + can be grouped into one timeline entity group.

    ]]> +     +     + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +     + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + This class contains the information about a timeline service domain, which is + used to a user to host a number of timeline entities, isolating them from + others'. The user can also define the reader and writer users/groups for + the domain, which is used to control the access to its entities. +

    +

    + The reader and writer users/groups pattern that the user can supply is the + same as what AccessControlList takes. +

    ]]> +     +     + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + The constuctor is used to construct a proxy {@link TimelineEntity} or its + subclass object from the real entity object that carries information. +

    + +

    + It is usually used in the case where we want to recover class polymorphism + after deserializing the entity from its JSON form. +

    + @param entity the real entity that carries information]]> +     +     + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Note: Entities will be stored in the order of idPrefix specified. + If users decide to set idPrefix for an entity, they MUST provide + the same prefix for every update of this entity. +

    + Example: 
    + TimelineEntity entity = new TimelineEntity();
+ entity.setIdPrefix(value);
+
    + Users can use {@link TimelineServiceHelper#invertLong(long)} to invert + the prefix if necessary. + + @param entityIdPrefix prefix for an entity.]]> +     +     + + + + + + + +     + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +     + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + name property as a + InetSocketAddress. On an HA cluster, + this fetches the address corresponding to the RM identified by + {@link #RM_HA_ID}. + @param name property name. + @param defaultAddress the default value + @param defaultPort the default port + @return InetSocketAddress]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + yarn.resourcemanager.scheduler.class + cannot handle placement constraints, the corresponding SchedulingRequests + will be rejected. As of now, only the capacity scheduler supports + SchedulingRequests. In particular, it currently supports anti-affinity + constraints (no affinity or cardinality) and places one container at a + time. The advantage of this handler compared to the placement-processor is + that it follows the same ordering rules for queues (sorted by utilization, + priority) and apps (sorted by FIFO/fairness/priority) as the ones followed + by the main scheduler.]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + OPPORTUNISTIC containers on the NM.]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
  • default
    • +
  • docker
    • +
  • javasandbox
    • + ]]> +     +     + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Default platform-specific CLASSPATH for YARN applications. A + comma-separated list of CLASSPATH entries constructed based on the client + OS environment expansion syntax. +

    +

    + Note: Use {@link #DEFAULT_YARN_CROSS_PLATFORM_APPLICATION_CLASSPATH} for + cross-platform practice i.e. submit an application from a Windows client to + a Linux/Unix server or vice versa. +

    ]]> +     +     + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +     + +     + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + The information is passed along to applications via + {@link StartContainersResponse#getAllServicesMetaData()} that is returned by + {@link ContainerManagementProtocol#startContainers(StartContainersRequest)} +

    + + @return meta-data for this service that should be made available to + applications.]]> +     +     + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +     + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + The method used by the NodeManager log aggregation service + to initial the policy object with parameters specified by the application + or the cluster-wide setting. +

    + + @param parameters parameters with scheme defined by the policy class.]]> +     +     + + + + + The method used by the NodeManager log aggregation service + to ask the policy object if a given container's logs should be aggregated. +

    + + @param logContext ContainerLogContext + @return Whether or not the container's logs should be aggregated.]]> +     +     + + + +     + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + The method used by administrators to ask SCM to run cleaner task right away +

    + + @param request request SharedCacheManager to run a cleaner task + @return SharedCacheManager returns an empty response + on success and throws an exception on rejecting the request + @throws YarnException + @throws IOException]]> +     +     + + + The protocol between administrators and the SharedCacheManager +

    ]]> +     +     + +     + + + + + + + + + + + + + + + + + Tag1(N1),P1:Tag2(N2),P2:...:TagN(Nn),Pn

    + + where TagN(Nn) is a key value pair to determine the source + allocation tag and the number of allocations, such as: + +

    foo(3)

    + + Optional when using NodeAttribute Constraint. + + and where Pn can be any form of a valid constraint expression, + such as: + +
      +
    • in,node,foo,bar
      • +
    • notin,node,foo,bar,1,2
      • +
    • and(notin,node,foo:notin,node,bar)
      • +
    + + and NodeAttribute Constraint such as + +
      +
    • yarn.rm.io/foo=true
      • +
    • java=1.7,1.8
      • +
    + @param expression expression string. + @return a map of source tags to placement constraint mapping. + @throws PlacementConstraintParseException]]> +     +     + + + + + +     + +     + + + + diff --git a/hadoop-yarn-project/hadoop-yarn/dev-support/jdiff/Apache_Hadoop_YARN_API_3.3.4.xml b/hadoop-yarn-project/hadoop-yarn/dev-support/jdiff/Apache_Hadoop_YARN_API_3.3.4.xml new file mode 100644 index 0000000000000..f96c8d559ddf1 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/dev-support/jdiff/Apache_Hadoop_YARN_API_3.3.4.xml @@ -0,0 +1,26407 @@ + + + + + + + + + + + + + + + + + + + + The interface used by clients to obtain a new {@link ApplicationId} for + submitting new applications.

    + +

    The ResourceManager responds with a new, monotonically + increasing, {@link ApplicationId} which is used by the client to submit + a new application.

    + +

    The ResourceManager also responds with details such + as maximum resource capabilities in the cluster as specified in + {@link GetNewApplicationResponse}.

    + + @param request request to get a new ApplicationId + @return response containing the new ApplicationId to be used + to submit an application + @throws YarnException + @throws IOException + @see #submitApplication(SubmitApplicationRequest)]]> +     +     + + + + + + The interface used by clients to submit a new application to the + ResourceManager.

    + +

    The client is required to provide details such as queue, + {@link Resource} required to run the ApplicationMaster, + the equivalent of {@link ContainerLaunchContext} for launching + the ApplicationMaster etc. via the + {@link SubmitApplicationRequest}.

    + +

    Currently the ResourceManager sends an immediate (empty) + {@link SubmitApplicationResponse} on accepting the submission and throws + an exception if it rejects the submission. However, this call needs to be + followed by {@link #getApplicationReport(GetApplicationReportRequest)} + to make sure that the application gets properly submitted - obtaining a + {@link SubmitApplicationResponse} from ResourceManager doesn't guarantee + that RM 'remembers' this application beyond failover or restart. If RM + failover or RM restart happens before ResourceManager saves the + application's state successfully, the subsequent + {@link #getApplicationReport(GetApplicationReportRequest)} will throw + a {@link ApplicationNotFoundException}. The Clients need to re-submit + the application with the same {@link ApplicationSubmissionContext} when + it encounters the {@link ApplicationNotFoundException} on the + {@link #getApplicationReport(GetApplicationReportRequest)} call.

    + +

    During the submission process, it checks whether the application + already exists. If the application exists, it will simply return + SubmitApplicationResponse

    + +

    In secure mode,the ResourceManager verifies access to + queues etc. before accepting the application submission.

    + + @param request request to submit a new application + @return (empty) response on accepting the submission + @throws YarnException + @throws IOException + @see #getNewApplication(GetNewApplicationRequest)]]> +     +     + + + + + + The interface used by clients to request the + ResourceManager to fail an application attempt.

    + +

    The client, via {@link FailApplicationAttemptRequest} provides the + {@link ApplicationAttemptId} of the attempt to be failed.

    + +

    In secure mode,the ResourceManager verifies access to the + application, queue etc. before failing the attempt.

    + +

    Currently, the ResourceManager returns an empty response + on success and throws an exception on rejecting the request.

    + + @param request request to fail an attempt + @return ResourceManager returns an empty response + on success and throws an exception on rejecting the request + @throws YarnException + @throws IOException + @see #getQueueUserAcls(GetQueueUserAclsInfoRequest)]]> +     +     + + + + + + The interface used by clients to request the + ResourceManager to abort submitted application.

    + +

    The client, via {@link KillApplicationRequest} provides the + {@link ApplicationId} of the application to be aborted.

    + +

    In secure mode,the ResourceManager verifies access to the + application, queue etc. before terminating the application.

    + +

    Currently, the ResourceManager returns an empty response + on success and throws an exception on rejecting the request.

    + + @param request request to abort a submitted application + @return ResourceManager returns an empty response + on success and throws an exception on rejecting the request + @throws YarnException + @throws IOException + @see #getQueueUserAcls(GetQueueUserAclsInfoRequest)]]> +     +     + + + + + + The interface used by clients to get metrics about the cluster from + the ResourceManager.

    + +

    The ResourceManager responds with a + {@link GetClusterMetricsResponse} which includes the + {@link YarnClusterMetrics} with details such as number of current + nodes in the cluster.

    + + @param request request for cluster metrics + @return cluster metrics + @throws YarnException + @throws IOException]]> +     +     + + + + + + The interface used by clients to get a report of all nodes + in the cluster from the ResourceManager.

    + +

    The ResourceManager responds with a + {@link GetClusterNodesResponse} which includes the + {@link NodeReport} for all the nodes in the cluster.

    + + @param request request for report on all nodes + @return report on all nodes + @throws YarnException + @throws IOException]]> +     +     + + + + + + The interface used by clients to get information about queues + from the ResourceManager.

    + +

    The client, via {@link GetQueueInfoRequest}, can ask for details such + as used/total resources, child queues, running applications etc.

    + +

    In secure mode,the ResourceManager verifies access before + providing the information.

    + + @param request request to get queue information + @return queue information + @throws YarnException + @throws IOException]]> +     +     + + + + + + The interface used by clients to get information about queue + acls for current user from the ResourceManager. +

    + +

    The ResourceManager responds with queue acls for all + existing queues.

    + + @param request request to get queue acls for current user + @return queue acls for current user + @throws YarnException + @throws IOException]]> +     +     + + + + + + + + + + + + + + The interface used by clients to obtain a new {@link ReservationId} for + submitting new reservations.

    + +

    The ResourceManager responds with a new, unique, + {@link ReservationId} which is used by the client to submit + a new reservation.

    + + @param request to get a new ReservationId + @return response containing the new ReservationId to be used + to submit a new reservation + @throws YarnException if the reservation system is not enabled. + @throws IOException on IO failures. + @see #submitReservation(ReservationSubmissionRequest)]]> +     +     + + + + + + + The interface used by clients to submit a new reservation to the + {@code ResourceManager}. +

    + +

    + The client packages all details of its request in a + {@link ReservationSubmissionRequest} object. This contains information + about the amount of capacity, temporal constraints, and concurrency needs. + Furthermore, the reservation might be composed of multiple stages, with + ordering dependencies among them. +

    + +

    + In order to respond, a new admission control component in the + {@code ResourceManager} performs an analysis of the resources that have + been committed over the period of time the user is requesting, verify that + the user requests can be fulfilled, and that it respect a sharing policy + (e.g., {@code CapacityOverTimePolicy}). Once it has positively determined + that the ReservationSubmissionRequest is satisfiable the + {@code ResourceManager} answers with a + {@link ReservationSubmissionResponse} that include a non-null + {@link ReservationId}. Upon failure to find a valid allocation the response + is an exception with the reason. + + On application submission the client can use this {@link ReservationId} to + obtain access to the reserved resources. +

    + +

    + The system guarantees that during the time-range specified by the user, the + reservationID will be corresponding to a valid reservation. The amount of + capacity dedicated to such queue can vary overtime, depending of the + allocation that has been determined. But it is guaranteed to satisfy all + the constraint expressed by the user in the + {@link ReservationSubmissionRequest}. +

    + + @param request the request to submit a new Reservation + @return response the {@link ReservationId} on accepting the submission + @throws YarnException if the request is invalid or reservation cannot be + created successfully + @throws IOException]]> +     +     + + + + + + + The interface used by clients to update an existing Reservation. This is + referred to as a re-negotiation process, in which a user that has + previously submitted a Reservation. +

    + +

    + The allocation is attempted by virtually substituting all previous + allocations related to this Reservation with new ones, that satisfy the new + {@link ReservationUpdateRequest}. Upon success the previous allocation is + substituted by the new one, and on failure (i.e., if the system cannot find + a valid allocation for the updated request), the previous allocation + remains valid. + + The {@link ReservationId} is not changed, and applications currently + running within this reservation will automatically receive the resources + based on the new allocation. +

    + + @param request to update an existing Reservation (the ReservationRequest + should refer to an existing valid {@link ReservationId}) + @return response empty on successfully updating the existing reservation + @throws YarnException if the request is invalid or reservation cannot be + updated successfully + @throws IOException]]> +     +     + + + + + + + The interface used by clients to remove an existing Reservation. + + Upon deletion of a reservation applications running with this reservation, + are automatically downgraded to normal jobs running without any dedicated + reservation. +

    + + @param request to remove an existing Reservation (the ReservationRequest + should refer to an existing valid {@link ReservationId}) + @return response empty on successfully deleting the existing reservation + @throws YarnException if the request is invalid or reservation cannot be + deleted successfully + @throws IOException]]> +     +     + + + + + + + The interface used by clients to get the list of reservations in a plan. + The reservationId will be used to search for reservations to list if it is + provided. Otherwise, it will select active reservations within the + startTime and endTime (inclusive). +

    + + @param request to list reservations in a plan. Contains fields to select + String queue, ReservationId reservationId, long startTime, + long endTime, and a bool includeReservationAllocations. + + queue: Required. Cannot be null or empty. Refers to the + reservable queue in the scheduler that was selected when + creating a reservation submission + {@link ReservationSubmissionRequest}. + + reservationId: Optional. If provided, other fields will + be ignored. + + startTime: Optional. If provided, only reservations that + end after the startTime will be selected. This defaults + to 0 if an invalid number is used. + + endTime: Optional. If provided, only reservations that + start on or before endTime will be selected. This defaults + to Long.MAX_VALUE if an invalid number is used. + + includeReservationAllocations: Optional. Flag that + determines whether the entire reservation allocations are + to be returned. Reservation allocations are subject to + change in the event of re-planning as described by + {@code ReservationDefinition}. + + @return response that contains information about reservations that are + being searched for. + @throws YarnException if the request is invalid + @throws IOException on IO failures]]> +     +     + + + + + + + The interface used by client to get node to labels mappings in existing cluster +

    + + @param request + @return node to labels mappings + @throws YarnException + @throws IOException]]> +     +     + + + + + + + The interface used by client to get labels to nodes mappings + in existing cluster +

    + + @param request + @return labels to nodes mappings + @throws YarnException + @throws IOException]]> +     +     + + + + + + + The interface used by client to get node labels in the cluster +

    + + @param request to get node labels collection of this cluster + @return node labels collection of this cluster + @throws YarnException + @throws IOException]]> +     +     + + + + + + + The interface used by client to set priority of an application. +

    + @param request to set priority of an application + @return an empty response + @throws YarnException + @throws IOException]]> +     +     + + + + + + The interface used by clients to request the + ResourceManager to signal a container. For example, + the client can send command OUTPUT_THREAD_DUMP to dump threads of the + container.

    + +

    The client, via {@link SignalContainerRequest} provides the + id of the container and the signal command.

    + +

    In secure mode,the ResourceManager verifies access to the + application before signaling the container. + The user needs to have MODIFY_APP permission.

    + +

    Currently, the ResourceManager returns an empty response + on success and throws an exception on rejecting the request.

    + + @param request request to signal a container + @return ResourceManager returns an empty response + on success and throws an exception on rejecting the request + @throws YarnException + @throws IOException]]> +     +     + + + + + + + The interface used by client to set ApplicationTimeouts of an application. + The UpdateApplicationTimeoutsRequest should have timeout value with + absolute time with ISO8601 format yyyy-MM-dd'T'HH:mm:ss.SSSZ. +

    + Note: If application timeout value is less than or equal to current + time then update application throws YarnException. + @param request to set ApplicationTimeouts of an application + @return a response with updated timeouts. + @throws YarnException if update request has empty values or application is + in completing states. + @throws IOException on IO failures]]> +     +     + + + + + + + The interface used by clients to get all the resource profiles that are + available on the ResourceManager. +

    + @param request request to get all the resource profiles + @return Response containing a map of the profile name to Resource + capabilities + @throws YARNFeatureNotEnabledException if resource-profile is disabled + @throws YarnException if any error happens inside YARN + @throws IOException in case of other errors]]> +     +     + + + + + + + The interface to get the details for a specific resource profile. +

    + @param request request to get the details of a resource profile + @return Response containing the details for a particular resource profile + @throws YARNFeatureNotEnabledException if resource-profile is disabled + @throws YarnException if any error happens inside YARN + @throws IOException in case of other errors]]> +     +     + + + + + + + The interface to get the details for a specific resource profile. +

    + @param request request to get the details of a resource profile + @return Response containing the details for a particular resource profile + @throws YarnException if any error happens inside YARN + @throws IOException in case of other errors]]> +     +     + + + + + + + The interface used by client to get attributes to nodes mappings + available in ResourceManager. +

    + + @param request request to get details of attributes to nodes mapping. + @return Response containing the details of attributes to nodes mappings. + @throws YarnException if any error happens inside YARN + @throws IOException incase of other errors]]> +     +     + + + + + + + The interface used by client to get node attributes available in + ResourceManager. +

    + + @param request request to get node attributes collection of this cluster. + @return Response containing node attributes collection. + @throws YarnException if any error happens inside YARN. + @throws IOException incase of other errors.]]> +     +     + + + + + + + The interface used by client to get node to attributes mappings. + in existing cluster. +

    + + @param request request to get nodes to attributes mapping. + @return nodes to attributes mappings. + @throws YarnException if any error happens inside YARN. + @throws IOException]]> +     +     + + The protocol between clients and the ResourceManager + to submit/abort jobs and to get information on applications, cluster metrics, + nodes, queues and ACLs.

    ]]> +     +     + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + The protocol between clients and the ApplicationHistoryServer to + get the information of completed applications etc. +

    ]]> +     +     + + + + + + + + + + The interface used by a new ApplicationMaster to register with + the ResourceManager. +

    + +

    + The ApplicationMaster needs to provide details such as RPC + Port, HTTP tracking url etc. as specified in + {@link RegisterApplicationMasterRequest}. +

    + +

    + The ResourceManager responds with critical details such as + maximum resource capabilities in the cluster as specified in + {@link RegisterApplicationMasterResponse}. +

    + +

    + Re-register is only allowed for Unmanaged Application Master + (UAM) HA, with + {@link org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext#getKeepContainersAcrossApplicationAttempts()} + set to true. +

    + + @param request registration request + @return registration respose + @throws YarnException + @throws IOException + @throws InvalidApplicationMasterRequestException The exception is thrown + when an ApplicationMaster tries to register more then once. + @see RegisterApplicationMasterRequest + @see RegisterApplicationMasterResponse]]> +     +     + + + + + + The interface used by an ApplicationMaster to notify the + ResourceManager about its completion (success or failed).

    + +

    The ApplicationMaster has to provide details such as + final state, diagnostics (in case of failures) etc. as specified in + {@link FinishApplicationMasterRequest}.

    + +

    The ResourceManager responds with + {@link FinishApplicationMasterResponse}.

    + + @param request completion request + @return completion response + @throws YarnException + @throws IOException + @see FinishApplicationMasterRequest + @see FinishApplicationMasterResponse]]> +     +     + + + + + + + The main interface between an ApplicationMaster and the + ResourceManager. +

    + +

    + The ApplicationMaster uses this interface to provide a list of + {@link ResourceRequest} and returns unused {@link Container} allocated to + it via {@link AllocateRequest}. Optionally, the + ApplicationMaster can also blacklist resources which + it doesn't want to use. +

    + +

    + This also doubles up as a heartbeat to let the + ResourceManager know that the ApplicationMaster + is alive. Thus, applications should periodically make this call to be kept + alive. The frequency depends on + {@link YarnConfiguration#RM_AM_EXPIRY_INTERVAL_MS} which defaults to + {@link YarnConfiguration#DEFAULT_RM_AM_EXPIRY_INTERVAL_MS}. +

    + +

    + The ResourceManager responds with list of allocated + {@link Container}, status of completed containers and headroom information + for the application. +

    + +

    + The ApplicationMaster can use the available headroom + (resources) to decide how to utilized allocated resources and make informed + decisions about future resource requests. +

    + + @param request + allocation request + @return allocation response + @throws YarnException + @throws IOException + @throws InvalidApplicationMasterRequestException + This exception is thrown when an ApplicationMaster calls allocate + without registering first. + @throws InvalidResourceBlacklistRequestException + This exception is thrown when an application provides an invalid + specification for blacklist of resources. + @throws InvalidResourceRequestException + This exception is thrown when a {@link ResourceRequest} is out of + the range of the configured lower and upper limits on the + resources. + @see AllocateRequest + @see AllocateResponse]]> +     +     + + The protocol between a live instance of ApplicationMaster + and the ResourceManager.

    + +

    This is used by the ApplicationMaster to register/unregister + and to request and obtain resources in the cluster from the + ResourceManager.

    ]]> +     +     + + + + + + + + + + The interface used by clients to claim a resource with the + SharedCacheManager. The client uses a checksum to identify the + resource and an {@link ApplicationId} to identify which application will be + using the resource. +

    + +

    + The SharedCacheManager responds with whether or not the + resource exists in the cache. If the resource exists, a Path + to the resource in the shared cache is returned. If the resource does not + exist, the response is empty. +

    + + @param request request to claim a resource in the shared cache + @return response indicating if the resource is already in the cache + @throws YarnException + @throws IOException]]> +     +     + + + + + + + The interface used by clients to release a resource with the + SharedCacheManager. This method is called once an application + is no longer using a claimed resource in the shared cache. The client uses + a checksum to identify the resource and an {@link ApplicationId} to + identify which application is releasing the resource. +

    + +

    + Note: This method is an optimization and the client is not required to call + it for correctness. +

    + +

    + Currently the SharedCacheManager sends an empty response. +

    + + @param request request to release a resource in the shared cache + @return (empty) response on releasing the resource + @throws YarnException + @throws IOException]]> +     +     + + + The protocol between clients and the SharedCacheManager to claim + and release resources in the shared cache. +

    ]]> +     +     + + + + + + + + + + The ApplicationMaster provides a list of + {@link StartContainerRequest}s to a NodeManager to + start {@link Container}s allocated to it using this interface. +

    + +

    + The ApplicationMaster has to provide details such as allocated + resource capability, security tokens (if enabled), command to be executed + to start the container, environment for the process, necessary + binaries/jar/shared-objects etc. via the {@link ContainerLaunchContext} in + the {@link StartContainerRequest}. +

    + +

    + The NodeManager sends a response via + {@link StartContainersResponse} which includes a list of + {@link Container}s of successfully launched {@link Container}s, a + containerId-to-exception map for each failed {@link StartContainerRequest} in + which the exception indicates errors from per container and a + allServicesMetaData map between the names of auxiliary services and their + corresponding meta-data. Note: None-container-specific exceptions will + still be thrown by the API method itself. +

    +

    + The ApplicationMaster can use + {@link #getContainerStatuses(GetContainerStatusesRequest)} to get updated + statuses of the to-be-launched or launched containers. +

    + + @param request + request to start a list of containers + @return response including conatinerIds of all successfully launched + containers, a containerId-to-exception map for failed requests and + a allServicesMetaData map. + @throws YarnException + @throws IOException]]> +     +     + + + + + + + The ApplicationMaster requests a NodeManager to + stop a list of {@link Container}s allocated to it using this + interface. +

    + +

    + The ApplicationMaster sends a {@link StopContainersRequest} + which includes the {@link ContainerId}s of the containers to be stopped. +

    + +

    + The NodeManager sends a response via + {@link StopContainersResponse} which includes a list of {@link ContainerId} + s of successfully stopped containers, a containerId-to-exception map for + each failed request in which the exception indicates errors from per + container. Note: None-container-specific exceptions will still be thrown by + the API method itself. ApplicationMaster can use + {@link #getContainerStatuses(GetContainerStatusesRequest)} to get updated + statuses of the containers. +

    + + @param request + request to stop a list of containers + @return response which includes a list of containerIds of successfully + stopped containers, a containerId-to-exception map for failed + requests. + @throws YarnException + @throws IOException]]> +     +     + + + + + + + The API used by the ApplicationMaster to request for current + statuses of Containers from the NodeManager. +

    + +

    + The ApplicationMaster sends a + {@link GetContainerStatusesRequest} which includes the {@link ContainerId}s + of all containers whose statuses are needed. +

    + +

    + The NodeManager responds with + {@link GetContainerStatusesResponse} which includes a list of + {@link ContainerStatus} of the successfully queried containers and a + containerId-to-exception map for each failed request in which the exception + indicates errors from per container. Note: None-container-specific + exceptions will still be thrown by the API method itself. +

    + + @param request + request to get ContainerStatuses of containers with + the specified ContainerIds + @return response containing the list of ContainerStatus of the + successfully queried containers and a containerId-to-exception map + for failed requests. + + @throws YarnException + @throws IOException]]> +     +     + + + + + + + The API used by the ApplicationMaster to request for + resource increase of running containers on the NodeManager. +

    + + @param request + request to increase resource of a list of containers + @return response which includes a list of containerIds of containers + whose resource has been successfully increased and a + containerId-to-exception map for failed requests. + + @throws YarnException + @throws IOException]]> +     +     + + + + + + + The API used by the ApplicationMaster to request for + resource update of running containers on the NodeManager. +

    + + @param request + request to update resource of a list of containers + @return response which includes a list of containerIds of containers + whose resource has been successfully updated and a + containerId-to-exception map for failed requests. + + @throws YarnException Exception specific to YARN + @throws IOException IOException thrown from NodeManager]]> +     +     + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + The protocol between an ApplicationMaster and a + NodeManager to start/stop and increase resource of containers + and to get status of running containers.

    + +

    If security is enabled the NodeManager verifies that the + ApplicationMaster has truly been allocated the container + by the ResourceManager and also verifies all interactions such + as stopping the container or obtaining status information for the container. +

    ]]> +     +     + +     + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + response id used to track duplicate responses. + @return response id]]> + + + + + + response id used to track duplicate responses. + @param id response id]]> + + + + + current progress of application. + @return current progress of application]]> + + + + + + current progress of application + @param progress current progress of application]]> + + + + + ResourceRequest to update the + ResourceManager about the application's resource requirements. + @return the list of ResourceRequest + @see ResourceRequest]]> + + + + + + ResourceRequest to update the + ResourceManager about the application's resource requirements. + @param resourceRequests list of ResourceRequest to update the + ResourceManager about the application's + resource requirements + @see ResourceRequest]]> + + + + + ContainerId of containers being + released by the ApplicationMaster. + @return list of ContainerId of containers being + released by the ApplicationMaster]]> + + + + + + ContainerId of containers being + released by the ApplicationMaster + @param releaseContainers list of ContainerId of + containers being released by the + ApplicationMaster]]> + + + + + ResourceBlacklistRequest being sent by the + ApplicationMaster. + @return the ResourceBlacklistRequest being sent by the + ApplicationMaster + @see ResourceBlacklistRequest]]> + + + + + + ResourceBlacklistRequest to inform the + ResourceManager about the blacklist additions and removals + per the ApplicationMaster. + + @param resourceBlacklistRequest the ResourceBlacklistRequest + to inform the ResourceManager about + the blacklist additions and removals + per the ApplicationMaster + @see ResourceBlacklistRequest]]> + + + + + ApplicationMaster. + @return list of {@link UpdateContainerRequest} + being sent by the + ApplicationMaster.]]> + + + + + + ResourceManager about the containers that need to be + updated. + @param updateRequests list of UpdateContainerRequest for + containers to be updated]]> + + + + + ApplicationMaster. + @return list of {@link SchedulingRequest} being sent by the + ApplicationMaster.]]> + + + + + + ResourceManager about the application's resource requirements + (potentially including allocation tags and placement constraints). + @param schedulingRequests list of {@link SchedulingRequest} to update + the ResourceManager about the application's resource + requirements.]]> + + + + + + + + + + + + + + + + + The core request sent by the ApplicationMaster to the + ResourceManager to obtain resources in the cluster.

    + +

    The request includes: +

      +
    • A response id to track duplicate responses.
      • +
    • Progress information.
      • +
    • + A list of {@link ResourceRequest} to inform the + ResourceManager about the application's + resource requirements. +
      • +
    • + A list of unused {@link Container} which are being returned. +
      • +
    • + A list of {@link UpdateContainerRequest} to inform + the ResourceManager about the change in + requirements of running containers. +
      • +
    + + @see ApplicationMasterProtocol#allocate(AllocateRequest)]]> +     +     + + + + + + + responseId of the request. + @see AllocateRequest#setResponseId(int) + @param responseId responseId of the request + @return {@link AllocateRequestBuilder}]]> + + + + + + progress of the request. + @see AllocateRequest#setProgress(float) + @param progress progress of the request + @return {@link AllocateRequestBuilder}]]> + + + + + + askList of the request. + @see AllocateRequest#setAskList(List) + @param askList askList of the request + @return {@link AllocateRequestBuilder}]]> + + + + + + releaseList of the request. + @see AllocateRequest#setReleaseList(List) + @param releaseList releaseList of the request + @return {@link AllocateRequestBuilder}]]> + + + + + + resourceBlacklistRequest of the request. + @see AllocateRequest#setResourceBlacklistRequest( + ResourceBlacklistRequest) + @param resourceBlacklistRequest + resourceBlacklistRequest of the request + @return {@link AllocateRequestBuilder}]]> + + + + + + updateRequests of the request. + @see AllocateRequest#setUpdateRequests(List) + @param updateRequests updateRequests of the request + @return {@link AllocateRequestBuilder}]]> + + + + + + schedulingRequests of the request. + @see AllocateRequest#setSchedulingRequests(List) + @param schedulingRequests SchedulingRequest of the request + @return {@link AllocateRequestBuilder}]]> + + + + + + trackingUrl of the request. + @see AllocateRequest#setTrackingUrl(String) + @param trackingUrl new tracking url + @return {@link AllocateRequestBuilder}]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + ResourceManager needs the + ApplicationMaster to take some action then it will send an + AMCommand to the ApplicationMaster. See AMCommand + for details on commands and actions for them. + @return AMCommand if the ApplicationMaster should + take action, null otherwise + @see AMCommand]]> + + + + + last response id. + @return last response id]]> + + + + + newly allocated Container by the + ResourceManager. + @return list of newly allocated Container]]> + + + + + available headroom for resources in the cluster for the + application. + @return limit of available headroom for resources in the cluster for the + application]]> + + + + + completed containers' statuses. + @return the list of completed containers' statuses]]> + + + + + updated NodeReports. Updates could + be changes in health, availability etc of the nodes. + @return The delta of updated nodes since the last response]]> + + + + + + + + + + + The message is a snapshot of the resources the RM wants back from the AM. + While demand persists, the RM will repeat its request; applications should + not interpret each message as a request for additional + resources on top of previous messages. Resources requested consistently + over some duration may be forcibly killed by the RM. + + @return A specification of the resources to reclaim from this AM.]]> + + + + + + 1) AM is receiving first container on underlying NodeManager.
    + OR
    + 2) NMToken master key rolled over in ResourceManager and AM is getting new + container on the same underlying NodeManager. +

    + AM will receive one NMToken per NM irrespective of the number of containers + issued on same NM. AM is expected to store these tokens until issued a + new token for the same NM. + @return list of NMTokens required for communicating with NM]]> + + + + + ResourceManager. + @return list of newly increased containers]]> + + + + + + + + + + + + + + + + + + + + + + + + + + UpdateContainerError for + containers updates requests that were in error]]> + + + + + ResourceManager from previous application attempts which + have not been reported to the Application Master yet. +
    + These containers were recovered by the RM after the application master + had already registered. This may happen after RM restart when some NMs get + delayed in connecting to the RM and reporting the active containers. + Since they were not reported in the registration + response, they are reported in the response to the AM heartbeat. + + @return the list of running containers as viewed by + ResourceManager from previous application attempts.]]> +     +     + + + + + + + ResourceManager the + ApplicationMaster during resource negotiation. +

    + The response, includes: +

      +
    • Response ID to track duplicate responses.
      • +
    • + An AMCommand sent by ResourceManager to let the + {@code ApplicationMaster} take some actions (resync, shutdown etc.). +
      • +
    • A list of newly allocated {@link Container}.
      • +
    • A list of completed {@link Container}s' statuses.
      • +
    • + The available headroom for resources in the cluster for the + application. +
      • +
    • A list of nodes whose status has been updated.
      • +
    • The number of available nodes in a cluster.
      • +
    • A description of resources requested back by the cluster
      • +
    • AMRMToken, if AMRMToken has been rolled over
      • +
    • + A list of {@link Container} representing the containers + whose resource has been increased. +
      • +
    • + A list of {@link Container} representing the containers + whose resource has been decreased. +
      • +
    + + @see ApplicationMasterProtocol#allocate(AllocateRequest)]]> +     +     + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Note: {@link NMToken} will be used for authenticating communication with + {@code NodeManager}. + @return the list of container tokens to be used for authorization during + container resource update. + @see NMToken]]> + + + + + + AllocateResponse.getUpdatedContainers. + The token contains the container id and resource capability required for + container resource update. + @param containersToUpdate the list of container tokens to be used + for container resource increase.]]> + + + + The request sent by Application Master to the + Node Manager to change the resource quota of a container.

    + + @see ContainerManagementProtocol#updateContainer(ContainerUpdateRequest)]]> +     +     + + + + + + + + + + + + + + + + + + + + + + The response sent by the NodeManager to the + ApplicationMaster when asked to update container resource. +

    + + @see ContainerManagementProtocol#updateContainer(ContainerUpdateRequest)]]> +     +     + + + + + + + + + + + ApplicationAttemptId of the attempt to be failed. + @return ApplicationAttemptId of the attempt.]]> + + + + + + + The request sent by the client to the ResourceManager + to fail an application attempt.

    + +

    The request includes the {@link ApplicationAttemptId} of the attempt to + be failed.

    + + @see ApplicationClientProtocol#failApplicationAttempt(FailApplicationAttemptRequest)]]> +     +     + + + + + + + The response sent by the ResourceManager to the client + failing an application attempt.

    + +

    Currently it's empty.

    + + @see ApplicationClientProtocol#failApplicationAttempt(FailApplicationAttemptRequest)]]> +     +     + + + + + + + + + + + + + final state of the ApplicationMaster. + @return final state of the ApplicationMaster]]> + + + + + + final state of the ApplicationMaster + @param finalState final state of the ApplicationMaster]]> + + + + + diagnostic information on application failure. + @return diagnostic information on application failure]]> + + + + + + diagnostic information on application failure. + @param diagnostics diagnostic information on application failure]]> + + + + + tracking URL for the ApplicationMaster. + This url if contains scheme then that will be used by resource manager + web application proxy otherwise it will default to http. + @return tracking URLfor the ApplicationMaster]]> + + + + + + final tracking URLfor the ApplicationMaster. + This is the web-URL to which ResourceManager or web-application proxy will + redirect client/users once the application is finished and the + ApplicationMaster is gone. +

    + If the passed url has a scheme then that will be used by the + ResourceManager and web-application proxy, otherwise the scheme will + default to http. +

    +

    + Empty, null, "N/A" strings are all valid besides a real URL. In case an url + isn't explicitly passed, it defaults to "N/A" on the ResourceManager. +

    + + @param url + tracking URLfor the ApplicationMaster]]> + + + + + The final request includes details such: +

      +
    • Final state of the {@code ApplicationMaster}
      • +
    • + Diagnostic information in case of failure of the + {@code ApplicationMaster} +
      • +
    • Tracking URL
      • +
    + + @see ApplicationMasterProtocol#finishApplicationMaster(FinishApplicationMasterRequest)]]> +     +     + + + + + + + + + + + + ResourceManager to a + ApplicationMaster on it's completion. +

    + The response, includes: +

      +
    • A flag which indicates that the application has successfully unregistered + with the RM and the application can safely stop.
      • +
    +

    + Note: The flag indicates whether the application has successfully + unregistered and is safe to stop. The application may stop after the flag is + true. If the application stops before the flag is true then the RM may retry + the application. + + @see ApplicationMasterProtocol#finishApplicationMaster(FinishApplicationMasterRequest)]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + ApplicationAttemptId of an application attempt. + + @return ApplicationAttemptId of an application attempt]]> + + + + + + ApplicationAttemptId of an application attempt + + @param applicationAttemptId + ApplicationAttemptId of an application attempt]]> + + + + + The request sent by a client to the ResourceManager to get an + {@link ApplicationAttemptReport} for an application attempt. +

    + +

    + The request should include the {@link ApplicationAttemptId} of the + application attempt. +

    + + @see ApplicationAttemptReport + @see ApplicationHistoryProtocol#getApplicationAttemptReport(GetApplicationAttemptReportRequest)]]> +     +     + + + + + + + + + + + ApplicationAttemptReport for the application attempt. + + @return ApplicationAttemptReport for the application attempt]]> + + + + + + ApplicationAttemptReport for the application attempt. + + @param applicationAttemptReport + ApplicationAttemptReport for the application attempt]]> + + + + + The response sent by the ResourceManager to a client requesting + an application attempt report. +

    + +

    + The response includes an {@link ApplicationAttemptReport} which has the + details about the particular application attempt +

    + + @see ApplicationAttemptReport + @see ApplicationHistoryProtocol#getApplicationAttemptReport(GetApplicationAttemptReportRequest)]]> +     +     + + + + + + + + + + + ApplicationId of an application + + @return ApplicationId of an application]]> + + + + + + ApplicationId of an application + + @param applicationId + ApplicationId of an application]]> + + + + + The request from clients to get a list of application attempt reports of an + application from the ResourceManager. +

    + + @see ApplicationHistoryProtocol#getApplicationAttempts(GetApplicationAttemptsRequest)]]> +     +     + + + + + + + + + + + ApplicationReport of an application. + + @return a list of ApplicationReport of an application]]> + + + + + + ApplicationReport of an application. + + @param applicationAttempts + a list of ApplicationReport of an application]]> + + + + + The response sent by the ResourceManager to a client requesting + a list of {@link ApplicationAttemptReport} for application attempts. +

    + +

    + The ApplicationAttemptReport for each application includes the + details of an application attempt. +

    + + @see ApplicationAttemptReport + @see ApplicationHistoryProtocol#getApplicationAttempts(GetApplicationAttemptsRequest)]]> +     +     + + + + + + + + + + + ApplicationId of the application. + @return ApplicationId of the application]]> + + + + + + ApplicationId of the application + @param applicationId ApplicationId of the application]]> + + + + The request sent by a client to the ResourceManager to + get an {@link ApplicationReport} for an application.

    + +

    The request should include the {@link ApplicationId} of the + application.

    + + @see ApplicationClientProtocol#getApplicationReport(GetApplicationReportRequest) + @see ApplicationReport]]> +     +     + + + + + + + + ApplicationReport for the application. + @return ApplicationReport for the application]]> + + + + The response sent by the ResourceManager to a client + requesting an application report.

    + +

    The response includes an {@link ApplicationReport} which has details such + as user, queue, name, host on which the ApplicationMaster is + running, RPC port, tracking URL, diagnostics, start time etc.

    + + @see ApplicationClientProtocol#getApplicationReport(GetApplicationReportRequest)]]> +     +     + + + + + + + + + + + + + + + + + + + + The request from clients to get a report of Applications matching the + giving application types in the cluster from the + ResourceManager. +

    + + @see ApplicationClientProtocol#getApplications(GetApplicationsRequest) + +

    Setting any of the parameters to null, would just disable that + filter

    + + @param scope {@link ApplicationsRequestScope} to filter by + @param users list of users to filter by + @param queues list of scheduler queues to filter by + @param applicationTypes types of applications + @param applicationTags application tags to filter by + @param applicationStates application states to filter by + @param startRange range of application start times to filter by + @param finishRange range of application finish times to filter by + @param limit number of applications to limit to + @return {@link GetApplicationsRequest} to be used with + {@link ApplicationClientProtocol#getApplications(GetApplicationsRequest)}]]> +     +     + + + + + The request from clients to get a report of Applications matching the + giving application types in the cluster from the + ResourceManager. +

    + + @param scope {@link ApplicationsRequestScope} to filter by + @see ApplicationClientProtocol#getApplications(GetApplicationsRequest) + @return a report of Applications in {@link GetApplicationsRequest}]]> +     +     + + + + + The request from clients to get a report of Applications matching the + giving application types in the cluster from the + ResourceManager. +

    + + + @see ApplicationClientProtocol#getApplications(GetApplicationsRequest) + @return a report of Applications in {@link GetApplicationsRequest}]]> +     +     + + + + + The request from clients to get a report of Applications matching the + giving application states in the cluster from the + ResourceManager. +

    + + + @see ApplicationClientProtocol#getApplications(GetApplicationsRequest) + @return a report of Applications in {@link GetApplicationsRequest}]]> +     +     + + + + + + The request from clients to get a report of Applications matching the + giving and application types and application types in the cluster from the + ResourceManager. +

    + + + @see ApplicationClientProtocol#getApplications(GetApplicationsRequest) + @return a report of Applications in GetApplicationsRequest]]> +     +     + + + + + + + + + + + + The request from clients to get a report of Applications + in the cluster from the ResourceManager.

    + + @see ApplicationClientProtocol#getApplications(GetApplicationsRequest)]]> +     +     + + + + + + + + ApplicationReport for applications. + @return ApplicationReport for applications]]> + + + + The response sent by the ResourceManager to a client + requesting an {@link ApplicationReport} for applications.

    + +

    The ApplicationReport for each application includes details + such as user, queue, name, host on which the ApplicationMaster + is running, RPC port, tracking URL, diagnostics, start time etc.

    + + @see ApplicationReport + @see ApplicationClientProtocol#getApplications(GetApplicationsRequest)]]> +     +     + + + + + + + + + + + + + + + + + + + + + + + + The request from clients to get node to attribute value mapping for all or + given set of Node AttributeKey's in the cluster from the + ResourceManager. +

    + + @see ApplicationClientProtocol#getAttributesToNodes + (GetAttributesToNodesRequest)]]> +     +     + + + + + + + + + + + + + + + + + + + The response sent by the ResourceManager to a client requesting + node to attribute value mapping for all or given set of Node AttributeKey's. +

    + + @see ApplicationClientProtocol#getAttributesToNodes + (GetAttributesToNodesRequest)]]> +     +     + + + + + + + + + The request sent by clients to get cluster metrics from the + ResourceManager.

    + +

    Currently, this is empty.

    + + @see ApplicationClientProtocol#getClusterMetrics(GetClusterMetricsRequest)]]> +     +     + + + + + + + + YarnClusterMetrics for the cluster. + @return YarnClusterMetrics for the cluster]]> + + + + ResourceManager to a client + requesting cluster metrics. + + @see YarnClusterMetrics + @see ApplicationClientProtocol#getClusterMetrics(GetClusterMetricsRequest)]]> + + + + + + + + + + + + + + + The request from clients to get node attributes in the cluster from the + ResourceManager. +

    + + @see ApplicationClientProtocol#getClusterNodeAttributes + (GetClusterNodeAttributesRequest)]]> +     +     + + + + + + + + + + + + + + + + + + + + + + + + + The response sent by the ResourceManager to a client requesting + a node attributes in cluster. +

    + + @see ApplicationClientProtocol#getClusterNodeAttributes + (GetClusterNodeAttributesRequest)]]> +     +     + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + The request from clients to get a report of all nodes + in the cluster from the ResourceManager.

    + + The request will ask for all nodes in the given {@link NodeState}s. + + @see ApplicationClientProtocol#getClusterNodes(GetClusterNodesRequest)]]> +     +     + + + + + + + + NodeReport for all nodes in the cluster. + @return NodeReport for all nodes in the cluster]]> + + + + The response sent by the ResourceManager to a client + requesting a {@link NodeReport} for all nodes.

    + +

    The NodeReport contains per-node information such as + available resources, number of containers, tracking url, rack name, health + status etc. + + @see NodeReport + @see ApplicationClientProtocol#getClusterNodes(GetClusterNodesRequest)]]> + + + + + + + + + + + + + ContainerId of the Container. + + @return ContainerId of the Container]]> + + + + + + ContainerId of the container + + @param containerId + ContainerId of the container]]> + + + + + The request sent by a client to the ResourceManager to get an + {@link ContainerReport} for a container. +

    ]]> +     +     + + + + + + + + + + + ContainerReport for the container. + + @return ContainerReport for the container]]> + + + + + + + + The response sent by the ResourceManager to a client requesting + a container report. +

    + +

    + The response includes a {@link ContainerReport} which has details of a + container. +

    ]]> +     +     + + + + + + + + + + + ApplicationAttemptId of an application attempt. + + @return ApplicationAttemptId of an application attempt]]> + + + + + + ApplicationAttemptId of an application attempt + + @param applicationAttemptId + ApplicationAttemptId of an application attempt]]> + + + + + The request from clients to get a list of container reports, which belong to + an application attempt from the ResourceManager. +

    + + @see ApplicationHistoryProtocol#getContainers(GetContainersRequest)]]> +     +     + + + + + + + + + + + ContainerReport for all the containers of an + application attempt. + + @return a list of ContainerReport for all the containers of an + application attempt]]> + + + + + + ContainerReport for all the containers of an + application attempt. + + @param containers + a list of ContainerReport for all the containers of + an application attempt]]> + + + + + The response sent by the ResourceManager to a client requesting + a list of {@link ContainerReport} for containers. +

    + +

    + The ContainerReport for each container includes the container + details. +

    + + @see ContainerReport + @see ApplicationHistoryProtocol#getContainers(GetContainersRequest)]]> +     +     + + + + + + + + + + + ContainerIds of containers for which to obtain + the ContainerStatus. + + @return the list of ContainerIds of containers for which to + obtain the ContainerStatus.]]> + + + + + + ContainerIds of containers for which to obtain + the ContainerStatus + + @param containerIds + a list of ContainerIds of containers for which to + obtain the ContainerStatus]]> + + + + ApplicationMaster to the + NodeManager to get {@link ContainerStatus} of requested + containers. + + @see ContainerManagementProtocol#getContainerStatuses(GetContainerStatusesRequest)]]> + + + + + + + + + + ContainerStatuses of the requested containers. + + @return ContainerStatuses of the requested containers.]]> + + + + + + + + + NodeManager to the + ApplicationMaster when asked to obtain the + ContainerStatus of requested containers. + + @see ContainerManagementProtocol#getContainerStatuses(GetContainerStatusesRequest)]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + The request sent by clients to get a new {@link ApplicationId} for + submitting an application.

    + +

    Currently, this is empty.

    + + @see ApplicationClientProtocol#getNewApplication(GetNewApplicationRequest)]]> +     +     + + + + + + + + new ApplicationId allocated by the + ResourceManager. + @return new ApplicationId allocated by the + ResourceManager]]> + + + + + ResourceManager in the cluster. + @return maximum capability of allocated resources in the cluster]]> + + + + The response sent by the ResourceManager to the client for + a request to get a new {@link ApplicationId} for submitting applications.

    + +

    Clients can submit an application with the returned + {@link ApplicationId}.

    + + @see ApplicationClientProtocol#getNewApplication(GetNewApplicationRequest)]]> +     +     + + + + + + + + + The request sent by clients to get a new {@code ReservationId} for + submitting an reservation.

    + + {@code ApplicationClientProtocol#getNewReservation(GetNewReservationRequest)}]]> +     +     + + + + + + + + + + + + The response sent by the ResourceManager to the client for + a request to get a new {@link ReservationId} for submitting reservations.

    + +

    Clients can submit an reservation with the returned + {@link ReservationId}.

    + + {@code ApplicationClientProtocol#getNewReservation(GetNewReservationRequest)}]]> +     +     + + + + + + + + + + + + + + + + + + + + + + The request from clients to get nodes to attributes mapping + in the cluster from the ResourceManager. +

    + + @see ApplicationClientProtocol#getNodesToAttributes + (GetNodesToAttributesRequest)]]> +     +     + + + + + + + + + + + + + + + + + + + The response sent by the ResourceManager to a client requesting + nodes to attributes mapping. +

    + + @see ApplicationClientProtocol#getNodesToAttributes + (GetNodesToAttributesRequest)]]> +     +     + + + + + + + + + + + + + + queue name for which to get queue information. + @return queue name for which to get queue information]]> + + + + + + queue name for which to get queue information + @param queueName queue name for which to get queue information]]> + + + + + active applications required? + @return true if applications' information is to be included, + else false]]> + + + + + + active applications? + @param includeApplications fetch information about active + applications?]]> + + + + + child queues required? + @return true if information about child queues is required, + else false]]> + + + + + + child queues? + @param includeChildQueues fetch information about child queues?]]> + + + + + child queue hierarchy required? + @return true if information about entire hierarchy is + required, false otherwise]]> + + + + + + child queue hierarchy? + @param recursive fetch information on the entire child queue + hierarchy?]]> + + + + The request sent by clients to get queue information + from the ResourceManager.

    + + @see ApplicationClientProtocol#getQueueInfo(GetQueueInfoRequest)]]> +     +     + + + + + + + + QueueInfo for the specified queue. + @return QueueInfo for the specified queue]]> + + + + + The response includes a {@link QueueInfo} which has details such as + queue name, used/total capacities, running applications, child queues etc. + + @see QueueInfo + @see ApplicationClientProtocol#getQueueInfo(GetQueueInfoRequest)]]> + + + + + + + + + + + The request sent by clients to the ResourceManager to + get queue acls for the current user.

    + +

    Currently, this is empty.

    + + @see ApplicationClientProtocol#getQueueUserAcls(GetQueueUserAclsInfoRequest)]]> +     +     + + + + + + + + QueueUserACLInfo per queue for the user. + @return QueueUserACLInfo per queue for the user]]> + + + + The response sent by the ResourceManager to clients + seeking queue acls for the user.

    + +

    The response contains a list of {@link QueueUserACLInfo} which + provides information about {@link QueueACL} per queue.

    + + @see QueueACL + @see QueueUserACLInfo + @see ApplicationClientProtocol#getQueueUserAcls(GetQueueUserAclsInfoRequest)]]> +     +     + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Note: {@link NMToken} will be used for authenticating communication with + {@code NodeManager}. + @return the list of container tokens to be used for authorization during + container resource increase. + @see NMToken]]> + + + + + + AllocateResponse.getIncreasedContainers. + The token contains the container id and resource capability required for + container resource increase. + @param containersToIncrease the list of container tokens to be used + for container resource increase.]]> + + + + The request sent by Application Master to the + Node Manager to change the resource quota of a container.

    + + @see ContainerManagementProtocol#increaseContainersResource(IncreaseContainersResourceRequest)]]> +     +     + + + + + + + + + + + + + + + + + + The response sent by the NodeManager to the + ApplicationMaster when asked to increase container resource. +

    + + @see ContainerManagementProtocol#increaseContainersResource(IncreaseContainersResourceRequest)]]> +     +     + + + + + + + + + + + ApplicationId of the application to be aborted. + @return ApplicationId of the application to be aborted]]> + + + + + + + + diagnostics to which the application is being killed. + @return diagnostics to which the application is being killed]]> + + + + + + diagnostics to which the application is being killed. + @param diagnostics diagnostics to which the application is being + killed]]> + + + + The request sent by the client to the ResourceManager + to abort a submitted application.

    + +

    The request includes the {@link ApplicationId} of the application to be + aborted.

    + + @see ApplicationClientProtocol#forceKillApplication(KillApplicationRequest)]]> +     +     + + + + + + + + + + + + ResourceManager to the client aborting + a submitted application. +

    + The response, includes: +

      +
    • + A flag which indicates that the process of killing the application is + completed or not. +
      • +
    + Note: user is recommended to wait until this flag becomes true, otherwise if + the ResourceManager crashes before the process of killing the + application is completed, the ResourceManager may retry this + application on recovery. + + @see ApplicationClientProtocol#forceKillApplication(KillApplicationRequest)]]> +     +     + + + + + + + + + + + + ApplicationId of the application to be moved. + @return ApplicationId of the application to be moved]]> + + + + + + ApplicationId of the application to be moved. + @param appId ApplicationId of the application to be moved]]> + + + + + + + + + + + + + + + The request sent by the client to the ResourceManager + to move a submitted application to a different queue.

    + +

    The request includes the {@link ApplicationId} of the application to be + moved and the queue to place it in.

    + + @see ApplicationClientProtocol#moveApplicationAcrossQueues(MoveApplicationAcrossQueuesRequest)]]> +     +     + + + + + + + + The response sent by the ResourceManager to the client moving + a submitted application to a different queue. +

    +

    + A response without exception means that the move has completed successfully. +

    + + @see ApplicationClientProtocol#moveApplicationAcrossQueues(MoveApplicationAcrossQueuesRequest)]]> +     +     + + + + + + + + + + + RegisterApplicationMasterRequest. + If port, trackingUrl is not used, use the following default value: +
      +
    • port: -1
      • +
    • trackingUrl: null
      • +
    + The port is allowed to be any integer larger than or equal to -1. + @return the new instance of RegisterApplicationMasterRequest]]> +     +     + + + host on which the ApplicationMaster is + running. + @return host on which the ApplicationMaster is running]]> + + + + + + host on which the ApplicationMaster is + running. + @param host host on which the ApplicationMaster + is running]]> + + + + + RPC port on which the {@code ApplicationMaster} is + responding. + @return the RPC port on which the {@code ApplicationMaster} + is responding]]> + + + + + + RPC port on which the {@code ApplicationMaster} is + responding. + @param port RPC port on which the {@code ApplicationMaster} + is responding]]> + + + + + tracking URL for the ApplicationMaster. + This url if contains scheme then that will be used by resource manager + web application proxy otherwise it will default to http. + @return tracking URL for the ApplicationMaster]]> + + + + + + tracking URLfor the ApplicationMaster while + it is running. This is the web-URL to which ResourceManager or + web-application proxy will redirect client/users while the application and + the ApplicationMaster are still running. +

    + If the passed url has a scheme then that will be used by the + ResourceManager and web-application proxy, otherwise the scheme will + default to http. +

    +

    + Empty, null, "N/A" strings are all valid besides a real URL. In case an url + isn't explicitly passed, it defaults to "N/A" on the ResourceManager. +

    + + @param trackingUrl + tracking URLfor the ApplicationMaster]]> + + + + + PlacementConstraint associated with the tags, i.e., each + {@link org.apache.hadoop.yarn.api.records.SchedulingRequest} that has those + tags will be placed taking into account the corresponding constraint. + + @return A map of Placement Constraints.]]> + + + + + + PlacementConstraint associated with the tags. + For example: + Map < + <hb_regionserver> -> node_anti_affinity, + <hb_regionserver, hb_master> -> rack_affinity, + ... + > + @param placementConstraints Placement Constraint Mapping.]]> + + + + + The registration includes details such as: +

      +
    • Hostname on which the AM is running.
      • +
    • RPC Port
      • +
    • Tracking URL
      • +
    + + @see ApplicationMasterProtocol#registerApplicationMaster(RegisterApplicationMasterRequest)]]> +     +     + + + + + + + + ResourceManager in the cluster. + @return maximum capability of allocated resources in the cluster]]> + + + + + ApplicationACLs for the application. + @return all the ApplicationACLs]]> + + + + + Get ClientToAMToken master key.

    +

    The ClientToAMToken master key is sent to ApplicationMaster + by ResourceManager via {@link RegisterApplicationMasterResponse} + , used to verify corresponding ClientToAMToken.

    + @return ClientToAMToken master key]]> +     +     + + + + + + + + + Get the queue that the application was placed in.

    + @return the queue that the application was placed in.]]> + + + + + + Set the queue that the application was placed in.

    ]]> + + + + + + Get the list of running containers as viewed by + ResourceManager from previous application attempts. +

    + + @return the list of running containers as viewed by + ResourceManager from previous application attempts + @see RegisterApplicationMasterResponse#getNMTokensFromPreviousAttempts()]]> +     +     + + + + + + + + + + + + + + + + + + + + + + + The response contains critical details such as: +
      +
    • Maximum capability for allocated resources in the cluster.
      • +
    • {@code ApplicationACL}s for the application.
      • +
    • ClientToAMToken master key.
      • +
    + + @see ApplicationMasterProtocol#registerApplicationMaster(RegisterApplicationMasterRequest)]]> +     +     + + + + + + + + + + + + + + + + ContainerId of the container to re-initialize. + + @return ContainerId of the container to re-initialize.]]> + + + + + ContainerLaunchContext to re-initialize the container + with. + + @return ContainerLaunchContext of to re-initialize the + container with.]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + ApplicationId of the resource to be released. + + @return ApplicationId]]> + + + + + + ApplicationId of the resource to be released. + + @param id ApplicationId]]> + + + + + key of the resource to be released. + + @return key]]> + + + + + + key of the resource to be released. + + @param key unique identifier for the resource]]> + + + + The request from clients to release a resource in the shared cache.

    ]]> +     +     + + + + + + + + The response to clients from the SharedCacheManager when + releasing a resource in the shared cache. +

    + +

    + Currently, this is empty. +

    ]]> +     +     + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + The response sent by the ResourceManager to a client on + reservation submission.

    + +

    Currently, this is empty.

    + + {@code ApplicationClientProtocol#submitReservation( + ReservationSubmissionRequest)}]]> +     +     + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + ContainerId of the container to localize resources. + + @return ContainerId of the container to localize resources.]]> + + + + + LocalResource required by the container. + + @return all LocalResource required by the container]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + ContainerId of the container to signal. + @return ContainerId of the container to signal.]]> + + + + + + ContainerId of the container to signal.]]> + + + + + SignalContainerCommand of the signal request. + @return SignalContainerCommand of the signal request.]]> + + + + + + SignalContainerCommand of the signal request.]]> + + + + The request sent by the client to the ResourceManager + or by the ApplicationMaster to the NodeManager + to signal a container. + @see SignalContainerCommand

    ]]> +     +     + + + + + + + The response sent by the ResourceManager to the client + signalling a container.

    + +

    Currently it's empty.

    + + @see ApplicationClientProtocol#signalToContainer(SignalContainerRequest)]]> +     +     + + + + + + + + + + + + ContainerLaunchContext for the container to be started + by the NodeManager. + + @return ContainerLaunchContext for the container to be started + by the NodeManager]]> + + + + + + ContainerLaunchContext for the container to be started + by the NodeManager + @param context ContainerLaunchContext for the container to be + started by the NodeManager]]> + + + + + + Note: {@link NMToken} will be used for authenticating communication with + {@code NodeManager}. + @return the container token to be used for authorization during starting + container. + @see NMToken + @see ContainerManagementProtocol#startContainers(StartContainersRequest)]]> + + + + + + + The request sent by the ApplicationMaster to the + NodeManager to start a container.

    + +

    The ApplicationMaster has to provide details such as + allocated resource capability, security tokens (if enabled), command + to be executed to start the container, environment for the process, + necessary binaries/jar/shared-objects etc. via the + {@link ContainerLaunchContext}.

    + + @see ContainerManagementProtocol#startContainers(StartContainersRequest)]]> +     +     + + + + + + + + + + + + + + + + + + + + + + The request which contains a list of {@link StartContainerRequest} sent by + the ApplicationMaster to the NodeManager to + start containers. +

    + +

    + In each {@link StartContainerRequest}, the ApplicationMaster has + to provide details such as allocated resource capability, security tokens (if + enabled), command to be executed to start the container, environment for the + process, necessary binaries/jar/shared-objects etc. via the + {@link ContainerLaunchContext}. +

    + + @see ContainerManagementProtocol#startContainers(StartContainersRequest)]]> +     +     + + + + + + + + ContainerId s of the containers that are + started successfully. + + @return the list of ContainerId s of the containers that are + started successfully. + @see ContainerManagementProtocol#startContainers(StartContainersRequest)]]> + + + + + + + + + + + Get the meta-data from all auxiliary services running on the + NodeManager. +

    +

    + The meta-data is returned as a Map between the auxiliary service names and + their corresponding per service meta-data as an opaque blob + ByteBuffer +

    + +

    + To be able to interpret the per-service meta-data, you should consult the + documentation for the Auxiliary-service configured on the NodeManager +

    + + @return a Map between the names of auxiliary services and their + corresponding meta-data]]> +     +     + + + The response sent by the NodeManager to the + ApplicationMaster when asked to start an allocated + container. +

    + + @see ContainerManagementProtocol#startContainers(StartContainersRequest)]]> +     +     + + + + + + + + + + + ContainerIds of the containers to be stopped. + @return ContainerIds of containers to be stopped]]> + + + + + + ContainerIds of the containers to be stopped. + @param containerIds ContainerIds of the containers to be stopped]]> + + + + The request sent by the ApplicationMaster to the + NodeManager to stop containers.

    + + @see ContainerManagementProtocol#stopContainers(StopContainersRequest)]]> +     +     + + + + + + + + + + + + + + + + + + The response sent by the NodeManager to the + ApplicationMaster when asked to stop allocated + containers. +

    + + @see ContainerManagementProtocol#stopContainers(StopContainersRequest)]]> +     +     + + + + + + + + + + + ApplicationSubmissionContext for the application. + @return ApplicationSubmissionContext for the application]]> + + + + + + ApplicationSubmissionContext for the application. + @param context ApplicationSubmissionContext for the + application]]> + + + + The request sent by a client to submit an application to the + ResourceManager.

    + +

    The request, via {@link ApplicationSubmissionContext}, contains + details such as queue, {@link Resource} required to run the + ApplicationMaster, the equivalent of + {@link ContainerLaunchContext} for launching the + ApplicationMaster etc. + + @see ApplicationClientProtocol#submitApplication(SubmitApplicationRequest)]]> + + + + + + + + + The response sent by the ResourceManager to a client on + application submission.

    + +

    Currently, this is empty.

    + + @see ApplicationClientProtocol#submitApplication(SubmitApplicationRequest)]]> +     +     + + + + + + + + + + + + ApplicationId of the application. + + @return ApplicationId of the application]]> + + + + + + ApplicationId of the application. + + @param applicationId ApplicationId of the application]]> + + + + + Priority of the application to be set. + + @return Priority of the application to be set.]]> + + + + + + Priority of the application. + + @param priority Priority of the application]]> + + + + + The request sent by the client to the ResourceManager to set or + update the application priority. +

    +

    + The request includes the {@link ApplicationId} of the application and + {@link Priority} to be set for an application +

    + + @see ApplicationClientProtocol#updateApplicationPriority(UpdateApplicationPriorityRequest)]]> +     +     + + + + + + + + + + + Priority of the application to be set. + @return Updated Priority of the application.]]> + + + + + + Priority of the application. + + @param priority Priority of the application]]> + + + + + The response sent by the ResourceManager to the client on update + the application priority. +

    +

    + A response without exception means that the move has completed successfully. +

    + + @see ApplicationClientProtocol#updateApplicationPriority(UpdateApplicationPriorityRequest)]]> +     +     + + + + + + + + + + + + ApplicationId of the application. + @return ApplicationId of the application]]> + + + + + + ApplicationId of the application. + @param applicationId ApplicationId of the application]]> + + + + + ApplicationTimeouts of the application. Timeout value is + in ISO8601 standard with format yyyy-MM-dd'T'HH:mm:ss.SSSZ. + @return all ApplicationTimeouts of the application.]]> + + + + + + ApplicationTimeouts for the application. Timeout value + is absolute. Timeout value should meet ISO8601 format. Support ISO8601 + format is yyyy-MM-dd'T'HH:mm:ss.SSSZ. All pre-existing Map entries + are cleared before adding the new Map. + @param applicationTimeouts ApplicationTimeoutss for the + application]]> + + + + + The request sent by the client to the ResourceManager to set or + update the application timeout. +

    +

    + The request includes the {@link ApplicationId} of the application and timeout + to be set for an application +

    ]]> +     +     + + + + + + + + + + ApplicationTimeouts of the application. Timeout value is + in ISO8601 standard with format yyyy-MM-dd'T'HH:mm:ss.SSSZ. + @return all ApplicationTimeouts of the application.]]> + + + + + + ApplicationTimeouts for the application. Timeout value + is absolute. Timeout value should meet ISO8601 format. Support ISO8601 + format is yyyy-MM-dd'T'HH:mm:ss.SSSZ. All pre-existing Map entries + are cleared before adding the new Map. + @param applicationTimeouts ApplicationTimeoutss for the + application]]> + + + + + The response sent by the ResourceManager to the client on update + application timeout. +

    +

    + A response without exception means that the update has completed + successfully. +

    ]]> +     +     + + + + + + + + ApplicationId of the resource to be used. + + @return ApplicationId]]> + + + + + + ApplicationId of the resource to be used. + + @param id ApplicationId]]> + + + + + key of the resource to be used. + + @return key]]> + + + + + + key of the resource to be used. + + @param key unique identifier for the resource]]> + + + + + The request from clients to the SharedCacheManager that claims a + resource in the shared cache. +

    ]]> +     +     + + + + + + + + Path corresponding to the requested resource in the + shared cache. + + @return String A Path if the resource exists in the shared + cache, null otherwise]]> + + + + + + Path corresponding to a resource in the shared cache. + + @param p A Path corresponding to a resource in the shared + cache]]> + + + + + The response from the SharedCacheManager to the client that indicates whether + a requested resource exists in the cache. +

    ]]> +     +     + +     + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + ApplicationId of the ApplicationAttempId. + @return ApplicationId of the ApplicationAttempId]]> + + + + + attempt id of the Application. + @return attempt id of the Application]]> + + + + + + + + + + + + + + + + + + + + + ApplicationAttemptId denotes the particular attempt + of an ApplicationMaster for a given {@link ApplicationId}.

    + +

    Multiple attempts might be needed to run an application to completion due + to temporal failures of the ApplicationMaster such as hardware + failures, connectivity issues etc. on the node on which it was scheduled.

    ]]> +     +     + + + + + + + + + + + + + + + + + + YarnApplicationAttemptState of the application attempt. + + @return YarnApplicationAttemptState of the application attempt]]> + + + + + RPC port of this attempt ApplicationMaster. + + @return RPC port of this attempt ApplicationMaster]]> + + + + + host on which this attempt of + ApplicationMaster is running. + + @return host on which this attempt of + ApplicationMaster is running]]> + + + + + diagnositic information of the application attempt in case + of errors. + + @return diagnositic information of the application attempt in case + of errors]]> + + + + + tracking url for the application attempt. + + @return tracking url for the application attempt]]> + + + + + original tracking url for the application attempt. + + @return original tracking url for the application attempt]]> + + + + + ApplicationAttemptId of this attempt of the + application + + @return ApplicationAttemptId of the attempt]]> + + + + + ContainerId of AMContainer for this attempt + + @return ContainerId of the attempt]]> + + + + + + + finish time of the application. + + @return finish time of the application]]> + + + + + It includes details such as: +
      +
    • {@link ApplicationAttemptId} of the application.
      • +
    • Host on which the ApplicationMaster of this attempt is + running.
      • +
    • RPC port of the ApplicationMaster of this attempt.
      • +
    • Tracking URL.
      • +
    • Diagnostic information in case of errors.
      • +
    • {@link YarnApplicationAttemptState} of the application attempt.
      • +
    • {@link ContainerId} of the master Container.
      • +
    ]]> +     +     + + + + + + + + + + + + + ApplicationId + which is unique for all applications started by a particular instance + of the ResourceManager. + @return short integer identifier of the ApplicationId]]> + + + + + start time of the ResourceManager which is + used to generate globally unique ApplicationId. + @return start time of the ResourceManager]]> + + + + + + + + + + + + + + + + + + + + + ApplicationId represents the globally unique + identifier for an application.

    + +

    The globally unique nature of the identifier is achieved by using the + cluster timestamp i.e. start-time of the + ResourceManager along with a monotonically increasing counter + for the application.

    ]]> +     +     + + + + + + + + ApplicationId of the application. + @return ApplicationId of the application]]> + + + + + ApplicationAttemptId of the current + attempt of the application + @return ApplicationAttemptId of the attempt]]> + + + + + user who submitted the application. + @return user who submitted the application]]> + + + + + queue to which the application was submitted. + @return queue to which the application was submitted]]> + + + + + name of the application. + @return name of the application]]> + + + + + host on which the ApplicationMaster + is running. + @return host on which the ApplicationMaster + is running]]> + + + + + RPC port of the ApplicationMaster. + @return RPC port of the ApplicationMaster]]> + + + + + client token for communicating with the + ApplicationMaster. +

    + ClientToAMToken is the security token used by the AMs to verify + authenticity of any client. +

    + +

    + The ResourceManager, provides a secure token (via + {@link ApplicationReport#getClientToAMToken()}) which is verified by the + ApplicationMaster when the client directly talks to an AM. +

    + @return client token for communicating with the + ApplicationMaster]]> +     +     + + + YarnApplicationState of the application. + @return YarnApplicationState of the application]]> + + + + + diagnositic information of the application in case of + errors. + @return diagnositic information of the application in case + of errors]]> + + + + + tracking url for the application. + @return tracking url for the application]]> + + + + + start time of the application. + @return start time of the application]]> + + + + + + + + + finish time of the application. + @return finish time of the application]]> + + + + + final finish status of the application. + @return final finish status of the application]]> + + + + + + + + + + + + + + + + + + + + + + + + + + The AMRM token is required for AM to RM scheduling operations. For + managed Application Masters YARN takes care of injecting it. For unmanaged + Applications Masters, the token must be obtained via this method and set + in the {@link org.apache.hadoop.security.UserGroupInformation} of the + current user. +

    + The AMRM token will be returned only if all the following conditions are + met: +

      +
    • the requester is the owner of the ApplicationMaster
      • +
    • the application master is an unmanaged ApplicationMaster
      • +
    • the application master is in ACCEPTED state
      • +
    + Else this method returns NULL. + + @return the AM to RM token if available.]]> +     +     + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + It includes details such as: +
      +
    • {@link ApplicationId} of the application.
      • +
    • Applications user.
      • +
    • Application queue.
      • +
    • Application name.
      • +
    • Host on which the ApplicationMaster is running.
      • +
    • RPC port of the ApplicationMaster.
      • +
    • Tracking URL.
      • +
    • {@link YarnApplicationState} of the application.
      • +
    • Diagnostic information in case of errors.
      • +
    • Start time of the application.
      • +
    • Client {@link Token} of the application (if security is enabled).
      • +
    + + @see ApplicationClientProtocol#getApplicationReport(org.apache.hadoop.yarn.api.protocolrecords.GetApplicationReportRequest)]]> +     +     + + + + + + + + + + + + + Resource. -1 for invalid/inaccessible reports. + @return the used Resource]]> + + + + + Resource. -1 for invalid/inaccessible reports. + @return the reserved Resource]]> + + + + + Resource. -1 for invalid/inaccessible reports. + @return the needed Resource]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + ApplicationId of the submitted application. + @return ApplicationId of the submitted application]]> + + + + + + ApplicationId of the submitted application. + @param applicationId ApplicationId of the submitted + application]]> + + + + + name. + @return application name]]> + + + + + + name. + @param applicationName application name]]> + + + + + queue to which the application is being submitted. + @return queue to which the application is being submitted]]> + + + + + + queue to which the application is being submitted + @param queue queue to which the application is being submitted]]> + + + + + Priority of the application. + @return Priority of the application]]> + + + + + ContainerLaunchContext to describe the + Container with which the ApplicationMaster is + launched. + @return ContainerLaunchContext for the + ApplicationMaster container]]> + + + + + + ContainerLaunchContext to describe the + Container with which the ApplicationMaster is + launched. + @param amContainer ContainerLaunchContext for the + ApplicationMaster container]]> + + + + + YarnApplicationState. + Such apps will not be retried by the RM on app attempt failure. + The default value is false. + @return true if the AM is not managed by the RM]]> + + + + + + + + + + + + + + + + + + + + + + ApplicationMaster for this + application. Please note this will be DEPRECATED, use getResource + in getAMContainerResourceRequest instead. + + @return the resource required by the ApplicationMaster for + this application.]]> + + + + + + ApplicationMaster for this + application. + + @param resource the resource required by the ApplicationMaster + for this application.]]> + + + + + + + + + + + + + + + + + + + + + + + For managed AM, if the flag is true, running containers will not be killed + when application attempt fails and these containers will be retrieved by + the new application attempt on registration via + {@link ApplicationMasterProtocol#registerApplicationMaster(RegisterApplicationMasterRequest)}. +

    +

    + For unmanaged AM, if the flag is true, RM allows re-register and returns + the running containers in the same attempt back to the UAM for HA. +

    + + @param keepContainers the flag which indicates whether to keep containers + across application attempts.]]> +     +     + + + + + + + + + + + + + + + + + + + + + + + + + getResource and getPriority of + ApplicationSubmissionContext. + + Number of containers and Priority will be ignored. + + @return ResourceRequest of the AM container + @deprecated See {@link #getAMContainerResourceRequests()}]]> + + + + + + + + + + + getAMContainerResourceRequest and its behavior. + + Number of containers and Priority will be ignored. + + @return List of ResourceRequests of the AM container]]> + + + + + + + + + + + + + + + + + + + + + + LogAggregationContext of the application + + @return LogAggregationContext of the application]]> + + + + + + LogAggregationContext for the application + + @param logAggregationContext + for the application]]> + + + + + + + + + + + + + + + + ApplicationTimeouts of the application. Timeout value is + in seconds. + @return all ApplicationTimeouts of the application.]]> + + + + + + ApplicationTimeouts for the application in seconds. + All pre-existing Map entries are cleared before adding the new Map. +

    + Note: If application timeout value is less than or equal to zero + then application submission will throw an exception. +

    + @param applicationTimeouts ApplicationTimeoutss for the + application]]> +     +     + + + + + + + + + + + + + + It includes details such as: +
      +
    • {@link ApplicationId} of the application.
      • +
    • Application user.
      • +
    • Application name.
      • +
    • {@link Priority} of the application.
      • +
    • + {@link ContainerLaunchContext} of the container in which the + ApplicationMaster is executed. +
      • +
    • + maxAppAttempts. The maximum number of application attempts. + It should be no larger than the global number of max attempts in the + YARN configuration. +
      • +
    • + attemptFailuresValidityInterval. The default value is -1. + when attemptFailuresValidityInterval in milliseconds is set to + {@literal >} 0, the failure number will no take failures which happen + out of the validityInterval into failure count. If failure count + reaches to maxAppAttempts, the application will be failed. +
      • +
    • Optional, application-specific {@link LogAggregationContext}
      • +
    + + @see ContainerLaunchContext + @see ApplicationClientProtocol#submitApplication(org.apache.hadoop.yarn.api.protocolrecords.SubmitApplicationRequest)]]> +     +     + + + + + + + + + + + + + + + + + + + + + + + + expiryTime for given timeout type. + @return expiryTime in ISO8601 standard with format + yyyy-MM-dd'T'HH:mm:ss.SSSZ.]]> + + + + + + expiryTime for given timeout type. + @param expiryTime in ISO8601 standard with format + yyyy-MM-dd'T'HH:mm:ss.SSSZ.]]> + + + + + Remaining Time of an application for given timeout type. + @return Remaining Time in seconds.]]> + + + + + + Remaining Time of an application for given timeout type. + @param remainingTime in seconds.]]> + + + + +
  • {@link ApplicationTimeoutType} of the timeout type.
    • +
  • Expiry time in ISO8601 standard with format + yyyy-MM-dd'T'HH:mm:ss.SSSZ or "UNLIMITED".
    • +
  • Remaining time in seconds.
    • + + The possible values for {ExpiryTime, RemainingTimeInSeconds} are +
      +
    • {UNLIMITED,-1} : Timeout is not configured for given timeout type + (LIFETIME).
      • +
    • {ISO8601 date string, 0} : Timeout is configured and application has + completed.
      • +
    • {ISO8601 date string, greater than zero} : Timeout is configured and + application is RUNNING. Application will be timed out after configured + value.
      • +
    ]]> +     +     + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Resource allocated to the container. + @return Resource allocated to the container]]> + + + + + Priority at which the Container was + allocated. + @return Priority at which the Container was + allocated]]> + + + + + ContainerToken for the container. +

    ContainerToken is the security token used by the framework + to verify authenticity of any Container.

    + +

    The ResourceManager, on container allocation provides a + secure token which is verified by the NodeManager on + container launch.

    + +

    Applications do not need to care about ContainerToken, they + are transparently handled by the framework - the allocated + Container includes the ContainerToken.

    + + @see ApplicationMasterProtocol#allocate(org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest) + @see ContainerManagementProtocol#startContainers(org.apache.hadoop.yarn.api.protocolrecords.StartContainersRequest) + + @return ContainerToken for the container]]> +     +     + + + ID corresponding to the original {@code + ResourceRequest{@link #getAllocationRequestId()}}s which is satisfied by + this allocated {@code Container}. +

    + The scheduler may return multiple {@code AllocateResponse}s corresponding + to the same ID as and when scheduler allocates {@code Container}s. + Applications can continue to completely ignore the returned ID in + the response and use the allocation for any of their outstanding requests. +

    + + @return the ID corresponding to the original allocation request + which is satisfied by this allocation.]]> + + + + + The {@code ResourceManager} is the sole authority to allocate any + {@code Container} to applications. The allocated {@code Container} + is always on a single node and has a unique {@link ContainerId}. It has + a specific amount of {@link Resource} allocated. +

    + It includes details such as: +

      +
    • {@link ContainerId} for the container, which is globally unique.
      • +
    • + {@link NodeId} of the node on which it is allocated. +
      • +
    • HTTP uri of the node.
      • +
    • {@link Resource} allocated to the container.
      • +
    • {@link Priority} at which the container was allocated.
      • +
    • + Container {@link Token} of the container, used to securely verify + authenticity of the allocation. +
      • +
    + + Typically, an {@code ApplicationMaster} receives the {@code Container} + from the {@code ResourceManager} during resource-negotiation and then + talks to the {@code NodeManager} to start/stop containers. + + @see ApplicationMasterProtocol#allocate(org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest) + @see ContainerManagementProtocol#startContainers(org.apache.hadoop.yarn.api.protocolrecords.StartContainersRequest) + @see ContainerManagementProtocol#stopContainers(org.apache.hadoop.yarn.api.protocolrecords.StopContainersRequest)]]> +     +     + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + ApplicationAttemptId of the application to which the + Container was assigned. +

    + Note: If containers are kept alive across application attempts via + {@link ApplicationSubmissionContext#setKeepContainersAcrossApplicationAttempts(boolean)} + the ContainerId does not necessarily contain the current + running application attempt's ApplicationAttemptId This + container can be allocated by previously exited application attempt and + managed by the current running attempt thus have the previous application + attempt's ApplicationAttemptId. +

    + + @return ApplicationAttemptId of the application to which the + Container was assigned]]> +     +     + + + ContainerId, + which doesn't include epoch. Note that this method will be marked as + deprecated, so please use getContainerId instead. + @return lower 32 bits of identifier of the ContainerId]]> + + + + + ContainerId. Upper 24 bits are + reserved as epoch of cluster, and lower 40 bits are reserved as + sequential number of containers. + @return identifier of the ContainerId]]> + + + + + + + + + + + + + + + + + + + + + + + + ContainerId represents a globally unique identifier + for a {@link Container} in the cluster.

    ]]> +     +     + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + LocalResource required by the container. + @return all LocalResource required by the container]]> + + + + + + LocalResource required by the container. All pre-existing + Map entries are cleared before adding the new Map + @param localResources LocalResource required by the container]]> + + + + + + Get application-specific binary service data. This is a map keyed + by the name of each {@link AuxiliaryService} that is configured on a + NodeManager and value correspond to the application specific data targeted + for the keyed {@link AuxiliaryService}. +

    + +

    + This will be used to initialize this application on the specific + {@link AuxiliaryService} running on the NodeManager by calling + {@link AuxiliaryService#initializeApplication(ApplicationInitializationContext)} +

    + + @return application-specific binary service data]]> +     +     + + + + + Set application-specific binary service data. This is a map keyed + by the name of each {@link AuxiliaryService} that is configured on a + NodeManager and value correspond to the application specific data targeted + for the keyed {@link AuxiliaryService}. All pre-existing Map entries are + preserved. +

    + + @param serviceData + application-specific binary service data]]> +     +     + + + environment variables for the container. + @return environment variables for the container]]> + + + + + + environment variables for the container. All pre-existing Map + entries are cleared before adding the new Map + @param environment environment variables for the container]]> + + + + + commands for launching the container. + @return the list of commands for launching the container]]> + + + + + + commands for launching the container. All + pre-existing List entries are cleared before adding the new List + @param commands the list of commands for launching the container]]> + + + + + ApplicationACLs for the application. + @return all the ApplicationACLs]]> + + + + + + ApplicationACLs for the application. All pre-existing + Map entries are cleared before adding the new Map + @param acls ApplicationACLs for the application]]> + + + + + ContainerRetryContext to relaunch container. + @return ContainerRetryContext to relaunch container.]]> + + + + + + ContainerRetryContext to relaunch container. + @param containerRetryContext ContainerRetryContext to + relaunch container.]]> + + + + + It includes details such as: +
      +
    • {@link ContainerId} of the container.
      • +
    • {@link Resource} allocated to the container.
      • +
    • User to whom the container is allocated.
      • +
    • Security tokens (if security is enabled).
      • +
    • + {@link LocalResource} necessary for running the container such + as binaries, jar, shared-objects, side-files etc. +
      • +
    • Optional, application-specific binary service data.
      • +
    • Environment variables for the launched process.
      • +
    • Command to launch the container.
      • +
    • Retry strategy when container exits with failure.
      • +
    + + @see ContainerManagementProtocol#startContainers(org.apache.hadoop.yarn.api.protocolrecords.StartContainersRequest)]]> +     +     + + + + + + + + ContainerId of the container. + + @return ContainerId of the container.]]> + + + + + + + + Resource of the container. + + @return allocated Resource of the container.]]> + + + + + + + + NodeId where container is running. + + @return allocated NodeId where container is running.]]> + + + + + + + + Priority of the container. + + @return allocated Priority of the container.]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + ContainerState of the container. + + @return final ContainerState of the container.]]> + + + + + + + + exit status of the container. + + @return final exit status of the container.]]> + + + + + + + + + + + + + + + + + + + + + + + It includes details such as: +
      +
    • {@link ContainerId} of the container.
      • +
    • Allocated Resources to the container.
      • +
    • Assigned Node id.
      • +
    • Assigned Priority.
      • +
    • Creation Time.
      • +
    • Finish Time.
      • +
    • Container Exit Status.
      • +
    • {@link ContainerState} of the container.
      • +
    • Diagnostic information in case of errors.
      • +
    • Log URL.
      • +
    • nodeHttpAddress
      • +
    ]]> +     +     + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + It provides details such as: +
      +
    • + {@link ContainerRetryPolicy} : + - NEVER_RETRY(DEFAULT value): no matter what error code is when container + fails to run, just do not retry. + - RETRY_ON_ALL_ERRORS: no matter what error code is, when container fails + to run, just retry. + - RETRY_ON_SPECIFIC_ERROR_CODES: when container fails to run, do retry if + the error code is one of errorCodes, otherwise do not retry. + + Note: if error code is 137(SIGKILL) or 143(SIGTERM), it will not retry + because it is usually killed on purpose. +
      • +
    • + maxRetries specifies how many times to retry if need to retry. + If the value is -1, it means retry forever. +
      • +
    • retryInterval specifies delaying some time before relaunch + container, the unit is millisecond.
      • +
    • + failuresValidityInterval: default value is -1. + When failuresValidityInterval in milliseconds is set to {@literal >} 0, + the failure number will not take failures which happen out of the + failuresValidityInterval into failure count. If failure count + reaches to maxRetries, the container will be failed. +
      • +
    ]]> +     +     + + + + + + + + + + Retry policy for relaunching a Container.

    ]]> +     +     + + + + + + + + + + State of a Container.

    ]]> +     +     + + + + + + + + ContainerId of the container. + @return ContainerId of the container]]> + + + + + ExecutionType of the container. + @return ExecutionType of the container]]> + + + + + ContainerState of the container. + @return ContainerState of the container]]> + + + + + Get the exit status for the container.

    + +

    Note: This is valid only for completed containers i.e. containers + with state {@link ContainerState#COMPLETE}. + Otherwise, it returns an ContainerExitStatus.INVALID. +

    + +

    Containers killed by the framework, either due to being released by + the application or being 'lost' due to node failures etc. have a special + exit code of ContainerExitStatus.ABORTED.

    + +

    When threshold number of the nodemanager-local-directories or + threshold number of the nodemanager-log-directories become bad, then + container is not launched and is exited with ContainersExitStatus.DISKS_FAILED. +

    + + @return exit status for the container]]> +     +     + + + diagnostic messages for failed containers. + @return diagnostic messages for failed containers]]> + + + + + Resource allocated to the container. + @return Resource allocated to the container]]> + + + + + + + + + + + + + + + + + + + + It provides details such as: +
      +
    • {@code ContainerId} of the container.
      • +
    • {@code ExecutionType} of the container.
      • +
    • {@code ContainerState} of the container.
      • +
    • Exit status of a completed container.
      • +
    • Diagnostic message for a failed container.
      • +
    • {@link Resource} allocated to the container.
      • +
    ]]> +     +     + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + The execution types are the following: +
      +
    • {@link #GUARANTEED} - this container is guaranteed to start its + execution, once the corresponding start container request is received by + an NM. +
    • {@link #OPPORTUNISTIC} - the execution of this container may not start + immediately at the NM that receives the corresponding start container + request (depending on the NM's available resources). Moreover, it may be + preempted if it blocks a GUARANTEED container from being executed. +
    ]]> +     +     + + + + + + + + + + + + + + + + + + + ExecutionType of the requested container. + + @param execType + ExecutionType of the requested container]]> + + + + + ExecutionType. + + @return ExecutionType.]]> + + + + + + + + + + + ResourceRequest. + Defaults to false. + @return whether ExecutionType request should be strictly honored]]> + + + + + + + + + ExecutionType as well as flag that explicitly asks the + configuredScheduler to return Containers of exactly the Execution Type + requested.]]> + + + + + + + + + + + + Application.]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
  • resource key
    • +
  • {@link LocalizationState} of the resource
    • + ]]> +     +     + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + location of the resource to be localized. + @return location of the resource to be localized]]> + + + + + + location of the resource to be localized. + @param resource location of the resource to be localized]]> + + + + + size of the resource to be localized. + @return size of the resource to be localized]]> + + + + + + size of the resource to be localized. + @param size size of the resource to be localized]]> + + + + + timestamp of the resource to be localized, used + for verification. + @return timestamp of the resource to be localized]]> + + + + + + timestamp of the resource to be localized, used + for verification. + @param timestamp timestamp of the resource to be localized]]> + + + + + LocalResourceType of the resource to be localized. + @return LocalResourceType of the resource to be localized]]> + + + + + + LocalResourceType of the resource to be localized. + @param type LocalResourceType of the resource to be localized]]> + + + + + LocalResourceVisibility of the resource to be + localized. + @return LocalResourceVisibility of the resource to be + localized]]> + + + + + + LocalResourceVisibility of the resource to be + localized. + @param visibility LocalResourceVisibility of the resource to be + localized]]> + + + + + pattern that should be used to extract entries from the + archive (only used when type is PATTERN). + @return pattern that should be used to extract entries from the + archive.]]> + + + + + + pattern that should be used to extract entries from the + archive (only used when type is PATTERN). + @param pattern pattern that should be used to extract entries + from the archive.]]> + + + + + + + + + + + shouldBeUploadedToSharedCache + of this request]]> + + + + LocalResource represents a local resource required to + run a container.

    + +

    The NodeManager is responsible for localizing the resource + prior to launching the container.

    + +

    Applications can specify {@link LocalResourceType} and + {@link LocalResourceVisibility}.

    + + @see LocalResourceType + @see LocalResourceVisibility + @see ContainerLaunchContext + @see ApplicationSubmissionContext + @see ContainerManagementProtocol#startContainers(org.apache.hadoop.yarn.api.protocolrecords.StartContainersRequest)]]> +     +     + + + + + + + + + + type + of a resource localized by the {@code NodeManager}. +

    + The type can be one of: +

      +
    • + {@link #FILE} - Regular file i.e. uninterpreted bytes. +
      • +
    • + {@link #ARCHIVE} - Archive, which is automatically unarchived by the + NodeManager. +
      • +
    • + {@link #PATTERN} - A hybrid between {@link #ARCHIVE} and {@link #FILE}. +
      • +
    + + @see LocalResource + @see ContainerLaunchContext + @see ApplicationSubmissionContext + @see ContainerManagementProtocol#startContainers(org.apache.hadoop.yarn.api.protocolrecords.StartContainersRequest)]]> +     +     + + + + + + + + + + visibility + of a resource localized by the {@code NodeManager}. +

    + The visibility can be one of: +

      +
    • {@link #PUBLIC} - Shared by all users on the node.
      • +
    • + {@link #PRIVATE} - Shared among all applications of the + same user on the node. +
      • +
    • + {@link #APPLICATION} - Shared only among containers of the + same application on the node. +
      • +
    + + @see LocalResource + @see ContainerLaunchContext + @see ApplicationSubmissionContext + @see ContainerManagementProtocol#startContainers(org.apache.hadoop.yarn.api.protocolrecords.StartContainersRequest)]]> +     +     + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + It includes details such as: +
      +
    • + includePattern. It uses Java Regex to filter the log files + which match the defined include pattern and those log files + will be uploaded when the application finishes. +
      • +
    • + excludePattern. It uses Java Regex to filter the log files + which match the defined exclude pattern and those log files + will not be uploaded when application finishes. If the log file + name matches both the include and the exclude pattern, this file + will be excluded eventually. +
      • +
    • + rolledLogsIncludePattern. It uses Java Regex to filter the log files + which match the defined include pattern and those log files + will be aggregated in a rolling fashion. +
      • +
    • + rolledLogsExcludePattern. It uses Java Regex to filter the log files + which match the defined exclude pattern and those log files + will not be aggregated in a rolling fashion. If the log file + name matches both the include and the exclude pattern, this file + will be excluded eventually. +
      • +
    • + policyClassName. The policy class name that implements + ContainerLogAggregationPolicy. At runtime, nodemanager will the policy + if a given container's log should be aggregated based on the + ContainerType and other runtime state such as exit code by calling + ContainerLogAggregationPolicy#shouldDoLogAggregation. + This is useful when the app only wants to aggregate logs of a subset of + containers. Here are the available policies. Please make sure to specify + the canonical name by prefixing org.apache.hadoop.yarn.server. + nodemanager.containermanager.logaggregation. + to the class simple name below. + NoneContainerLogAggregationPolicy: skip aggregation for all containers. + AllContainerLogAggregationPolicy: aggregate all containers. + AMOrFailedContainerLogAggregationPolicy: aggregate application master + or failed containers. + FailedOrKilledContainerLogAggregationPolicy: aggregate failed or killed + containers + FailedContainerLogAggregationPolicy: aggregate failed containers + AMOnlyLogAggregationPolicy: aggregate application master containers + SampleContainerLogAggregationPolicy: sample logs of successful worker + containers, in addition to application master and failed/killed + containers. + LimitSizeContainerLogAggregationPolicy: skip aggregation for killed + containers whose log size exceeds the limit of container log size. + If it isn't specified, it will use the cluster-wide default policy + defined by configuration yarn.nodemanager.log-aggregation.policy.class. + The default value of yarn.nodemanager.log-aggregation.policy.class is + AllContainerLogAggregationPolicy. +
      • +
    • + policyParameters. The parameters passed to the policy class via + ContainerLogAggregationPolicy#parseParameters during the policy object + initialization. This is optional. Some policy class might use parameters + to adjust its settings. It is up to policy class to define the scheme of + parameters. + For example, SampleContainerLogAggregationPolicy supports the format of + "SR:0.5,MIN:50", which means sample rate of 50% beyond the first 50 + successful worker containers. +
      • +
    + + @see ApplicationSubmissionContext]]> +     +     + + + + + + + + NodeManager for which the NMToken + is used to authenticate. + @return the {@link NodeId} of the NodeManager for which the + NMToken is used to authenticate.]]> + + + + + + + + NodeManager + @return the {@link Token} used for authenticating with NodeManager]]> + + + + + + + + + + + + The NMToken is used for authenticating communication with + NodeManager

    +

    It is issued by ResourceMananger when ApplicationMaster + negotiates resource with ResourceManager and + validated on NodeManager side.

    + @see AllocateResponse#getNMTokens()]]> +     +     + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Node Attribute is a kind of a label which represents one of the + attribute/feature of a Node. Its different from node partition label as + resource guarantees across the queues will not be maintained for these type + of labels. +

    +

    + A given Node can be mapped with any kind of attribute, few examples are + HAS_SSD=true, JAVA_VERSION=JDK1.8, OS_TYPE=WINDOWS. +

    +

    + Its not compulsory for all the attributes to have value, empty string is the + default value of the NodeAttributeType.STRING +

    +

    + Node Attribute Prefix is used as namespace to segregate the attributes. +

    ]]> +     +     + + + + + + + + + + + + + + + + + + + + + + + + + Node Attribute Info describes a NodeAttribute. +

    ]]> +     +     + + + + + + + + + + + + + + + + + + + + + + + + + Node AttributeKey uniquely identifies a given Node Attribute. Node Attribute + is identified based on attribute prefix and name. +

    +

    + Node Attribute Prefix is used as namespace to segregate the attributes. +

    ]]> +     +     + + + + + + + + + + + + + + + + + + + + + + + Type of a node Attribute. +

    + Based on this attribute expressions and values will be evaluated.]]> +     +     + + + + + + + + + + + + + hostname of the node. + @return hostname of the node]]> + + + + + port for communicating with the node. + @return port for communicating with the node]]> + + + + + + + + + + + + + + + + + + + NodeId is the unique identifier for a node.

    + +

    It includes the hostname and port to uniquely + identify the node. Thus, it is unique across restarts of any + NodeManager.

    ]]> +     +     + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + NodeId of the node. + @return NodeId of the node]]> + + + + + NodeState of the node. + @return NodeState of the node]]> + + + + + http address of the node. + @return http address of the node]]> + + + + + rack name for the node. + @return rack name for the node]]> + + + + + used Resource on the node. + @return used Resource on the node]]> + + + + + total Resource on the node. + @return total Resource on the node]]> + + + + + diagnostic health report of the node. + @return diagnostic health report of the node]]> + + + + + last timestamp at which the health report was received. + @return last timestamp at which the health report was received]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + It includes details such as: +
      +
    • {@link NodeId} of the node.
      • +
    • HTTP Tracking URL of the node.
      • +
    • Rack name for the node.
      • +
    • Used {@link Resource} on the node.
      • +
    • Total available {@link Resource} of the node.
      • +
    • Number of running containers on the node.
      • +
    + + @see ApplicationClientProtocol#getClusterNodes(org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodesRequest)]]> +     +     + + + + + + + + + + + + + + + + State of a Node.

    ]]> +     +     + + + + + + + + + + + + + + + + + + + + + + Mapping of Attribute Value to a Node. +

    ]]> +     +     + + + + + + + + + + + + ResourceManager. + @see PreemptionContract + @see StrictPreemptionContract]]> + + + + + + + + + + ApplicationMaster about resources requested back by the + ResourceManager. + @see AllocateRequest#setAskList(List)]]> + + + + + ApplicationMaster that may be reclaimed by the + ResourceManager. If the AM prefers a different set of + containers, then it may checkpoint or kill containers matching the + description in {@link #getResourceRequest}. + @return Set of containers at risk if the contract is not met.]]> + + + + ResourceManager. + The ApplicationMaster (AM) can satisfy this request according + to its own priorities to prevent containers from being forcibly killed by + the platform. + @see PreemptionMessage]]> + + + + + + + + + + ResourceManager]]> + + + + + + + + + + The AM should decode both parts of the message. The {@link + StrictPreemptionContract} specifies particular allocations that the RM + requires back. The AM can checkpoint containers' state, adjust its execution + plan to move the computation, or take no action and hope that conditions that + caused the RM to ask for the container will change. +

    + In contrast, the {@link PreemptionContract} also includes a description of + resources with a set of containers. If the AM releases containers matching + that profile, then the containers enumerated in {@link + PreemptionContract#getContainers()} may not be killed. +

    + Each preemption message reflects the RM's current understanding of the + cluster state, so a request to return N containers may not + reflect containers the AM is releasing, recently exited containers the RM has + yet to learn about, or new containers allocated before the message was + generated. Conversely, an RM may request a different profile of containers in + subsequent requests. +

    + The policy enforced by the RM is part of the scheduler. Generally, only + containers that have been requested consistently should be killed, but the + details are not specified.]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + The ACL is one of: +

      +
    • + {@link #SUBMIT_APPLICATIONS} - ACL to submit applications to the queue. +
      • +
    • {@link #ADMINISTER_QUEUE} - ACL to administer the queue.
      • +
    + + @see QueueInfo + @see ApplicationClientProtocol#getQueueUserAcls(org.apache.hadoop.yarn.api.protocolrecords.GetQueueUserAclsInfoRequest)]]> +     +     + + + + + + + + name of the queue. + @return name of the queue]]> + + + + + configured capacity of the queue. + @return configured capacity of the queue]]> + + + + + maximum capacity of the queue. + @return maximum capacity of the queue]]> + + + + + current capacity of the queue. + @return current capacity of the queue]]> + + + + + child queues of the queue. + @return child queues of the queue]]> + + + + + running applications of the queue. + @return running applications of the queue]]> + + + + + QueueState of the queue. + @return QueueState of the queue]]> + + + + + accessible node labels of the queue. + @return accessible node labels of the queue]]> + + + + + default node label expression of the queue, this takes + affect only when the ApplicationSubmissionContext and + ResourceRequest don't specify their + NodeLabelExpression. + + @return default node label expression of the queue]]> + + + + + + + + queue stats for the queue + + @return queue stats of the queue]]> + + + + + + + + + + + preemption status of the queue. + @return if property is not in proto, return null; + otherwise, return preemption status of the queue]]> + + + + + + + + + + + + + + + It includes information such as: +
      +
    • Queue name.
      • +
    • Capacity of the queue.
      • +
    • Maximum capacity of the queue.
      • +
    • Current capacity of the queue.
      • +
    • Child queues.
      • +
    • Running applications.
      • +
    • {@link QueueState} of the queue.
      • +
    • {@link QueueConfigurations} of the queue.
      • +
    + + @see QueueState + @see QueueConfigurations + @see ApplicationClientProtocol#getQueueInfo(org.apache.hadoop.yarn.api.protocolrecords.GetQueueInfoRequest)]]> +     +     + + + + + + + + + + + A queue is in one of: +
      +
    • {@link #RUNNING} - normal state.
      • +
    • {@link #STOPPED} - not accepting new application submissions.
      • +
    • + {@link #DRAINING} - not accepting new application submissions + and waiting for applications finish. +
      • +
    + + @see QueueInfo + @see ApplicationClientProtocol#getQueueInfo(org.apache.hadoop.yarn.api.protocolrecords.GetQueueInfoRequest)]]> +     +     + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + queue name of the queue. + @return queue name of the queue]]> + + + + + QueueACL for the given user. + @return list of QueueACL for the given user]]> + + + + QueueUserACLInfo provides information {@link QueueACL} for + the given user.

    + + @see QueueACL + @see ApplicationClientProtocol#getQueueUserAcls(org.apache.hadoop.yarn.api.protocolrecords.GetQueueUserAclsInfoRequest)]]> +     +     + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + The ACL is one of: +
      +
    • + {@link #ADMINISTER_RESERVATIONS} - ACL to create, list, update and + delete reservations. +
      • +
    • {@link #LIST_RESERVATIONS} - ACL to list reservations.
      • +
    • {@link #SUBMIT_RESERVATIONS} - ACL to create reservations.
      • +
    + Users can always list, update and delete their own reservations.]]> +     +     + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + It includes: +
      +
    • Duration of the reservation.
      • +
    • Acceptance time of the duration.
      • +
    • + List of {@link ResourceAllocationRequest}, which includes the time + interval, and capability of the allocation. + {@code ResourceAllocationRequest} represents an allocation + made for a reservation for the current state of the queue. This can be + changed for reasons such as re-planning, but will always be subject to + the constraints of the user contract as described by + {@link ReservationDefinition} +
      • +
    • {@link ReservationId} of the reservation.
      • +
    • {@link ReservationDefinition} used to make the reservation.
      • +
    + + @see ResourceAllocationRequest + @see ReservationId + @see ReservationDefinition]]> +     +     + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + start time of the {@code ResourceManager} which is used to + generate globally unique {@link ReservationId}. + + @return start time of the {@code ResourceManager}]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + {@link ReservationId} represents the globally unique identifier for + a reservation. +

    + +

    + The globally unique nature of the identifier is achieved by using the + cluster timestamp i.e. start-time of the {@code ResourceManager} + along with a monotonically increasing counter for the reservation. +

    ]]> +     +     + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + It includes: +
      +
    • {@link Resource} required for each request.
      • +
    • + Number of containers, of above specifications, which are required by the + application. +
      • +
    • Concurrency that indicates the gang size of the request.
      • +
    ]]> +     +     + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + memory of the resource. Note - while memory has + never had a unit specified, all YARN configurations have specified memory + in MB. The assumption has been that the daemons and applications are always + using the same units. With the introduction of the ResourceInformation + class we have support for units - so this function will continue to return + memory but in the units of MB + + @return memory(in MB) of the resource]]> + + + + + memory of the resource. Note - while memory has + never had a unit specified, all YARN configurations have specified memory + in MB. The assumption has been that the daemons and applications are always + using the same units. With the introduction of the ResourceInformation + class we have support for units - so this function will continue to return + memory but in the units of MB + + @return memory of the resource]]> + + + + + + memory of the resource. Note - while memory has + never had a unit specified, all YARN configurations have specified memory + in MB. The assumption has been that the daemons and applications are always + using the same units. With the introduction of the ResourceInformation + class we have support for units - so this function will continue to set + memory but the assumption is that the value passed is in units of MB. + + @param memory memory(in MB) of the resource]]> + + + + + + memory of the resource. + @param memory memory of the resource]]> + + + + + number of virtual cpu cores of the resource. + + Virtual cores are a unit for expressing CPU parallelism. A node's capacity + should be configured with virtual cores equal to its number of physical + cores. A container should be requested with the number of cores it can + saturate, i.e. the average number of threads it expects to have runnable + at a time. + + @return num of virtual cpu cores of the resource]]> + + + + + + number of virtual cpu cores of the resource. + + Virtual cores are a unit for expressing CPU parallelism. A node's capacity + should be configured with virtual cores equal to its number of physical + cores. A container should be requested with the number of cores it can + saturate, i.e. the average number of threads it expects to have runnable + at a time. + + @param vCores number of virtual cpu cores of the resource]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Resource models a set of computer resources in the + cluster.

    + +

    Currently it models both memory and CPU.

    + +

    The unit for memory is megabytes. CPU is modeled with virtual cores + (vcores), a unit for expressing parallelism. A node's capacity should + be configured with virtual cores equal to its number of physical cores. A + container should be requested with the number of cores it can saturate, i.e. + the average number of threads it expects to have runnable at a time.

    + +

    Virtual cores take integer values and thus currently CPU-scheduling is + very coarse. A complementary axis for CPU requests that represents + processing power will likely be added in the future to enable finer-grained + resource configuration.

    + +

    Typically, applications request Resource of suitable + capability to run their component tasks.

    + + @see ResourceRequest + @see ApplicationMasterProtocol#allocate(org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest)]]> +     +     + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + It includes: +
      +
    • StartTime of the allocation.
      • +
    • EndTime of the allocation.
      • +
    • {@link Resource} reserved for the allocation.
      • +
    + + @see Resource]]> +     +     + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + blacklist of resources + for the application. + + @see ResourceRequest + @see ApplicationMasterProtocol#allocate(org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest)]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + host/rack string represents an arbitrary + host name. + + @param hostName host/rack on which the allocation is desired + @return whether the given host/rack string represents an arbitrary + host name]]> + + + + + Priority of the request. + @return Priority of the request]]> + + + + + + Priority of the request + @param priority Priority of the request]]> + + + + + host/rack) on which the allocation + is desired. + + A special value of * signifies that any resource + (host/rack) is acceptable. + + @return resource (e.g. host/rack) on which the allocation + is desired]]> + + + + + + host/rack) on which the allocation + is desired. + + A special value of * signifies that any resource name + (e.g. host/rack) is acceptable. + + @param resourceName (e.g. host/rack) on which the + allocation is desired]]> + + + + + + + + + + + + + + + + ResourceRequest. Defaults to true. + + @return whether locality relaxation is enabled with this + ResourceRequest.]]> + + + + + + ExecutionTypeRequest of the requested container. + + @param execSpec + ExecutionTypeRequest of the requested container]]> + + + + + ResourceRequest. Defaults to true. + + @return whether locality relaxation is enabled with this + ResourceRequest.]]> + + + + + + For a request at a network hierarchy level, set whether locality can be relaxed + to that level and beyond.

    + +

    If the flag is off on a rack-level ResourceRequest, + containers at that request's priority will not be assigned to nodes on that + request's rack unless requests specifically for those nodes have also been + submitted.

    + +

    If the flag is off on an {@link ResourceRequest#ANY}-level + ResourceRequest, containers at that request's priority will + only be assigned on racks for which specific requests have also been + submitted.

    + +

    For example, to request a container strictly on a specific node, the + corresponding rack-level and any-level requests should have locality + relaxation set to false. Similarly, to request a container strictly on a + specific rack, the corresponding any-level request should have locality + relaxation set to false.

    + + @param relaxLocality whether locality relaxation is enabled with this + ResourceRequest.]]> + + + + + + + + + + + + + + + + ID corresponding to this allocation request. This + ID is an identifier for different {@code ResourceRequest}s from the same + application. The allocated {@code Container}(s) received as part of the + {@code AllocateResponse} response will have the ID corresponding to the + original {@code ResourceRequest} for which the RM made the allocation. +

    + The scheduler may return multiple {@code AllocateResponse}s corresponding + to the same ID as and when scheduler allocates {@code Container}(s). + Applications can continue to completely ignore the returned ID in + the response and use the allocation for any of their outstanding requests. +

    + If one wishes to replace an entire {@code ResourceRequest} corresponding to + a specific ID, they can simply cancel the corresponding {@code + ResourceRequest} and submit a new one afresh. + + @return the ID corresponding to this allocation request.]]> + + + + + + ID corresponding to this allocation request. This + ID is an identifier for different {@code ResourceRequest}s from the same + application. The allocated {@code Container}(s) received as part of the + {@code AllocateResponse} response will have the ID corresponding to the + original {@code ResourceRequest} for which the RM made the allocation. +

    + The scheduler may return multiple {@code AllocateResponse}s corresponding + to the same ID as and when scheduler allocates {@code Container}(s). + Applications can continue to completely ignore the returned ID in + the response and use the allocation for any of their outstanding requests. +

    + If one wishes to replace an entire {@code ResourceRequest} corresponding to + a specific ID, they can simply cancel the corresponding {@code + ResourceRequest} and submit a new one afresh. +

    + If the ID is not set, scheduler will continue to work as previously and all + allocated {@code Container}(s) will have the default ID, -1. + + @param allocationRequestID the ID corresponding to this allocation + request.]]> + + + + + + Resource capability of the request. + @param capability Resource capability of the request]]> + + + + + Resource capability of the request. + @return Resource capability of the request]]> + + + + + + + + + + + + + + + + + + It includes: +

      +
    • {@link Priority} of the request.
      • +
    • + The name of the host or rack on which the allocation is + desired. A special value of * signifies that + any host/rack is acceptable to the application. +
      • +
    • {@link Resource} required for each request.
      • +
    • + Number of containers, of above specifications, which are required + by the application. +
      • +
    • + A boolean relaxLocality flag, defaulting to {@code true}, + which tells the {@code ResourceManager} if the application wants + locality to be loose (i.e. allows fall-through to rack or any) + or strict (i.e. specify hard constraint on resource allocation). +
      • +
    + + @see Resource + @see ApplicationMasterProtocol#allocate(org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest)]]> +     +     + + + + + + + priority of the request. + @see ResourceRequest#setPriority(Priority) + @param priority priority of the request + @return {@link ResourceRequestBuilder}]]> + + + + + + resourceName of the request. + @see ResourceRequest#setResourceName(String) + @param resourceName resourceName of the request + @return {@link ResourceRequestBuilder}]]> + + + + + + capability of the request. + @see ResourceRequest#setCapability(Resource) + @param capability capability of the request + @return {@link ResourceRequestBuilder}]]> + + + + + + numContainers of the request. + @see ResourceRequest#setNumContainers(int) + @param numContainers numContainers of the request + @return {@link ResourceRequestBuilder}]]> + + + + + + relaxLocality of the request. + @see ResourceRequest#setRelaxLocality(boolean) + @param relaxLocality relaxLocality of the request + @return {@link ResourceRequestBuilder}]]> + + + + + + nodeLabelExpression of the request. + @see ResourceRequest#setNodeLabelExpression(String) + @param nodeLabelExpression + nodeLabelExpression of the request + @return {@link ResourceRequestBuilder}]]> + + + + + + executionTypeRequest of the request. + @see ResourceRequest#setExecutionTypeRequest( + ExecutionTypeRequest) + @param executionTypeRequest + executionTypeRequest of the request + @return {@link ResourceRequestBuilder}]]> + + + + + + executionTypeRequest of the request with 'ensure + execution type' flag set to true. + @see ResourceRequest#setExecutionTypeRequest( + ExecutionTypeRequest) + @param executionType executionType of the request. + @return {@link ResourceRequestBuilder}]]> + + + + + + allocationRequestId of the request. + @see ResourceRequest#setAllocationRequestId(long) + @param allocationRequestId + allocationRequestId of the request + @return {@link ResourceRequestBuilder}]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + virtual memory. + + @return virtual memory in MB]]> + + + + + + virtual memory. + + @param vmem virtual memory in MB]]> + + + + + physical memory. + + @return physical memory in MB]]> + + + + + + physical memory. + + @param pmem physical memory in MB]]> + + + + + CPU utilization (The amount of vcores used). + + @return CPU utilization]]> + + + + + + CPU utilization (The amount of vcores used). + + @param cpu CPU utilization]]> + + + + + + custom resource utilization + (The amount of custom resource used). + + @param resourceName resourceName of custom resource + @return resourceName utilization]]> + + + + + + + + + + + + custom resource utilization + (The amount of custom resource used). + @param resourceName resourceName + @param utilization utilization of custom resource]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + ResourceUtilization models the utilization of a set of computer + resources in the cluster. +

    ]]> +     +     + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + allocationRequestId of the request. + + @see SchedulingRequest#setAllocationRequestId(long) + @param allocationRequestId allocationRequestId of the + request + @return {@link SchedulingRequest.SchedulingRequestBuilder}]]> + + + + + + priority of the request. + + @param priority priority of the request + @return {@link SchedulingRequest.SchedulingRequestBuilder} + @see SchedulingRequest#setPriority(Priority)]]> + + + + + + executionType of the request. + + @see SchedulingRequest#setExecutionType(ExecutionTypeRequest) + @param executionType executionType of the request + @return {@link SchedulingRequest.SchedulingRequestBuilder}]]> + + + + + + allocationTags of the request. + + @see SchedulingRequest#setAllocationTags(Set) + @param allocationTags allocationsTags of the request + @return {@link SchedulingRequest.SchedulingRequestBuilder}]]> + + + + + + executionType of the request. + + @see SchedulingRequest#setResourceSizing(ResourceSizing) + @param resourceSizing resourceSizing of the request + @return {@link SchedulingRequest.SchedulingRequestBuilder}]]> + + + + + + placementConstraintExpression of the request. + + @see SchedulingRequest#setPlacementConstraint( + PlacementConstraint) + @param placementConstraintExpression placementConstraints of + the request + @return {@link SchedulingRequest.SchedulingRequestBuilder}]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + ApplicationMaster that may be reclaimed by the + ResourceManager. + @return the set of {@link ContainerId} to be preempted.]]> + + + + ApplicationMaster (AM) + may attempt to checkpoint work or adjust its execution plan to accommodate + it. In contrast to {@link PreemptionContract}, the AM has no flexibility in + selecting which resources to return to the cluster. + @see PreemptionMessage]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Token is the security entity used by the framework + to verify authenticity of any resource.

    ]]> +     +     + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + ContainerId of the container. + @return ContainerId of the container]]> + + + + + + + + + + + ContainerUpdateType of the container. + @return ContainerUpdateType of the container.]]> + + + + + + ContainerUpdateType of the container. + @param updateType of the Container]]> + + + + + ContainerId of the container. + @return ContainerId of the container]]> + + + + + + ContainerId of the container. + @param containerId ContainerId of the container]]> + + + + + ExecutionType of the container. + @return ExecutionType of the container]]> + + + + + + ExecutionType of the container. + @param executionType ExecutionType of the container]]> + + + + + + Resource capability of the request. + @param capability Resource capability of the request]]> + + + + + Resource capability of the request. + @return Resource capability of the request]]> + + + + + + + + + + + + It includes: +
      +
    • version for the container.
      • +
    • {@link ContainerId} for the container.
      • +
    • + {@link Resource} capability of the container after the update request + is completed. +
      • +
    • + {@link ExecutionType} of the container after the update request is + completed. +
      • +
    + + Update rules: +
      +
    • + Currently only ONE aspect of the container can be updated per request + (user can either update Capability OR ExecutionType in one request.. + not both). +
      • +
    • + There must be only 1 update request per container in an allocate call. +
      • +
    • + If a new update request is sent for a container (in a subsequent allocate + call) before the first one is satisfied by the Scheduler, it will + overwrite the previous request. +
      • +
    + @see ApplicationMasterProtocol#allocate(org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest)]]> +     +     + + + + + + + + + + + + + + + ContainerUpdateType. + @return ContainerUpdateType]]> + + + + + + ContainerUpdateType. + @param updateType ContainerUpdateType]]> + + + + + Container. + @return Container]]> + + + + + + Container. + @param container Container]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + URL represents a serializable {@link java.net.URL}.

    ]]> +     +     + + + + + + + + + + RMAppAttempt.]]> + + + + + + + + + + + + ApplicationMaster.]]> + + + + + + + + + + NodeManagers in the cluster. + @return number of NodeManagers in the cluster]]> + + + + + DecommissionedNodeManagers in the cluster. + + @return number of DecommissionedNodeManagers in the cluster]]> + + + + + ActiveNodeManagers in the cluster. + + @return number of ActiveNodeManagers in the cluster]]> + + + + + LostNodeManagers in the cluster. + + @return number of LostNodeManagers in the cluster]]> + + + + + UnhealthyNodeManagers in the cluster. + + @return number of UnhealthyNodeManagers in the cluster]]> + + + + + RebootedNodeManagers in the cluster. + + @return number of RebootedNodeManagers in the cluster]]> + + + + YarnClusterMetrics represents cluster metrics.

    + +

    Currently only number of NodeManagers is provided.

    ]]> +     +     + +     + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + This class contains the information about a timeline domain, which is used + to a user to host a number of timeline entities, isolating them from others'. + The user can also define the reader and writer users/groups for the the + domain, which is used to control the access to its entities. +

    + +

    + The reader and writer users/groups pattern that the user can supply is the + same as what AccessControlList takes. +

    ]]> +     +     + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + The class that contains the the meta information of some conceptual entity + and its related events. The entity can be an application, an application + attempt, a container or whatever the user-defined object. +

    + +

    + Primary filters will be used to index the entities in + TimelineStore, such that users should carefully choose the + information they want to store as the primary filters. The remaining can be + stored as other information. +

    ]]> +     +     + + + + + + + + + + + + + ApplicationId of the + TimelineEntityGroupId. + + @return ApplicationId of the + TimelineEntityGroupId]]> + + + + + + + + timelineEntityGroupId. + + @return timelineEntityGroupId]]> + + + + + + + + + + + + + + + + + + + TimelineEntityGroupId is an abstract way for + timeline service users to represent #a group of related timeline data. + For example, all entities that represents one data flow DAG execution + can be grouped into one timeline entity group.

    ]]> +     +     + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +     + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + This class contains the information about a timeline service domain, which is + used to a user to host a number of timeline entities, isolating them from + others'. The user can also define the reader and writer users/groups for + the domain, which is used to control the access to its entities. +

    +

    + The reader and writer users/groups pattern that the user can supply is the + same as what AccessControlList takes. +

    ]]> +     +     + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + The constuctor is used to construct a proxy {@link TimelineEntity} or its + subclass object from the real entity object that carries information. +

    + +

    + It is usually used in the case where we want to recover class polymorphism + after deserializing the entity from its JSON form. +

    + @param entity the real entity that carries information]]> +     +     + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Note: Entities will be stored in the order of idPrefix specified. + If users decide to set idPrefix for an entity, they MUST provide + the same prefix for every update of this entity. +

    + Example: 
    + TimelineEntity entity = new TimelineEntity();
+ entity.setIdPrefix(value);
+
    + Users can use {@link TimelineServiceHelper#invertLong(long)} to invert + the prefix if necessary. + + @param entityIdPrefix prefix for an entity.]]> +     +     + + + + + + + +     + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +     + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + name property as a + InetSocketAddress. On an HA cluster, + this fetches the address corresponding to the RM identified by + {@link #RM_HA_ID}. + @param name property name. + @param defaultAddress the default value + @param defaultPort the default port + @return InetSocketAddress]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + yarn.resourcemanager.scheduler.class + cannot handle placement constraints, the corresponding SchedulingRequests + will be rejected. As of now, only the capacity scheduler supports + SchedulingRequests. In particular, it currently supports anti-affinity + constraints (no affinity or cardinality) and places one container at a + time. The advantage of this handler compared to the placement-processor is + that it follows the same ordering rules for queues (sorted by utilization, + priority) and apps (sorted by FIFO/fairness/priority) as the ones followed + by the main scheduler.]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + OPPORTUNISTIC containers on the NM.]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
  • default
    • +
  • docker
    • +
  • javasandbox
    • +
  • runc
    • + ]]> +     +     + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
  • NONE - the RM will do nothing special.
    • +
  • LENIENT - the RM will generate and provide a keystore and truststore + to the AM, which it is free to use for HTTPS in its tracking URL web + server. The RM proxy will still allow HTTP connections to AMs that opt + not to use HTTPS.
    • +
  • STRICT - this is the same as LENIENT, except that the RM proxy will + only allow HTTPS connections to AMs; HTTP connections will be blocked + and result in a warning page to the user.
    • + ]]> +     +     + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Default platform-specific CLASSPATH for YARN applications. A + comma-separated list of CLASSPATH entries constructed based on the client + OS environment expansion syntax. +

    +

    + Note: Use {@link #DEFAULT_YARN_CROSS_PLATFORM_APPLICATION_CLASSPATH} for + cross-platform practice i.e. submit an application from a Windows client to + a Linux/Unix server or vice versa. +

    ]]> +     +     + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +     + +     + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + The information is passed along to applications via + {@link StartContainersResponse#getAllServicesMetaData()} that is returned by + {@link ContainerManagementProtocol#startContainers(StartContainersRequest)} +

    + + @return meta-data for this service that should be made available to + applications.]]> +     +     + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +     + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + The method used by the NodeManager log aggregation service + to initial the policy object with parameters specified by the application + or the cluster-wide setting. +

    + + @param parameters parameters with scheme defined by the policy class.]]> +     +     + + + + + The method used by the NodeManager log aggregation service + to ask the policy object if a given container's logs should be aggregated. +

    + + @param logContext ContainerLogContext + @return Whether or not the container's logs should be aggregated.]]> +     +     + + + +     + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + The method used by administrators to ask SCM to run cleaner task right away +

    + + @param request request SharedCacheManager to run a cleaner task + @return SharedCacheManager returns an empty response + on success and throws an exception on rejecting the request + @throws YarnException + @throws IOException]]> +     +     + + + The protocol between administrators and the SharedCacheManager +

    ]]> +     +     + +     + + + + + + + + + + + + + + + + + Tag1(N1),P1:Tag2(N2),P2:...:TagN(Nn),Pn

    + + where TagN(Nn) is a key value pair to determine the source + allocation tag and the number of allocations, such as: + +

    foo(3)

    + + Optional when using NodeAttribute Constraint. + + and where Pn can be any form of a valid constraint expression, + such as: + +
      +
    • in,node,foo,bar
      • +
    • notin,node,foo,bar,1,2
      • +
    • and(notin,node,foo:notin,node,bar)
      • +
    + + and NodeAttribute Constraint such as + +
      +
    • yarn.rm.io/foo=true
      • +
    • java=1.7,1.8
      • +
    + @param expression expression string. + @return a map of source tags to placement constraint mapping. + @throws PlacementConstraintParseException]]> +     +     + + + + + +     + +     + + + + + +     diff --git a/hadoop-yarn-project/hadoop-yarn/dev-support/jdiff/Apache_Hadoop_YARN_Client_3.2.4.xml b/hadoop-yarn-project/hadoop-yarn/dev-support/jdiff/Apache_Hadoop_YARN_Client_3.2.4.xml new file mode 100644 index 0000000000000..b8f2a2e529c2e --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/dev-support/jdiff/Apache_Hadoop_YARN_Client_3.2.4.xml @@ -0,0 +1,3006 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + In secure mode, YARN verifies access to the application, queue + etc. before accepting the request. +

    + If the user does not have VIEW_APP access then the following + fields in the report will be set to stubbed values: +

      +
    • host - set to "N/A"
      • +
    • RPC port - set to -1
      • +
    • client token - set to "N/A"
      • +
    • diagnostics - set to "N/A"
      • +
    • tracking URL - set to "N/A"
      • +
    • original tracking URL - set to "N/A"
      • +
    • resource usage report - all values are -1
      • +
    + + @param appId + {@link ApplicationId} of the application that needs a report + @return application report + @throws YarnException + @throws IOException]]> +     +     + + + + + + Get a report (ApplicationReport) of all Applications in the cluster. +

    + +

    + If the user does not have VIEW_APP access for an application + then the corresponding report will be filtered as described in + {@link #getApplicationReport(ApplicationId)}. +

    + + @return a list of reports for all applications + @throws YarnException + @throws IOException]]> +     +     + + + + + + + Get a report of the given ApplicationAttempt. +

    + +

    + In secure mode, YARN verifies access to the application, queue + etc. before accepting the request. +

    + + @param applicationAttemptId + {@link ApplicationAttemptId} of the application attempt that needs + a report + @return application attempt report + @throws YarnException + @throws ApplicationAttemptNotFoundException if application attempt + not found + @throws IOException]]> +     +     + + + + + + + Get a report of all (ApplicationAttempts) of Application in the cluster. +

    + + @param applicationId + @return a list of reports for all application attempts for specified + application + @throws YarnException + @throws IOException]]> +     +     + + + + + + + Get a report of the given Container. +

    + +

    + In secure mode, YARN verifies access to the application, queue + etc. before accepting the request. +

    + + @param containerId + {@link ContainerId} of the container that needs a report + @return container report + @throws YarnException + @throws ContainerNotFoundException if container not found + @throws IOException]]> +     +     + + + + + + + Get a report of all (Containers) of ApplicationAttempt in the cluster. +

    + + @param applicationAttemptId + @return a list of reports of all containers for specified application + attempt + @throws YarnException + @throws IOException]]> +     +     +     + + + + + + + + + {@code + AMRMClient.createAMRMClientContainerRequest() + } + @return the newly create AMRMClient instance.]]> + + + + + + + + + + + + + + + + RegisterApplicationMasterResponse + @throws YarnException + @throws IOException]]> + + + + + + + + + + + RegisterApplicationMasterResponse + @throws YarnException + @throws IOException]]> + + + + + + + + addContainerRequest are sent to the + ResourceManager. New containers assigned to the master are + retrieved. Status of completed containers and node health updates are also + retrieved. This also doubles up as a heartbeat to the ResourceManager and + must be made periodically. The call may not always return any new + allocations of containers. App should not make concurrent allocate + requests. May cause request loss. + +

    + Note : If the user has not removed container requests that have already + been satisfied, then the re-register may end up sending the entire + container requests to the RM (including matched requests). Which would mean + the RM could end up giving it a lot of new allocated containers. +

    + + @param progressIndicator Indicates progress made by the master + @return the response of the allocate request + @throws YarnException + @throws IOException]]> +     +     + + + + + + + + + + + + + + allocate + @param req Resource request]]> + + + + + + + + + + + + + allocate. + Any previous pending resource change request of the same container will be + removed. + + Application that calls this method is expected to maintain the + Containers that are returned from previous successful + allocations or resource changes. By passing in the existing container and a + target resource capability to this method, the application requests the + ResourceManager to change the existing resource allocation to the target + resource allocation. + + @deprecated use + {@link #requestContainerUpdate(Container, UpdateContainerRequest)} + + @param container The container returned from the last successful resource + allocation or resource change + @param capability The target resource capability of the container]]> + + + + + + + allocate. + Any previous pending update request of the same container will be + removed. + + @param container The container returned from the last successful resource + allocation or update + @param updateContainerRequest The UpdateContainerRequest.]]> + + + + + + + + + + + + + + + + + + + + + + + + ContainerRequests matching the given + parameters. These ContainerRequests should have been added via + addContainerRequest earlier in the lifecycle. For performance, + the AMRMClient may return its internal collection directly without creating + a copy. Users should not perform mutable operations on the return value. + Each collection in the list contains requests with identical + Resource size that fit in the given capability. In a + collection, requests will be returned in the same order as they were added. + + NOTE: This API only matches Container requests that were created by the + client WITHOUT the allocationRequestId being set. + + @return Collection of request matching the parameters]]> + + + + + + + + + ContainerRequests matching the given + parameters. These ContainerRequests should have been added via + addContainerRequest earlier in the lifecycle. For performance, + the AMRMClient may return its internal collection directly without creating + a copy. Users should not perform mutable operations on the return value. + Each collection in the list contains requests with identical + Resource size that fit in the given capability. In a + collection, requests will be returned in the same order as they were added. + specify an ExecutionType. + + NOTE: This API only matches Container requests that were created by the + client WITHOUT the allocationRequestId being set. + + @param priority Priority + @param resourceName Location + @param executionType ExecutionType + @param capability Capability + @return Collection of request matching the parameters]]> + + + + + + + + + + + + + ContainerRequests matching the given + allocationRequestId. These ContainerRequests should have been added via + addContainerRequest earlier in the lifecycle. For performance, + the AMRMClient may return its internal collection directly without creating + a copy. Users should not perform mutable operations on the return value. + + NOTE: This API only matches Container requests that were created by the + client WITH the allocationRequestId being set to a non-default value. + + @param allocationRequestId Allocation Request Id + @return Collection of request matching the parameters]]> + + + + + + + + + + + + + AMRMClient. This cache must + be shared with the {@link NMClient} used to manage containers for the + AMRMClient +

    + If a NM token cache is not set, the {@link NMTokenCache#getSingleton()} + singleton instance will be used. + + @param nmTokenCache the NM token cache to use.]]> + + + + + AMRMClient. This cache must be + shared with the {@link NMClient} used to manage containers for the + AMRMClient. +

    + If a NM token cache is not set, the {@link NMTokenCache#getSingleton()} + singleton instance will be used. + + @return the NM token cache.]]> + + + + + + + + + + + + + + + + + + + + + + + + check to return true for each 1000 ms. + See also {@link #waitFor(java.util.function.Supplier, int)} + and {@link #waitFor(java.util.function.Supplier, int, int)} + @param check the condition for which it should wait]]> + + + + + + + + check to return true for each + checkEveryMillis ms. + See also {@link #waitFor(java.util.function.Supplier, int, int)} + @param check user defined checker + @param checkEveryMillis interval to call check]]> + + + + + + + + + check to return true for each + checkEveryMillis ms. In the main loop, this method will log + the message "waiting in main loop" for each logInterval times + iteration to confirm the thread is alive. + @param check user defined checker + @param checkEveryMillis interval to call check + @param logInterval interval to log for each]]> + + + + + + + + + + + + + + + + + + + + + + + + + + Start an allocated container.

    + +

    The ApplicationMaster or other applications that use the + client must provide the details of the allocated container, including the + Id, the assigned node's Id and the token via {@link Container}. In + addition, the AM needs to provide the {@link ContainerLaunchContext} as + well.

    + + @param container the allocated container + @param containerLaunchContext the context information needed by the + NodeManager to launch the + container + @return a map between the auxiliary service names and their outputs + @throws YarnException YarnException. + @throws IOException IOException.]]> +     +     + + + + + + Increase the resource of a container.

    + +

    The ApplicationMaster or other applications that use the + client must provide the details of the container, including the Id and + the target resource encapsulated in the updated container token via + {@link Container}. +

    + + @param container the container with updated token. + + @throws YarnException YarnException. + @throws IOException IOException.]]> +     +     + + + + + + Update the resources of a container.

    + +

    The ApplicationMaster or other applications that use the + client must provide the details of the container, including the Id and + the target resource encapsulated in the updated container token via + {@link Container}. +

    + + @param container the container with updated token. + + @throws YarnException YarnException. + @throws IOException IOException.]]> +     +     + + + + + + + Stop an started container.

    + + @param containerId the Id of the started container + @param nodeId the Id of the NodeManager + + @throws YarnException YarnException. + @throws IOException IOException.]]> +     +     + + + + + + + Query the status of a container.

    + + @param containerId the Id of the started container + @param nodeId the Id of the NodeManager + + @return the status of a container. + + @throws YarnException YarnException. + @throws IOException IOException.]]> +     +     + + + + + + + + Re-Initialize the Container.

    + + @param containerId the Id of the container to Re-Initialize. + @param containerLaunchContex the updated ContainerLaunchContext. + @param autoCommit commit re-initialization automatically ? + + @throws YarnException YarnException. + @throws IOException IOException.]]> +     +     + + + + + + Restart the specified container.

    + + @param containerId the Id of the container to restart. + + @throws YarnException YarnException. + @throws IOException IOException.]]> +     +     + + + + + + Rollback last reInitialization of the specified container.

    + + @param containerId the Id of the container to restart. + + @throws YarnException YarnException. + @throws IOException IOException.]]> +     +     + + + + + + Commit last reInitialization of the specified container.

    + + @param containerId the Id of the container to commit reInitialize. + + @throws YarnException YarnException. + @throws IOException IOException.]]> +     +     + + + + Set whether the containers that are started by this client, and are + still running should be stopped when the client stops. By default, the + feature should be enabled.

    However, containers will be stopped only + when service is stopped. i.e. after {@link NMClient#stop()}. + + @param enabled whether the feature is enabled or not]]> +     +     + + + + NMClient. This cache must be + shared with the {@link AMRMClient} that requested the containers managed + by this NMClient +

    + If a NM token cache is not set, the {@link NMTokenCache#getSingleton()} + singleton instance will be used. + + @param nmTokenCache the NM token cache to use.]]> + + + + + NMClient. This cache must be + shared with the {@link AMRMClient} that requested the containers managed + by this NMClient +

    + If a NM token cache is not set, the {@link NMTokenCache#getSingleton()} + singleton instance will be used. + + @return the NM token cache]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + By default YARN client libraries {@link AMRMClient} and {@link NMClient} use + {@link #getSingleton()} instance of the cache. +

      +
    • + Using the singleton instance of the cache is appropriate when running a + single ApplicationMaster in the same JVM. +
      • +
    • + When using the singleton, users don't need to do anything special, + {@link AMRMClient} and {@link NMClient} are already set up to use the + default singleton {@link NMTokenCache} +
      • +
    + If running multiple Application Masters in the same JVM, a different cache + instance should be used for each Application Master. +
      +
    • + If using the {@link AMRMClient} and the {@link NMClient}, setting up + and using an instance cache is as follows: + 
      +   NMTokenCache nmTokenCache = new NMTokenCache();
+   AMRMClient rmClient = AMRMClient.createAMRMClient();
+   NMClient nmClient = NMClient.createNMClient();
+   nmClient.setNMTokenCache(nmTokenCache);
+   ...
+
      +
      • +
    • + If using the {@link AMRMClientAsync} and the {@link NMClientAsync}, + setting up and using an instance cache is as follows: + 
      +   NMTokenCache nmTokenCache = new NMTokenCache();
+   AMRMClient rmClient = AMRMClient.createAMRMClient();
+   NMClient nmClient = NMClient.createNMClient();
+   nmClient.setNMTokenCache(nmTokenCache);
+   AMRMClientAsync rmClientAsync = new AMRMClientAsync(rmClient, 1000, [AMRM_CALLBACK]);
+   NMClientAsync nmClientAsync = new NMClientAsync("nmClient", nmClient, [NM_CALLBACK]);
+   ...
+
      +
      • +
    • + If using {@link ApplicationMasterProtocol} and + {@link ContainerManagementProtocol} directly, setting up and using an + instance cache is as follows: + 
      +   NMTokenCache nmTokenCache = new NMTokenCache();
+   ...
+   ApplicationMasterProtocol amPro = ClientRMProxy.createRMProxy(conf, ApplicationMasterProtocol.class);
+   ...
+   AllocateRequest allocateRequest = ...
+   ...
+   AllocateResponse allocateResponse = rmClient.allocate(allocateRequest);
+   for (NMToken token : allocateResponse.getNMTokens()) {
+     nmTokenCache.setToken(token.getNodeId().toString(), token.getToken());
+   }
+   ...
+   ContainerManagementProtocolProxy nmPro = ContainerManagementProtocolProxy(conf, nmTokenCache);
+   ...
+   nmPro.startContainer(container, containerContext);
+   ...
+
      +
      • +
    + It is also possible to mix the usage of a client ({@code AMRMClient} or + {@code NMClient}, or the async versions of them) with a protocol proxy + ({@code ContainerManagementProtocolProxy} or + {@code ApplicationMasterProtocol}).]]> +     +     + + + + + + + + + + + + + + The method to claim a resource with the SharedCacheManager. + The client uses a checksum to identify the resource and an + {@link ApplicationId} to identify which application will be using the + resource. +

    + +

    + The SharedCacheManager responds with whether or not the + resource exists in the cache. If the resource exists, a URL to + the resource in the shared cache is returned. If the resource does not + exist, null is returned instead. +

    + +

    + Once a URL has been returned for a resource, that URL is safe to use for + the lifetime of the application that corresponds to the provided + ApplicationId. +

    + + @param applicationId ApplicationId of the application using the resource + @param resourceKey the key (i.e. checksum) that identifies the resource + @return URL to the resource, or null if it does not exist]]> +     +     + + + + + + + The method to release a resource with the SharedCacheManager. + This method is called once an application is no longer using a claimed + resource in the shared cache. The client uses a checksum to identify the + resource and an {@link ApplicationId} to identify which application is + releasing the resource. +

    + +

    + Note: This method is an optimization and the client is not required to call + it for correctness. +

    + + @param applicationId ApplicationId of the application releasing the + resource + @param resourceKey the key (i.e. checksum) that identifies the resource]]> +     +     + + + + + + + + + + +     + + + + + + + + + + + + + + + + Obtain a {@link YarnClientApplication} for a new application, + which in turn contains the {@link ApplicationSubmissionContext} and + {@link org.apache.hadoop.yarn.api.protocolrecords.GetNewApplicationResponse} + objects. +

    + + @return {@link YarnClientApplication} built for a new application + @throws YarnException + @throws IOException]]> +     +     + + + + + + + Submit a new application to YARN. It is a blocking call - it + will not return {@link ApplicationId} until the submitted application is + submitted successfully and accepted by the ResourceManager. +

    + +

    + Users should provide an {@link ApplicationId} as part of the parameter + {@link ApplicationSubmissionContext} when submitting a new application, + otherwise it will throw the {@link ApplicationIdNotProvidedException}. +

    + +

    This internally calls {@link ApplicationClientProtocol#submitApplication + (SubmitApplicationRequest)}, and after that, it internally invokes + {@link ApplicationClientProtocol#getApplicationReport + (GetApplicationReportRequest)} and waits till it can make sure that the + application gets properly submitted. If RM fails over or RM restart + happens before ResourceManager saves the application's state, + {@link ApplicationClientProtocol + #getApplicationReport(GetApplicationReportRequest)} will throw + the {@link ApplicationNotFoundException}. This API automatically resubmits + the application with the same {@link ApplicationSubmissionContext} when it + catches the {@link ApplicationNotFoundException}

    + + @param appContext + {@link ApplicationSubmissionContext} containing all the details + needed to submit a new application + @return {@link ApplicationId} of the accepted application + @throws YarnException + @throws IOException + @see #createApplication()]]> +     +     + + + + + + + Fail an application attempt identified by given ID. +

    + + @param applicationAttemptId + {@link ApplicationAttemptId} of the attempt to fail. + @throws YarnException + in case of errors or if YARN rejects the request due to + access-control restrictions. + @throws IOException + @see #getQueueAclsInfo()]]> +     +     + + + + + + + Kill an application identified by given ID. +

    + + @param applicationId + {@link ApplicationId} of the application that needs to be killed + @throws YarnException + in case of errors or if YARN rejects the request due to + access-control restrictions. + @throws IOException + @see #getQueueAclsInfo()]]> +     +     + + + + + + + + Kill an application identified by given ID. +

    + @param applicationId {@link ApplicationId} of the application that needs to + be killed + @param diagnostics for killing an application. + @throws YarnException in case of errors or if YARN rejects the request due + to access-control restrictions. + @throws IOException]]> +     +     + + + + + + + Get a report of the given Application. +

    + +

    + In secure mode, YARN verifies access to the application, queue + etc. before accepting the request. +

    + +

    + If the user does not have VIEW_APP access then the following + fields in the report will be set to stubbed values: +

      +
    • host - set to "N/A"
      • +
    • RPC port - set to -1
      • +
    • client token - set to "N/A"
      • +
    • diagnostics - set to "N/A"
      • +
    • tracking URL - set to "N/A"
      • +
    • original tracking URL - set to "N/A"
      • +
    • resource usage report - all values are -1
      • +
    + + @param appId + {@link ApplicationId} of the application that needs a report + @return application report + @throws YarnException + @throws IOException]]> +     +     + + + + + + + The AMRM token is required for AM to RM scheduling operations. For + managed Application Masters YARN takes care of injecting it. For unmanaged + Applications Masters, the token must be obtained via this method and set + in the {@link org.apache.hadoop.security.UserGroupInformation} of the + current user. +

    + The AMRM token will be returned only if all the following conditions are + met: +

      +
    • the requester is the owner of the ApplicationMaster
      • +
    • the application master is an unmanaged ApplicationMaster
      • +
    • the application master is in ACCEPTED state
      • +
    + Else this method returns NULL. + + @param appId {@link ApplicationId} of the application to get the AMRM token + @return the AMRM token if available + @throws YarnException + @throws IOException]]> +     +     + + + + + + Get a report (ApplicationReport) of all Applications in the cluster. +

    + +

    + If the user does not have VIEW_APP access for an application + then the corresponding report will be filtered as described in + {@link #getApplicationReport(ApplicationId)}. +

    + + @return a list of reports of all running applications + @throws YarnException + @throws IOException]]> +     +     + + + + + + + Get a report (ApplicationReport) of Applications + matching the given application types in the cluster. +

    + +

    + If the user does not have VIEW_APP access for an application + then the corresponding report will be filtered as described in + {@link #getApplicationReport(ApplicationId)}. +

    + + @param applicationTypes set of application types you are interested in + @return a list of reports of applications + @throws YarnException + @throws IOException]]> +     +     + + + + + + + Get a report (ApplicationReport) of Applications matching the given + application states in the cluster. +

    + +

    + If the user does not have VIEW_APP access for an application + then the corresponding report will be filtered as described in + {@link #getApplicationReport(ApplicationId)}. +

    + + @param applicationStates set of application states you are interested in + @return a list of reports of applications + @throws YarnException + @throws IOException]]> +     +     + + + + + + + + Get a report (ApplicationReport) of Applications matching the given + application types and application states in the cluster. +

    + +

    + If the user does not have VIEW_APP access for an application + then the corresponding report will be filtered as described in + {@link #getApplicationReport(ApplicationId)}. +

    + + @param applicationTypes set of application types you are interested in + @param applicationStates set of application states you are interested in + @return a list of reports of applications + @throws YarnException + @throws IOException]]> +     +     + + + + + + + + + Get a report (ApplicationReport) of Applications matching the given + application types, application states and application tags in the cluster. +

    + +

    + If the user does not have VIEW_APP access for an application + then the corresponding report will be filtered as described in + {@link #getApplicationReport(ApplicationId)}. +

    + + @param applicationTypes set of application types you are interested in + @param applicationStates set of application states you are interested in + @param applicationTags set of application tags you are interested in + @return a list of reports of applications + @throws YarnException + @throws IOException]]> +     +     + + + + + + + + + + Get a report (ApplicationReport) of Applications matching the given users, + queues, application types and application states in the cluster. If any of + the params is set to null, it is not used when filtering. +

    + +

    + If the user does not have VIEW_APP access for an application + then the corresponding report will be filtered as described in + {@link #getApplicationReport(ApplicationId)}. +

    + + @param queues set of queues you are interested in + @param users set of users you are interested in + @param applicationTypes set of application types you are interested in + @param applicationStates set of application states you are interested in + @return a list of reports of applications + @throws YarnException + @throws IOException]]> +     +     + + + + + + + Get a list of ApplicationReports that match the given + {@link GetApplicationsRequest}. +

    + +

    + If the user does not have VIEW_APP access for an application + then the corresponding report will be filtered as described in + {@link #getApplicationReport(ApplicationId)}. +

    + + @param request the request object to get the list of applications. + @return The list of ApplicationReports that match the request + @throws YarnException Exception specific to YARN. + @throws IOException Exception mostly related to connection errors.]]> +     +     + + + + + + Get metrics ({@link YarnClusterMetrics}) about the cluster. +

    + + @return cluster metrics + @throws YarnException + @throws IOException]]> +     +     + + + + + + + Get a report of nodes ({@link NodeReport}) in the cluster. +

    + + @param states The {@link NodeState}s to filter on. If no filter states are + given, nodes in all states will be returned. + @return A list of node reports + @throws YarnException + @throws IOException]]> +     +     + + + + + + + Get a delegation token so as to be able to talk to YARN using those tokens. + + @param renewer + Address of the renewer who can renew these tokens when needed by + securely talking to YARN. + @return a delegation token ({@link Token}) that can be used to + talk to YARN + @throws YarnException + @throws IOException]]> + + + + + + + + + Get information ({@link QueueInfo}) about a given queue. +

    + + @param queueName + Name of the queue whose information is needed + @return queue information + @throws YarnException + in case of errors or if YARN rejects the request due to + access-control restrictions. + @throws IOException]]> +     +     + + + + + + Get information ({@link QueueInfo}) about all queues, recursively if there + is a hierarchy +

    + + @return a list of queue-information for all queues + @throws YarnException + @throws IOException]]> +     +     + + + + + + Get information ({@link QueueInfo}) about top level queues. +

    + + @return a list of queue-information for all the top-level queues + @throws YarnException + @throws IOException]]> +     +     + + + + + + + Get information ({@link QueueInfo}) about all the immediate children queues + of the given queue +

    + + @param parent + Name of the queue whose child-queues' information is needed + @return a list of queue-information for all queues who are direct children + of the given parent queue. + @throws YarnException + @throws IOException]]> +     +     + + + + + + Get information about acls for current user on all the + existing queues. +

    + + @return a list of queue acls ({@link QueueUserACLInfo}) for + current user + @throws YarnException + @throws IOException]]> +     +     + + + + + + + Get a report of the given ApplicationAttempt. +

    + +

    + In secure mode, YARN verifies access to the application, queue + etc. before accepting the request. +

    + + @param applicationAttemptId + {@link ApplicationAttemptId} of the application attempt that needs + a report + @return application attempt report + @throws YarnException + @throws ApplicationAttemptNotFoundException if application attempt + not found + @throws IOException]]> +     +     + + + + + + + Get a report of all (ApplicationAttempts) of Application in the cluster. +

    + + @param applicationId application id of the app + @return a list of reports for all application attempts for specified + application. + @throws YarnException + @throws IOException]]> +     +     + + + + + + + Get a report of the given Container. +

    + +

    + In secure mode, YARN verifies access to the application, queue + etc. before accepting the request. +

    + + @param containerId + {@link ContainerId} of the container that needs a report + @return container report + @throws YarnException + @throws ContainerNotFoundException if container not found. + @throws IOException]]> +     +     + + + + + + + Get a report of all (Containers) of ApplicationAttempt in the cluster. +

    + + @param applicationAttemptId application attempt id + @return a list of reports of all containers for specified application + attempts + @throws YarnException + @throws IOException]]> +     +     + + + + + + + + Attempts to move the given application to the given queue. +

    + + @param appId + Application to move. + @param queue + Queue to place it in to. + @throws YarnException + @throws IOException]]> +     +     + + + + + + Obtain a {@link GetNewReservationResponse} for a new reservation, + which contains the {@link ReservationId} object. +

    + + @return The {@link GetNewReservationResponse} containing a new + {@link ReservationId} object. + @throws YarnException if reservation cannot be created. + @throws IOException if reservation cannot be created.]]> +     +     + + + + + + + The interface used by clients to submit a new reservation to the + {@code ResourceManager}. +

    + +

    + The client packages all details of its request in a + {@link ReservationSubmissionRequest} object. This contains information + about the amount of capacity, temporal constraints, and gang needs. + Furthermore, the reservation might be composed of multiple stages, with + ordering dependencies among them. +

    + +

    + In order to respond, a new admission control component in the + {@code ResourceManager} performs an analysis of the resources that have + been committed over the period of time the user is requesting, verify that + the user requests can be fulfilled, and that it respect a sharing policy + (e.g., {@code CapacityOverTimePolicy}). Once it has positively determined + that the ReservationRequest is satisfiable the {@code ResourceManager} + answers with a {@link ReservationSubmissionResponse} that includes a + {@link ReservationId}. Upon failure to find a valid allocation the response + is an exception with the message detailing the reason of failure. +

    + +

    + The semantics guarantees that the {@link ReservationId} returned, + corresponds to a valid reservation existing in the time-range request by + the user. The amount of capacity dedicated to such reservation can vary + overtime, depending of the allocation that has been determined. But it is + guaranteed to satisfy all the constraint expressed by the user in the + {@link ReservationDefinition} +

    + + @param request request to submit a new Reservation + @return response contains the {@link ReservationId} on accepting the + submission + @throws YarnException if the reservation cannot be created successfully + @throws IOException]]> +     +     + + + + + + + The interface used by clients to update an existing Reservation. This is + referred to as a re-negotiation process, in which a user that has + previously submitted a Reservation. +

    + +

    + The allocation is attempted by virtually substituting all previous + allocations related to this Reservation with new ones, that satisfy the new + {@link ReservationDefinition}. Upon success the previous allocation is + atomically substituted by the new one, and on failure (i.e., if the system + cannot find a valid allocation for the updated request), the previous + allocation remains valid. +

    + + @param request to update an existing Reservation (the + {@link ReservationUpdateRequest} should refer to an existing valid + {@link ReservationId}) + @return response empty on successfully updating the existing reservation + @throws YarnException if the request is invalid or reservation cannot be + updated successfully + @throws IOException]]> +     +     + + + + + + + The interface used by clients to remove an existing Reservation. +

    + + @param request to remove an existing Reservation (the + {@link ReservationDeleteRequest} should refer to an existing valid + {@link ReservationId}) + @return response empty on successfully deleting the existing reservation + @throws YarnException if the request is invalid or reservation cannot be + deleted successfully + @throws IOException]]> +     +     + + + + + + + The interface used by clients to get the list of reservations in a plan. + The reservationId will be used to search for reservations to list if it is + provided. Otherwise, it will select active reservations within the + startTime and endTime (inclusive). +

    + + @param request to list reservations in a plan. Contains fields to select + String queue, ReservationId reservationId, long startTime, + long endTime, and a bool includeReservationAllocations. + + queue: Required. Cannot be null or empty. Refers to the + reservable queue in the scheduler that was selected when + creating a reservation submission + {@link ReservationSubmissionRequest}. + + reservationId: Optional. If provided, other fields will + be ignored. + + startTime: Optional. If provided, only reservations that + end after the startTime will be selected. This defaults + to 0 if an invalid number is used. + + endTime: Optional. If provided, only reservations that + start on or before endTime will be selected. This defaults + to Long.MAX_VALUE if an invalid number is used. + + includeReservationAllocations: Optional. Flag that + determines whether the entire reservation allocations are + to be returned. Reservation allocations are subject to + change in the event of re-planning as described by + {@link ReservationDefinition}. + + @return response that contains information about reservations that are + being searched for. + @throws YarnException if the request is invalid + @throws IOException if the request failed otherwise]]> +     +     + + + + + + The interface used by client to get node to labels mappings in existing cluster +

    + + @return node to labels mappings + @throws YarnException + @throws IOException]]> +     +     + + + + + + The interface used by client to get labels to nodes mapping + in existing cluster +

    + + @return node to labels mappings + @throws YarnException + @throws IOException]]> +     +     + + + + + + + The interface used by client to get labels to nodes mapping + for specified labels in existing cluster +

    + + @param labels labels for which labels to nodes mapping has to be retrieved + @return labels to nodes mappings for specific labels + @throws YarnException + @throws IOException]]> +     +     + + + + + + The interface used by client to get node labels in the cluster +

    + + @return cluster node labels collection + @throws YarnException when there is a failure in + {@link ApplicationClientProtocol} + @throws IOException when there is a failure in + {@link ApplicationClientProtocol}]]> +     +     + + + + + + + + The interface used by client to set priority of an application +

    + @param applicationId + @param priority + @return updated priority of an application. + @throws YarnException + @throws IOException]]> +     +     + + + + + + + + Signal a container identified by given ID. +

    + + @param containerId + {@link ContainerId} of the container that needs to be signaled + @param command the signal container command + @throws YarnException + @throws IOException]]> +     +     + + + + + + + + + + + Get the resource profiles available in the RM. +

    + @return a Map of the resource profile names to their capabilities + @throws YARNFeatureNotEnabledException if resource-profile is disabled + @throws YarnException if any error happens inside YARN + @throws IOException in case of other errors]]> +     +     + + + + + + + Get the details of a specific resource profile from the RM. +

    + @param profile the profile name + @return resource profile name with its capabilities + @throws YARNFeatureNotEnabledException if resource-profile is disabled + @throws YarnException if any error happens inside YARN + @throws IOException in case of other others]]> +     +     + + + + + + Get available resource types supported by RM. +

    + @return list of supported resource types with detailed information + @throws YarnException if any issue happens inside YARN + @throws IOException in case of other others]]> +     +     + + + + + + The interface used by client to get node attributes in the cluster. +

    + + @return cluster node attributes collection + @throws YarnException when there is a failure in + {@link ApplicationClientProtocol} + @throws IOException when there is a failure in + {@link ApplicationClientProtocol}]]> +     +     + + + + + + + The interface used by client to get mapping of AttributeKey to associated + NodeToAttributeValue list for specified node attributeKeys in the cluster. +

    + + @param attributes AttributeKeys for which associated NodeToAttributeValue + mapping value has to be retrieved. If empty or null is set then + will return mapping for all attributeKeys in the cluster + @return mapping of AttributeKey to List of associated + NodeToAttributeValue's. + @throws YarnException + @throws IOException]]> +     +     + + + + + + + The interface used by client to get all node to attribute mapping in + existing cluster. +

    + + @param hostNames HostNames for which host to attributes mapping has to + be retrived.If empty or null is set then will return + all nodes to attributes mapping in cluster. + @return Node to attribute mappings + @throws YarnException + @throws IOException]]> +     +     +     + + + + + + + + + + + +     + + + + + + + + + + + + + + + + Create a new instance of AMRMClientAsync.

    + + @param intervalMs heartbeat interval in milliseconds between AM and RM + @param callbackHandler callback handler that processes responses from + the ResourceManager]]> +     +     + + + + + + Create a new instance of AMRMClientAsync.

    + + @param client the AMRMClient instance + @param intervalMs heartbeat interval in milliseconds between AM and RM + @param callbackHandler callback handler that processes responses from + the ResourceManager]]> +     +     + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + RegisterApplicationMasterResponse + @throws YarnException + @throws IOException]]> + + + + + + + + + + + + + + + + allocate + @param req Resource request]]> + + + + + + + + + + + + + allocate. + Any previous pending resource change request of the same container will be + removed. + + Application that calls this method is expected to maintain the + Containers that are returned from previous successful + allocations or resource changes. By passing in the existing container and a + target resource capability to this method, the application requests the + ResourceManager to change the existing resource allocation to the target + resource allocation. + + @deprecated use + {@link #requestContainerUpdate(Container, UpdateContainerRequest)} + + @param container The container returned from the last successful resource + allocation or resource change + @param capability The target resource capability of the container]]> + + + + + + + allocate. + Any previous pending update request of the same container will be + removed. + + @param container The container returned from the last successful resource + allocation or update + @param updateContainerRequest The UpdateContainerRequest.]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + check to return true for each 1000 ms. + See also {@link #waitFor(java.util.function.Supplier, int)} + and {@link #waitFor(java.util.function.Supplier, int, int)} + @param check the condition for which it should wait]]> + + + + + + + + check to return true for each + checkEveryMillis ms. + See also {@link #waitFor(java.util.function.Supplier, int, int)} + @param check user defined checker + @param checkEveryMillis interval to call check]]> + + + + + + + + + check to return true for each + checkEveryMillis ms. In the main loop, this method will log + the message "waiting in main loop" for each logInterval times + iteration to confirm the thread is alive. + @param check user defined checker + @param checkEveryMillis interval to call check + @param logInterval interval to log for each]]> + + + + + + + + + + AMRMClientAsync handles communication with the ResourceManager + and provides asynchronous updates on events such as container allocations and + completions. It contains a thread that sends periodic heartbeats to the + ResourceManager. + + It should be used by implementing a CallbackHandler: + 
    + {@code
+ class MyCallbackHandler extends AMRMClientAsync.AbstractCallbackHandler {
+   public void onContainersAllocated(List containers) {
+     [run tasks on the containers]
+   }
+
+   public void onContainersUpdated(List containers) {
+     [determine if resource allocation of containers have been increased in
+      the ResourceManager, and if so, inform the NodeManagers to increase the
+      resource monitor/enforcement on the containers]
+   }
+
+   public void onContainersCompleted(List statuses) {
+     [update progress, check whether app is done]
+   }
+   
+   public void onNodesUpdated(List updated) {}
+   
+   public void onReboot() {}
+ }
+ }
+
    + + The client's lifecycle should be managed similarly to the following: + + 
    + {@code
+ AMRMClientAsync asyncClient = 
+     createAMRMClientAsync(appAttId, 1000, new MyCallbackhandler());
+ asyncClient.init(conf);
+ asyncClient.start();
+ RegisterApplicationMasterResponse response = asyncClient
+    .registerApplicationMaster(appMasterHostname, appMasterRpcPort,
+       appMasterTrackingUrl);
+ asyncClient.addContainerRequest(containerRequest);
+ [... wait for application to complete]
+ asyncClient.unregisterApplicationMaster(status, appMsg, trackingUrl);
+ asyncClient.stop();
+ }
+
    ]]> +     +     + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Update the resources of a container.

    + +

    The ApplicationMaster or other applications that use the + client must provide the details of the container, including the Id and + the target resource encapsulated in the updated container token via + {@link Container}. +

    + + @param container the container with updated token.]]> +     +     + + + + + + Re-Initialize the Container.

    + + @param containerId the Id of the container to Re-Initialize. + @param containerLaunchContex the updated ContainerLaunchContext. + @param autoCommit commit re-initialization automatically ?]]> +     +     + + + + Restart the specified container.

    + + @param containerId the Id of the container to restart.]]> +     +     + + + + Rollback last reInitialization of the specified container.

    + + @param containerId the Id of the container to restart.]]> +     +     + + + + Commit last reInitialization of the specified container.

    + + @param containerId the Id of the container to commit reInitialize.]]> +     +     + + + + + + + + + + + + + + + + + + + + + + + + NMClientAsync handles communication with all the NodeManagers + and provides asynchronous updates on getting responses from them. It + maintains a thread pool to communicate with individual NMs where a number of + worker threads process requests to NMs by using {@link NMClientImpl}. The max + size of the thread pool is configurable through + {@link YarnConfiguration#NM_CLIENT_ASYNC_THREAD_POOL_MAX_SIZE}. + + It should be used in conjunction with a CallbackHandler. For example + + 
    + {@code
+ class MyCallbackHandler extends NMClientAsync.AbstractCallbackHandler {
+   public void onContainerStarted(ContainerId containerId,
+       Map allServiceResponse) {
+     [post process after the container is started, process the response]
+   }
+
+   public void onContainerResourceIncreased(ContainerId containerId,
+       Resource resource) {
+     [post process after the container resource is increased]
+   }
+
+   public void onContainerStatusReceived(ContainerId containerId,
+       ContainerStatus containerStatus) {
+     [make use of the status of the container]
+   }
+
+   public void onContainerStopped(ContainerId containerId) {
+     [post process after the container is stopped]
+   }
+
+   public void onStartContainerError(
+       ContainerId containerId, Throwable t) {
+     [handle the raised exception]
+   }
+
+   public void onGetContainerStatusError(
+       ContainerId containerId, Throwable t) {
+     [handle the raised exception]
+   }
+
+   public void onStopContainerError(
+       ContainerId containerId, Throwable t) {
+     [handle the raised exception]
+   }
+ }
+ }
+
    + + The client's life-cycle should be managed like the following: + + 
    + {@code
+ NMClientAsync asyncClient = 
+     NMClientAsync.createNMClientAsync(new MyCallbackhandler());
+ asyncClient.init(conf);
+ asyncClient.start();
+ asyncClient.startContainer(container, containerLaunchContext);
+ [... wait for container being started]
+ asyncClient.getContainerStatus(container.getId(), container.getNodeId(),
+     container.getContainerToken());
+ [... handle the status in the callback instance]
+ asyncClient.stopContainer(container.getId(), container.getNodeId(),
+     container.getContainerToken());
+ [... wait for container being stopped]
+ asyncClient.stop();
+ }
+
    ]]> +     +     + +     + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +     diff --git a/hadoop-yarn-project/hadoop-yarn/dev-support/jdiff/Apache_Hadoop_YARN_Client_3.3.4.xml b/hadoop-yarn-project/hadoop-yarn/dev-support/jdiff/Apache_Hadoop_YARN_Client_3.3.4.xml new file mode 100644 index 0000000000000..aa23d55a710ec --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/dev-support/jdiff/Apache_Hadoop_YARN_Client_3.3.4.xml @@ -0,0 +1,3067 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + In secure mode, YARN verifies access to the application, queue + etc. before accepting the request. +

    + If the user does not have VIEW_APP access then the following + fields in the report will be set to stubbed values: +

      +
    • host - set to "N/A"
      • +
    • RPC port - set to -1
      • +
    • client token - set to "N/A"
      • +
    • diagnostics - set to "N/A"
      • +
    • tracking URL - set to "N/A"
      • +
    • original tracking URL - set to "N/A"
      • +
    • resource usage report - all values are -1
      • +
    + + @param appId + {@link ApplicationId} of the application that needs a report + @return application report + @throws YarnException + @throws IOException]]> +     +     + + + + + + Get a report (ApplicationReport) of all Applications in the cluster. +

    + +

    + If the user does not have VIEW_APP access for an application + then the corresponding report will be filtered as described in + {@link #getApplicationReport(ApplicationId)}. +

    + + @return a list of reports for all applications + @throws YarnException + @throws IOException]]> +     +     + + + + + + + Get a report of the given ApplicationAttempt. +

    + +

    + In secure mode, YARN verifies access to the application, queue + etc. before accepting the request. +

    + + @param applicationAttemptId + {@link ApplicationAttemptId} of the application attempt that needs + a report + @return application attempt report + @throws YarnException + @throws ApplicationAttemptNotFoundException if application attempt + not found + @throws IOException]]> +     +     + + + + + + + Get a report of all (ApplicationAttempts) of Application in the cluster. +

    + + @param applicationId + @return a list of reports for all application attempts for specified + application + @throws YarnException + @throws IOException]]> +     +     + + + + + + + Get a report of the given Container. +

    + +

    + In secure mode, YARN verifies access to the application, queue + etc. before accepting the request. +

    + + @param containerId + {@link ContainerId} of the container that needs a report + @return container report + @throws YarnException + @throws ContainerNotFoundException if container not found + @throws IOException]]> +     +     + + + + + + + Get a report of all (Containers) of ApplicationAttempt in the cluster. +

    + + @param applicationAttemptId + @return a list of reports of all containers for specified application + attempt + @throws YarnException + @throws IOException]]> +     +     +     + + + + + + + + + {@code + AMRMClient.createAMRMClientContainerRequest() + } + @return the newly create AMRMClient instance.]]> + + + + + + + + + + + + + + + + RegisterApplicationMasterResponse + @throws YarnException + @throws IOException]]> + + + + + + + + + + + RegisterApplicationMasterResponse + @throws YarnException + @throws IOException]]> + + + + + + + + addContainerRequest are sent to the + ResourceManager. New containers assigned to the master are + retrieved. Status of completed containers and node health updates are also + retrieved. This also doubles up as a heartbeat to the ResourceManager and + must be made periodically. The call may not always return any new + allocations of containers. App should not make concurrent allocate + requests. May cause request loss. + +

    + Note : If the user has not removed container requests that have already + been satisfied, then the re-register may end up sending the entire + container requests to the RM (including matched requests). Which would mean + the RM could end up giving it a lot of new allocated containers. +

    + + @param progressIndicator Indicates progress made by the master + @return the response of the allocate request + @throws YarnException + @throws IOException]]> +     +     + + + + + + + + + + + + + + allocate + @param req Resource request]]> + + + + + + + + + + + + + allocate. + Any previous pending resource change request of the same container will be + removed. + + Application that calls this method is expected to maintain the + Containers that are returned from previous successful + allocations or resource changes. By passing in the existing container and a + target resource capability to this method, the application requests the + ResourceManager to change the existing resource allocation to the target + resource allocation. + + @deprecated use + {@link #requestContainerUpdate(Container, UpdateContainerRequest)} + + @param container The container returned from the last successful resource + allocation or resource change + @param capability The target resource capability of the container]]> + + + + + + + allocate. + Any previous pending update request of the same container will be + removed. + + @param container The container returned from the last successful resource + allocation or update + @param updateContainerRequest The UpdateContainerRequest.]]> + + + + + + + + + + + + + + + + + + + + + + + + ContainerRequests matching the given + parameters. These ContainerRequests should have been added via + addContainerRequest earlier in the lifecycle. For performance, + the AMRMClient may return its internal collection directly without creating + a copy. Users should not perform mutable operations on the return value. + Each collection in the list contains requests with identical + Resource size that fit in the given capability. In a + collection, requests will be returned in the same order as they were added. + + NOTE: This API only matches Container requests that were created by the + client WITHOUT the allocationRequestId being set. + + @return Collection of request matching the parameters]]> + + + + + + + + + ContainerRequests matching the given + parameters. These ContainerRequests should have been added via + addContainerRequest earlier in the lifecycle. For performance, + the AMRMClient may return its internal collection directly without creating + a copy. Users should not perform mutable operations on the return value. + Each collection in the list contains requests with identical + Resource size that fit in the given capability. In a + collection, requests will be returned in the same order as they were added. + specify an ExecutionType. + + NOTE: This API only matches Container requests that were created by the + client WITHOUT the allocationRequestId being set. + + @param priority Priority + @param resourceName Location + @param executionType ExecutionType + @param capability Capability + @return Collection of request matching the parameters]]> + + + + + + + + + + + + + ContainerRequests matching the given + allocationRequestId. These ContainerRequests should have been added via + addContainerRequest earlier in the lifecycle. For performance, + the AMRMClient may return its internal collection directly without creating + a copy. Users should not perform mutable operations on the return value. + + NOTE: This API only matches Container requests that were created by the + client WITH the allocationRequestId being set to a non-default value. + + @param allocationRequestId Allocation Request Id + @return Collection of request matching the parameters]]> + + + + + + + + + + + + + AMRMClient. This cache must + be shared with the {@link NMClient} used to manage containers for the + AMRMClient +

    + If a NM token cache is not set, the {@link NMTokenCache#getSingleton()} + singleton instance will be used. + + @param nmTokenCache the NM token cache to use.]]> + + + + + AMRMClient. This cache must be + shared with the {@link NMClient} used to manage containers for the + AMRMClient. +

    + If a NM token cache is not set, the {@link NMTokenCache#getSingleton()} + singleton instance will be used. + + @return the NM token cache.]]> + + + + + + + + + + + + + + + + + + + + + + + + check to return true for each 1000 ms. + See also {@link #waitFor(java.util.function.Supplier, int)} + and {@link #waitFor(java.util.function.Supplier, int, int)} + @param check the condition for which it should wait]]> + + + + + + + + check to return true for each + checkEveryMillis ms. + See also {@link #waitFor(java.util.function.Supplier, int, int)} + @param check user defined checker + @param checkEveryMillis interval to call check]]> + + + + + + + + + check to return true for each + checkEveryMillis ms. In the main loop, this method will log + the message "waiting in main loop" for each logInterval times + iteration to confirm the thread is alive. + @param check user defined checker + @param checkEveryMillis interval to call check + @param logInterval interval to log for each]]> + + + + + + + + + + + + + + + + + + + + + + + + + + Start an allocated container.

    + +

    The ApplicationMaster or other applications that use the + client must provide the details of the allocated container, including the + Id, the assigned node's Id and the token via {@link Container}. In + addition, the AM needs to provide the {@link ContainerLaunchContext} as + well.

    + + @param container the allocated container + @param containerLaunchContext the context information needed by the + NodeManager to launch the + container + @return a map between the auxiliary service names and their outputs + @throws YarnException YarnException. + @throws IOException IOException.]]> +     +     + + + + + + Increase the resource of a container.

    + +

    The ApplicationMaster or other applications that use the + client must provide the details of the container, including the Id and + the target resource encapsulated in the updated container token via + {@link Container}. +

    + + @param container the container with updated token. + + @throws YarnException YarnException. + @throws IOException IOException.]]> +     +     + + + + + + Update the resources of a container.

    + +

    The ApplicationMaster or other applications that use the + client must provide the details of the container, including the Id and + the target resource encapsulated in the updated container token via + {@link Container}. +

    + + @param container the container with updated token. + + @throws YarnException YarnException. + @throws IOException IOException.]]> +     +     + + + + + + + Stop an started container.

    + + @param containerId the Id of the started container + @param nodeId the Id of the NodeManager + + @throws YarnException YarnException. + @throws IOException IOException.]]> +     +     + + + + + + + Query the status of a container.

    + + @param containerId the Id of the started container + @param nodeId the Id of the NodeManager + + @return the status of a container. + + @throws YarnException YarnException. + @throws IOException IOException.]]> +     +     + + + + + + + + Re-Initialize the Container.

    + + @param containerId the Id of the container to Re-Initialize. + @param containerLaunchContex the updated ContainerLaunchContext. + @param autoCommit commit re-initialization automatically ? + + @throws YarnException YarnException. + @throws IOException IOException.]]> +     +     + + + + + + Restart the specified container.

    + + @param containerId the Id of the container to restart. + + @throws YarnException YarnException. + @throws IOException IOException.]]> +     +     + + + + + + Rollback last reInitialization of the specified container.

    + + @param containerId the Id of the container to restart. + + @throws YarnException YarnException. + @throws IOException IOException.]]> +     +     + + + + + + Commit last reInitialization of the specified container.

    + + @param containerId the Id of the container to commit reInitialize. + + @throws YarnException YarnException. + @throws IOException IOException.]]> +     +     + + + + Set whether the containers that are started by this client, and are + still running should be stopped when the client stops. By default, the + feature should be enabled.

    However, containers will be stopped only + when service is stopped. i.e. after {@link NMClient#stop()}. + + @param enabled whether the feature is enabled or not]]> +     +     + + + + NMClient. This cache must be + shared with the {@link AMRMClient} that requested the containers managed + by this NMClient +

    + If a NM token cache is not set, the {@link NMTokenCache#getSingleton()} + singleton instance will be used. + + @param nmTokenCache the NM token cache to use.]]> + + + + + NMClient. This cache must be + shared with the {@link AMRMClient} that requested the containers managed + by this NMClient +

    + If a NM token cache is not set, the {@link NMTokenCache#getSingleton()} + singleton instance will be used. + + @return the NM token cache]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + By default YARN client libraries {@link AMRMClient} and {@link NMClient} use + {@link #getSingleton()} instance of the cache. +

      +
    • + Using the singleton instance of the cache is appropriate when running a + single ApplicationMaster in the same JVM. +
      • +
    • + When using the singleton, users don't need to do anything special, + {@link AMRMClient} and {@link NMClient} are already set up to use the + default singleton {@link NMTokenCache} +
      • +
    + If running multiple Application Masters in the same JVM, a different cache + instance should be used for each Application Master. +
      +
    • + If using the {@link AMRMClient} and the {@link NMClient}, setting up + and using an instance cache is as follows: + 
      +   NMTokenCache nmTokenCache = new NMTokenCache();
+   AMRMClient rmClient = AMRMClient.createAMRMClient();
+   NMClient nmClient = NMClient.createNMClient();
+   nmClient.setNMTokenCache(nmTokenCache);
+   ...
+
      +
      • +
    • + If using the {@link AMRMClientAsync} and the {@link NMClientAsync}, + setting up and using an instance cache is as follows: + 
      +   NMTokenCache nmTokenCache = new NMTokenCache();
+   AMRMClient rmClient = AMRMClient.createAMRMClient();
+   NMClient nmClient = NMClient.createNMClient();
+   nmClient.setNMTokenCache(nmTokenCache);
+   AMRMClientAsync rmClientAsync = new AMRMClientAsync(rmClient, 1000, [AMRM_CALLBACK]);
+   NMClientAsync nmClientAsync = new NMClientAsync("nmClient", nmClient, [NM_CALLBACK]);
+   ...
+
      +
      • +
    • + If using {@link ApplicationMasterProtocol} and + {@link ContainerManagementProtocol} directly, setting up and using an + instance cache is as follows: + 
      +   NMTokenCache nmTokenCache = new NMTokenCache();
+   ...
+   ApplicationMasterProtocol amPro = ClientRMProxy.createRMProxy(conf, ApplicationMasterProtocol.class);
+   ...
+   AllocateRequest allocateRequest = ...
+   ...
+   AllocateResponse allocateResponse = rmClient.allocate(allocateRequest);
+   for (NMToken token : allocateResponse.getNMTokens()) {
+     nmTokenCache.setToken(token.getNodeId().toString(), token.getToken());
+   }
+   ...
+   ContainerManagementProtocolProxy nmPro = ContainerManagementProtocolProxy(conf, nmTokenCache);
+   ...
+   nmPro.startContainer(container, containerContext);
+   ...
+
      +
      • +
    + It is also possible to mix the usage of a client ({@code AMRMClient} or + {@code NMClient}, or the async versions of them) with a protocol proxy + ({@code ContainerManagementProtocolProxy} or + {@code ApplicationMasterProtocol}).]]> +     +     + + + + + + + + + + + + + + The method to claim a resource with the SharedCacheManager. + The client uses a checksum to identify the resource and an + {@link ApplicationId} to identify which application will be using the + resource. +

    + +

    + The SharedCacheManager responds with whether or not the + resource exists in the cache. If the resource exists, a URL to + the resource in the shared cache is returned. If the resource does not + exist, null is returned instead. +

    + +

    + Once a URL has been returned for a resource, that URL is safe to use for + the lifetime of the application that corresponds to the provided + ApplicationId. +

    + + @param applicationId ApplicationId of the application using the resource + @param resourceKey the key (i.e. checksum) that identifies the resource + @return URL to the resource, or null if it does not exist]]> +     +     + + + + + + + The method to release a resource with the SharedCacheManager. + This method is called once an application is no longer using a claimed + resource in the shared cache. The client uses a checksum to identify the + resource and an {@link ApplicationId} to identify which application is + releasing the resource. +

    + +

    + Note: This method is an optimization and the client is not required to call + it for correctness. +

    + + @param applicationId ApplicationId of the application releasing the + resource + @param resourceKey the key (i.e. checksum) that identifies the resource]]> +     +     + + + + + + + + + + +     + + + + + + + + + + + + + + + + Obtain a {@link YarnClientApplication} for a new application, + which in turn contains the {@link ApplicationSubmissionContext} and + {@link org.apache.hadoop.yarn.api.protocolrecords.GetNewApplicationResponse} + objects. +

    + + @return {@link YarnClientApplication} built for a new application + @throws YarnException + @throws IOException]]> +     +     + + + + + + + Submit a new application to YARN. It is a blocking call - it + will not return {@link ApplicationId} until the submitted application is + submitted successfully and accepted by the ResourceManager. +

    + +

    + Users should provide an {@link ApplicationId} as part of the parameter + {@link ApplicationSubmissionContext} when submitting a new application, + otherwise it will throw the {@link ApplicationIdNotProvidedException}. +

    + +

    This internally calls {@link ApplicationClientProtocol#submitApplication + (SubmitApplicationRequest)}, and after that, it internally invokes + {@link ApplicationClientProtocol#getApplicationReport + (GetApplicationReportRequest)} and waits till it can make sure that the + application gets properly submitted. If RM fails over or RM restart + happens before ResourceManager saves the application's state, + {@link ApplicationClientProtocol + #getApplicationReport(GetApplicationReportRequest)} will throw + the {@link ApplicationNotFoundException}. This API automatically resubmits + the application with the same {@link ApplicationSubmissionContext} when it + catches the {@link ApplicationNotFoundException}

    + + @param appContext + {@link ApplicationSubmissionContext} containing all the details + needed to submit a new application + @return {@link ApplicationId} of the accepted application + @throws YarnException + @throws IOException + @see #createApplication()]]> +     +     + + + + + + + Fail an application attempt identified by given ID. +

    + + @param applicationAttemptId + {@link ApplicationAttemptId} of the attempt to fail. + @throws YarnException + in case of errors or if YARN rejects the request due to + access-control restrictions. + @throws IOException + @see #getQueueAclsInfo()]]> +     +     + + + + + + + Kill an application identified by given ID. +

    + + @param applicationId + {@link ApplicationId} of the application that needs to be killed + @throws YarnException + in case of errors or if YARN rejects the request due to + access-control restrictions. + @throws IOException + @see #getQueueAclsInfo()]]> +     +     + + + + + + + + Kill an application identified by given ID. +

    + @param applicationId {@link ApplicationId} of the application that needs to + be killed + @param diagnostics for killing an application. + @throws YarnException in case of errors or if YARN rejects the request due + to access-control restrictions. + @throws IOException]]> +     +     + + + + + + + Get a report of the given Application. +

    + +

    + In secure mode, YARN verifies access to the application, queue + etc. before accepting the request. +

    + +

    + If the user does not have VIEW_APP access then the following + fields in the report will be set to stubbed values: +

      +
    • host - set to "N/A"
      • +
    • RPC port - set to -1
      • +
    • client token - set to "N/A"
      • +
    • diagnostics - set to "N/A"
      • +
    • tracking URL - set to "N/A"
      • +
    • original tracking URL - set to "N/A"
      • +
    • resource usage report - all values are -1
      • +
    + + @param appId + {@link ApplicationId} of the application that needs a report + @return application report + @throws YarnException + @throws IOException]]> +     +     + + + + + + + The AMRM token is required for AM to RM scheduling operations. For + managed Application Masters YARN takes care of injecting it. For unmanaged + Applications Masters, the token must be obtained via this method and set + in the {@link org.apache.hadoop.security.UserGroupInformation} of the + current user. +

    + The AMRM token will be returned only if all the following conditions are + met: +

      +
    • the requester is the owner of the ApplicationMaster
      • +
    • the application master is an unmanaged ApplicationMaster
      • +
    • the application master is in ACCEPTED state
      • +
    + Else this method returns NULL. + + @param appId {@link ApplicationId} of the application to get the AMRM token + @return the AMRM token if available + @throws YarnException + @throws IOException]]> +     +     + + + + + + Get a report (ApplicationReport) of all Applications in the cluster. +

    + +

    + If the user does not have VIEW_APP access for an application + then the corresponding report will be filtered as described in + {@link #getApplicationReport(ApplicationId)}. +

    + + @return a list of reports of all running applications + @throws YarnException + @throws IOException]]> +     +     + + + + + + + Get a report (ApplicationReport) of Applications + matching the given application types in the cluster. +

    + +

    + If the user does not have VIEW_APP access for an application + then the corresponding report will be filtered as described in + {@link #getApplicationReport(ApplicationId)}. +

    + + @param applicationTypes set of application types you are interested in + @return a list of reports of applications + @throws YarnException + @throws IOException]]> +     +     + + + + + + + Get a report (ApplicationReport) of Applications matching the given + application states in the cluster. +

    + +

    + If the user does not have VIEW_APP access for an application + then the corresponding report will be filtered as described in + {@link #getApplicationReport(ApplicationId)}. +

    + + @param applicationStates set of application states you are interested in + @return a list of reports of applications + @throws YarnException + @throws IOException]]> +     +     + + + + + + + + Get a report (ApplicationReport) of Applications matching the given + application types and application states in the cluster. +

    + +

    + If the user does not have VIEW_APP access for an application + then the corresponding report will be filtered as described in + {@link #getApplicationReport(ApplicationId)}. +

    + + @param applicationTypes set of application types you are interested in + @param applicationStates set of application states you are interested in + @return a list of reports of applications + @throws YarnException + @throws IOException]]> +     +     + + + + + + + + + Get a report (ApplicationReport) of Applications matching the given + application types, application states and application tags in the cluster. +

    + +

    + If the user does not have VIEW_APP access for an application + then the corresponding report will be filtered as described in + {@link #getApplicationReport(ApplicationId)}. +

    + + @param applicationTypes set of application types you are interested in + @param applicationStates set of application states you are interested in + @param applicationTags set of application tags you are interested in + @return a list of reports of applications + @throws YarnException + @throws IOException]]> +     +     + + + + + + + + + + Get a report (ApplicationReport) of Applications matching the given users, + queues, application types and application states in the cluster. If any of + the params is set to null, it is not used when filtering. +

    + +

    + If the user does not have VIEW_APP access for an application + then the corresponding report will be filtered as described in + {@link #getApplicationReport(ApplicationId)}. +

    + + @param queues set of queues you are interested in + @param users set of users you are interested in + @param applicationTypes set of application types you are interested in + @param applicationStates set of application states you are interested in + @return a list of reports of applications + @throws YarnException + @throws IOException]]> +     +     + + + + + + + Get a list of ApplicationReports that match the given + {@link GetApplicationsRequest}. +

    + +

    + If the user does not have VIEW_APP access for an application + then the corresponding report will be filtered as described in + {@link #getApplicationReport(ApplicationId)}. +

    + + @param request the request object to get the list of applications. + @return The list of ApplicationReports that match the request + @throws YarnException Exception specific to YARN. + @throws IOException Exception mostly related to connection errors.]]> +     +     + + + + + + Get metrics ({@link YarnClusterMetrics}) about the cluster. +

    + + @return cluster metrics + @throws YarnException + @throws IOException]]> +     +     + + + + + + + Get a report of nodes ({@link NodeReport}) in the cluster. +

    + + @param states The {@link NodeState}s to filter on. If no filter states are + given, nodes in all states will be returned. + @return A list of node reports + @throws YarnException + @throws IOException]]> +     +     + + + + + + + Get a delegation token so as to be able to talk to YARN using those tokens. + + @param renewer + Address of the renewer who can renew these tokens when needed by + securely talking to YARN. + @return a delegation token ({@link Token}) that can be used to + talk to YARN + @throws YarnException + @throws IOException]]> + + + + + + + + + Get information ({@link QueueInfo}) about a given queue. +

    + + @param queueName + Name of the queue whose information is needed + @return queue information + @throws YarnException + in case of errors or if YARN rejects the request due to + access-control restrictions. + @throws IOException]]> +     +     + + + + + + Get information ({@link QueueInfo}) about all queues, recursively if there + is a hierarchy +

    + + @return a list of queue-information for all queues + @throws YarnException + @throws IOException]]> +     +     + + + + + + Get information ({@link QueueInfo}) about top level queues. +

    + + @return a list of queue-information for all the top-level queues + @throws YarnException + @throws IOException]]> +     +     + + + + + + + Get information ({@link QueueInfo}) about all the immediate children queues + of the given queue +

    + + @param parent + Name of the queue whose child-queues' information is needed + @return a list of queue-information for all queues who are direct children + of the given parent queue. + @throws YarnException + @throws IOException]]> +     +     + + + + + + Get information about acls for current user on all the + existing queues. +

    + + @return a list of queue acls ({@link QueueUserACLInfo}) for + current user + @throws YarnException + @throws IOException]]> +     +     + + + + + + + Get a report of the given ApplicationAttempt. +

    + +

    + In secure mode, YARN verifies access to the application, queue + etc. before accepting the request. +

    + + @param applicationAttemptId + {@link ApplicationAttemptId} of the application attempt that needs + a report + @return application attempt report + @throws YarnException + @throws ApplicationAttemptNotFoundException if application attempt + not found + @throws IOException]]> +     +     + + + + + + + Get a report of all (ApplicationAttempts) of Application in the cluster. +

    + + @param applicationId application id of the app + @return a list of reports for all application attempts for specified + application. + @throws YarnException + @throws IOException]]> +     +     + + + + + + + Get a report of the given Container. +

    + +

    + In secure mode, YARN verifies access to the application, queue + etc. before accepting the request. +

    + + @param containerId + {@link ContainerId} of the container that needs a report + @return container report + @throws YarnException + @throws ContainerNotFoundException if container not found. + @throws IOException]]> +     +     + + + + + + + Get a report of all (Containers) of ApplicationAttempt in the cluster. +

    + + @param applicationAttemptId application attempt id + @return a list of reports of all containers for specified application + attempts + @throws YarnException + @throws IOException]]> +     +     + + + + + + + + Attempts to move the given application to the given queue. +

    + + @param appId + Application to move. + @param queue + Queue to place it in to. + @throws YarnException + @throws IOException]]> +     +     + + + + + + Obtain a {@link GetNewReservationResponse} for a new reservation, + which contains the {@link ReservationId} object. +

    + + @return The {@link GetNewReservationResponse} containing a new + {@link ReservationId} object. + @throws YarnException if reservation cannot be created. + @throws IOException if reservation cannot be created.]]> +     +     + + + + + + + The interface used by clients to submit a new reservation to the + {@code ResourceManager}. +

    + +

    + The client packages all details of its request in a + {@link ReservationSubmissionRequest} object. This contains information + about the amount of capacity, temporal constraints, and gang needs. + Furthermore, the reservation might be composed of multiple stages, with + ordering dependencies among them. +

    + +

    + In order to respond, a new admission control component in the + {@code ResourceManager} performs an analysis of the resources that have + been committed over the period of time the user is requesting, verify that + the user requests can be fulfilled, and that it respect a sharing policy + (e.g., {@code CapacityOverTimePolicy}). Once it has positively determined + that the ReservationRequest is satisfiable the {@code ResourceManager} + answers with a {@link ReservationSubmissionResponse} that includes a + {@link ReservationId}. Upon failure to find a valid allocation the response + is an exception with the message detailing the reason of failure. +

    + +

    + The semantics guarantees that the {@link ReservationId} returned, + corresponds to a valid reservation existing in the time-range request by + the user. The amount of capacity dedicated to such reservation can vary + overtime, depending of the allocation that has been determined. But it is + guaranteed to satisfy all the constraint expressed by the user in the + {@link ReservationDefinition} +

    + + @param request request to submit a new Reservation + @return response contains the {@link ReservationId} on accepting the + submission + @throws YarnException if the reservation cannot be created successfully + @throws IOException]]> +     +     + + + + + + + The interface used by clients to update an existing Reservation. This is + referred to as a re-negotiation process, in which a user that has + previously submitted a Reservation. +

    + +

    + The allocation is attempted by virtually substituting all previous + allocations related to this Reservation with new ones, that satisfy the new + {@link ReservationDefinition}. Upon success the previous allocation is + atomically substituted by the new one, and on failure (i.e., if the system + cannot find a valid allocation for the updated request), the previous + allocation remains valid. +

    + + @param request to update an existing Reservation (the + {@link ReservationUpdateRequest} should refer to an existing valid + {@link ReservationId}) + @return response empty on successfully updating the existing reservation + @throws YarnException if the request is invalid or reservation cannot be + updated successfully + @throws IOException]]> +     +     + + + + + + + The interface used by clients to remove an existing Reservation. +

    + + @param request to remove an existing Reservation (the + {@link ReservationDeleteRequest} should refer to an existing valid + {@link ReservationId}) + @return response empty on successfully deleting the existing reservation + @throws YarnException if the request is invalid or reservation cannot be + deleted successfully + @throws IOException]]> +     +     + + + + + + + The interface used by clients to get the list of reservations in a plan. + The reservationId will be used to search for reservations to list if it is + provided. Otherwise, it will select active reservations within the + startTime and endTime (inclusive). +

    + + @param request to list reservations in a plan. Contains fields to select + String queue, ReservationId reservationId, long startTime, + long endTime, and a bool includeReservationAllocations. + + queue: Required. Cannot be null or empty. Refers to the + reservable queue in the scheduler that was selected when + creating a reservation submission + {@link ReservationSubmissionRequest}. + + reservationId: Optional. If provided, other fields will + be ignored. + + startTime: Optional. If provided, only reservations that + end after the startTime will be selected. This defaults + to 0 if an invalid number is used. + + endTime: Optional. If provided, only reservations that + start on or before endTime will be selected. This defaults + to Long.MAX_VALUE if an invalid number is used. + + includeReservationAllocations: Optional. Flag that + determines whether the entire reservation allocations are + to be returned. Reservation allocations are subject to + change in the event of re-planning as described by + {@link ReservationDefinition}. + + @return response that contains information about reservations that are + being searched for. + @throws YarnException if the request is invalid + @throws IOException if the request failed otherwise]]> +     +     + + + + + + The interface used by client to get node to labels mappings in existing cluster +

    + + @return node to labels mappings + @throws YarnException + @throws IOException]]> +     +     + + + + + + The interface used by client to get labels to nodes mapping + in existing cluster +

    + + @return node to labels mappings + @throws YarnException + @throws IOException]]> +     +     + + + + + + + The interface used by client to get labels to nodes mapping + for specified labels in existing cluster +

    + + @param labels labels for which labels to nodes mapping has to be retrieved + @return labels to nodes mappings for specific labels + @throws YarnException + @throws IOException]]> +     +     + + + + + + The interface used by client to get node labels in the cluster +

    + + @return cluster node labels collection + @throws YarnException when there is a failure in + {@link ApplicationClientProtocol} + @throws IOException when there is a failure in + {@link ApplicationClientProtocol}]]> +     +     + + + + + + + + The interface used by client to set priority of an application +

    + @param applicationId + @param priority + @return updated priority of an application. + @throws YarnException + @throws IOException]]> +     +     + + + + + + + + Signal a container identified by given ID. +

    + + @param containerId + {@link ContainerId} of the container that needs to be signaled + @param command the signal container command + @throws YarnException + @throws IOException]]> +     +     + + + + + + + + + + + Get the resource profiles available in the RM. +

    + @return a Map of the resource profile names to their capabilities + @throws YARNFeatureNotEnabledException if resource-profile is disabled + @throws YarnException if any error happens inside YARN + @throws IOException in case of other errors]]> +     +     + + + + + + + Get the details of a specific resource profile from the RM. +

    + @param profile the profile name + @return resource profile name with its capabilities + @throws YARNFeatureNotEnabledException if resource-profile is disabled + @throws YarnException if any error happens inside YARN + @throws IOException in case of other others]]> +     +     + + + + + + Get available resource types supported by RM. +

    + @return list of supported resource types with detailed information + @throws YarnException if any issue happens inside YARN + @throws IOException in case of other others]]> +     +     + + + + + + The interface used by client to get node attributes in the cluster. +

    + + @return cluster node attributes collection + @throws YarnException when there is a failure in + {@link ApplicationClientProtocol} + @throws IOException when there is a failure in + {@link ApplicationClientProtocol}]]> +     +     + + + + + + + The interface used by client to get mapping of AttributeKey to associated + NodeToAttributeValue list for specified node attributeKeys in the cluster. +

    + + @param attributes AttributeKeys for which associated NodeToAttributeValue + mapping value has to be retrieved. If empty or null is set then + will return mapping for all attributeKeys in the cluster + @return mapping of AttributeKey to List of associated + NodeToAttributeValue's. + @throws YarnException + @throws IOException]]> +     +     + + + + + + + The interface used by client to get all node to attribute mapping in + existing cluster. +

    + + @param hostNames HostNames for which host to attributes mapping has to + be retrived.If empty or null is set then will return + all nodes to attributes mapping in cluster. + @return Node to attribute mappings + @throws YarnException + @throws IOException]]> +     +     + + + + + + + The interface used by client to get a shell to a container. +

    + + @param containerId Container ID + @param command Shell type + @throws IOException if connection fails.]]> +     +     +     + + + + + + + + + + + +     + + + + + + + + + + + + + + + + Create a new instance of AMRMClientAsync.

    + + @param intervalMs heartbeat interval in milliseconds between AM and RM + @param callbackHandler callback handler that processes responses from + the ResourceManager]]> +     +     + + + + + + Create a new instance of AMRMClientAsync.

    + + @param client the AMRMClient instance + @param intervalMs heartbeat interval in milliseconds between AM and RM + @param callbackHandler callback handler that processes responses from + the ResourceManager]]> +     +     + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + RegisterApplicationMasterResponse + @throws YarnException + @throws IOException]]> + + + + + + + + + + + + + + + + allocate + @param req Resource request]]> + + + + + + + + + + + + + allocate. + Any previous pending resource change request of the same container will be + removed. + + Application that calls this method is expected to maintain the + Containers that are returned from previous successful + allocations or resource changes. By passing in the existing container and a + target resource capability to this method, the application requests the + ResourceManager to change the existing resource allocation to the target + resource allocation. + + @deprecated use + {@link #requestContainerUpdate(Container, UpdateContainerRequest)} + + @param container The container returned from the last successful resource + allocation or resource change + @param capability The target resource capability of the container]]> + + + + + + + allocate. + Any previous pending update request of the same container will be + removed. + + @param container The container returned from the last successful resource + allocation or update + @param updateContainerRequest The UpdateContainerRequest.]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + check to return true for each 1000 ms. + See also {@link #waitFor(java.util.function.Supplier, int)} + and {@link #waitFor(java.util.function.Supplier, int, int)} + @param check the condition for which it should wait]]> + + + + + + + + check to return true for each + checkEveryMillis ms. + See also {@link #waitFor(java.util.function.Supplier, int, int)} + @param check user defined checker + @param checkEveryMillis interval to call check]]> + + + + + + + + + check to return true for each + checkEveryMillis ms. In the main loop, this method will log + the message "waiting in main loop" for each logInterval times + iteration to confirm the thread is alive. + @param check user defined checker + @param checkEveryMillis interval to call check + @param logInterval interval to log for each]]> + + + + + + + + + + AMRMClientAsync handles communication with the ResourceManager + and provides asynchronous updates on events such as container allocations and + completions. It contains a thread that sends periodic heartbeats to the + ResourceManager. + + It should be used by implementing a CallbackHandler: + 
    + {@code
+ class MyCallbackHandler extends AMRMClientAsync.AbstractCallbackHandler {
+   public void onContainersAllocated(List containers) {
+     [run tasks on the containers]
+   }
+
+   public void onContainersUpdated(List containers) {
+     [determine if resource allocation of containers have been increased in
+      the ResourceManager, and if so, inform the NodeManagers to increase the
+      resource monitor/enforcement on the containers]
+   }
+
+   public void onContainersCompleted(List statuses) {
+     [update progress, check whether app is done]
+   }
+   
+   public void onNodesUpdated(List updated) {}
+   
+   public void onReboot() {}
+ }
+ }
+
    + + The client's lifecycle should be managed similarly to the following: + + 
    + {@code
+ AMRMClientAsync asyncClient = 
+     createAMRMClientAsync(appAttId, 1000, new MyCallbackhandler());
+ asyncClient.init(conf);
+ asyncClient.start();
+ RegisterApplicationMasterResponse response = asyncClient
+    .registerApplicationMaster(appMasterHostname, appMasterRpcPort,
+       appMasterTrackingUrl);
+ asyncClient.addContainerRequest(containerRequest);
+ [... wait for application to complete]
+ asyncClient.unregisterApplicationMaster(status, appMsg, trackingUrl);
+ asyncClient.stop();
+ }
+
    ]]> +     +     + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Update the resources of a container.

    + +

    The ApplicationMaster or other applications that use the + client must provide the details of the container, including the Id and + the target resource encapsulated in the updated container token via + {@link Container}. +

    + + @param container the container with updated token.]]> +     +     + + + + + + Re-Initialize the Container.

    + + @param containerId the Id of the container to Re-Initialize. + @param containerLaunchContex the updated ContainerLaunchContext. + @param autoCommit commit re-initialization automatically ?]]> +     +     + + + + Restart the specified container.

    + + @param containerId the Id of the container to restart.]]> +     +     + + + + Rollback last reInitialization of the specified container.

    + + @param containerId the Id of the container to restart.]]> +     +     + + + + Commit last reInitialization of the specified container.

    + + @param containerId the Id of the container to commit reInitialize.]]> +     +     + + + + + + + + + + + + + + + + + + + + + + + + NMClientAsync handles communication with all the NodeManagers + and provides asynchronous updates on getting responses from them. It + maintains a thread pool to communicate with individual NMs where a number of + worker threads process requests to NMs by using {@link NMClientImpl}. The max + size of the thread pool is configurable through + {@link YarnConfiguration#NM_CLIENT_ASYNC_THREAD_POOL_MAX_SIZE}. + + It should be used in conjunction with a CallbackHandler. For example + + 
    + {@code
+ class MyCallbackHandler extends NMClientAsync.AbstractCallbackHandler {
+   public void onContainerStarted(ContainerId containerId,
+       Map allServiceResponse) {
+     [post process after the container is started, process the response]
+   }
+
+   public void onContainerResourceIncreased(ContainerId containerId,
+       Resource resource) {
+     [post process after the container resource is increased]
+   }
+
+   public void onContainerStatusReceived(ContainerId containerId,
+       ContainerStatus containerStatus) {
+     [make use of the status of the container]
+   }
+
+   public void onContainerStopped(ContainerId containerId) {
+     [post process after the container is stopped]
+   }
+
+   public void onStartContainerError(
+       ContainerId containerId, Throwable t) {
+     [handle the raised exception]
+   }
+
+   public void onGetContainerStatusError(
+       ContainerId containerId, Throwable t) {
+     [handle the raised exception]
+   }
+
+   public void onStopContainerError(
+       ContainerId containerId, Throwable t) {
+     [handle the raised exception]
+   }
+ }
+ }
+
    + + The client's life-cycle should be managed like the following: + + 
    + {@code
+ NMClientAsync asyncClient = 
+     NMClientAsync.createNMClientAsync(new MyCallbackhandler());
+ asyncClient.init(conf);
+ asyncClient.start();
+ asyncClient.startContainer(container, containerLaunchContext);
+ [... wait for container being started]
+ asyncClient.getContainerStatus(container.getId(), container.getNodeId(),
+     container.getContainerToken());
+ [... handle the status in the callback instance]
+ asyncClient.stopContainer(container.getId(), container.getNodeId(),
+     container.getContainerToken());
+ [... wait for container being stopped]
+ asyncClient.stop();
+ }
+
    ]]> +     +     + +     + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +     diff --git a/hadoop-yarn-project/hadoop-yarn/dev-support/jdiff/Apache_Hadoop_YARN_Common_3.2.4.xml b/hadoop-yarn-project/hadoop-yarn/dev-support/jdiff/Apache_Hadoop_YARN_Common_3.2.4.xml new file mode 100644 index 0000000000000..5ca1716cca2d7 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/dev-support/jdiff/Apache_Hadoop_YARN_Common_3.2.4.xml @@ -0,0 +1,3964 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Type of proxy. + @return Proxy to the ResourceManager for the specified client protocol. + @throws IOException]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Create a new instance of AppAdminClient. +

    + + @param appType application type + @param conf configuration + @return app admin client]]> +     +     + + + + + + + + + + Launch a new YARN application. +

    + + @param fileName specification of application + @param appName name of the application + @param lifetime lifetime of the application + @param queue queue of the application + @return exit code + @throws IOException IOException + @throws YarnException exception in client or server]]> +     +     + + + + + + + Stop a YARN application (attempt to stop gracefully before killing the + application). In the case of a long-running service, the service may be + restarted later. +

    + + @param appName the name of the application + @return exit code + @throws IOException IOException + @throws YarnException exception in client or server]]> +     +     + + + + + + + Start a YARN application from a previously saved specification. In the + case of a long-running service, the service must have been previously + launched/started and then stopped, or previously saved but not started. +

    + + @param appName the name of the application + @return exit code + @throws IOException IOException + @throws YarnException exception in client or server]]> +     +     + + + + + + + + + + Save the specification for a YARN application / long-running service. + The application may be started later. +

    + + @param fileName specification of application to save + @param appName name of the application + @param lifetime lifetime of the application + @param queue queue of the application + @return exit code + @throws IOException IOException + @throws YarnException exception in client or server]]> +     +     + + + + + + + Remove the specification and all application data for a YARN application. + The application cannot be running. +

    + + @param appName the name of the application + @return exit code + @throws IOException IOException + @throws YarnException exception in client or server]]> +     +     + + + + + + + + Change the number of running containers for a component of a YARN + application / long-running service. +

    + + @param appName the name of the application + @param componentCounts map of component name to new component count or + amount to change existing component count (e.g. + 5, +5, -5) + @return exit code + @throws IOException IOException + @throws YarnException exception in client or server]]> +     +     + + + + + + + Upload AM dependencies to HDFS. This makes future application launches + faster since the dependencies do not have to be uploaded on each launch. +

    + + @param destinationFolder + an optional HDFS folder where dependency tarball will be uploaded + @return exit code + @throws IOException + IOException + @throws YarnException + exception in client or server]]> +     +     + + + + + + + Get detailed app specific status string for a YARN application. +

    + + @param appIdOrName appId or appName + @return status string + @throws IOException IOException + @throws YarnException exception in client or server]]> +     +     + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +     + + + + + + + + + + + + + + + + + + Send the information of a number of conceptual entities to the timeline + server. It is a blocking API. The method will not return until it gets the + response from the timeline server. +

    + + @param entities + the collection of {@link TimelineEntity} + @return the error information if the sent entities are not correctly stored + @throws IOException if there are I/O errors + @throws YarnException if entities are incomplete/invalid]]> +     +     + + + + + + + + + Send the information of a number of conceptual entities to the timeline + server. It is a blocking API. The method will not return until it gets the + response from the timeline server. + + This API is only for timeline service v1.5 +

    + + @param appAttemptId {@link ApplicationAttemptId} + @param groupId {@link TimelineEntityGroupId} + @param entities + the collection of {@link TimelineEntity} + @return the error information if the sent entities are not correctly stored + @throws IOException if there are I/O errors + @throws YarnException if entities are incomplete/invalid]]> +     +     + + + + + + + Send the information of a domain to the timeline server. It is a + blocking API. The method will not return until it gets the response from + the timeline server. +

    + + @param domain + an {@link TimelineDomain} object + @throws IOException + @throws YarnException]]> +     +     + + + + + + + + Send the information of a domain to the timeline server. It is a + blocking API. The method will not return until it gets the response from + the timeline server. + + This API is only for timeline service v1.5 +

    + + @param domain + an {@link TimelineDomain} object + @param appAttemptId {@link ApplicationAttemptId} + @throws IOException + @throws YarnException]]> +     +     + + + + + + + Get a delegation token so as to be able to talk to the timeline server in a + secure way. +

    + + @param renewer + Address of the renewer who can renew these tokens when needed by + securely talking to the timeline server + @return a delegation token ({@link Token}) that can be used to talk to the + timeline server + @throws IOException + @throws YarnException]]> +     +     + + + + + + + Renew a timeline delegation token. +

    + + @param timelineDT + the delegation token to renew + @return the new expiration time + @throws IOException + @throws YarnException]]> +     +     + + + + + + + Cancel a timeline delegation token. +

    + + @param timelineDT + the delegation token to cancel + @throws IOException + @throws YarnException]]> +     +     + + + +     + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +     + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + parameterized event of type T]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + InputStream to be checksumed + @return the message digest of the input stream + @throws IOException]]> + + + + + + + + + + + + SharedCacheChecksum object based on the configurable + algorithm implementation + (see yarn.sharedcache.checksum.algo.impl) + + @return SharedCacheChecksum object]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + The object type on which this state machine operates. + @param The state of the entity. + @param The external eventType to be handled. + @param The event object.]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + When {@link #limit} would be reached on append, past messages will be + truncated from head, and a header telling the user about truncation will be + prepended, with ellipses in between header and messages. +

    + Note that header and ellipses are not counted against {@link #limit}. +

    + An example: + + 

    + {@code
+   // At the beginning it's an empty string
+   final Appendable shortAppender = new BoundedAppender(80);
+   // The whole message fits into limit
+   shortAppender.append(
+       "message1 this is a very long message but fitting into limit\n");
+   // The first message is truncated, the second not
+   shortAppender.append("message2 this is shorter than the previous one\n");
+   // The first message is deleted, the second truncated, the third
+   // preserved
+   shortAppender.append("message3 this is even shorter message, maybe.\n");
+   // The first two are deleted, the third one truncated, the last preserved
+   shortAppender.append("message4 the shortest one, yet the greatest :)");
+   // Current contents are like this:
+   // Diagnostic messages truncated, showing last 80 chars out of 199:
+   // ...s is even shorter message, maybe.
+   // message4 the shortest one, yet the greatest :)
+ }
+
    +

    + Note that null values are {@link #append(CharSequence) append}ed + just like in {@link StringBuilder#append(CharSequence) original + implementation}. +

    + Note that this class is not thread safe.]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/hadoop-yarn-project/hadoop-yarn/dev-support/jdiff/Apache_Hadoop_YARN_Common_3.3.4.xml b/hadoop-yarn-project/hadoop-yarn/dev-support/jdiff/Apache_Hadoop_YARN_Common_3.3.4.xml new file mode 100644 index 0000000000000..3ec09cfb059b3 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/dev-support/jdiff/Apache_Hadoop_YARN_Common_3.3.4.xml @@ -0,0 +1,3975 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Type of proxy. + @return Proxy to the ResourceManager for the specified client protocol. + @throws IOException]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Create a new instance of AppAdminClient. +

    + + @param appType application type + @param conf configuration + @return app admin client]]> +     + + + + + + + + + + + Launch a new YARN application. +

    + + @param fileName specification of application + @param appName name of the application + @param lifetime lifetime of the application + @param queue queue of the application + @return exit code + @throws IOException IOException + @throws YarnException exception in client or server]]> +     +     + + + + + + + Stop a YARN application (attempt to stop gracefully before killing the + application). In the case of a long-running service, the service may be + restarted later. +

    + + @param appName the name of the application + @return exit code + @throws IOException IOException + @throws YarnException exception in client or server]]> +     +     + + + + + + + Start a YARN application from a previously saved specification. In the + case of a long-running service, the service must have been previously + launched/started and then stopped, or previously saved but not started. +

    + + @param appName the name of the application + @return exit code + @throws IOException IOException + @throws YarnException exception in client or server]]> +     +     + + + + + + + + + + Save the specification for a YARN application / long-running service. + The application may be started later. +

    + + @param fileName specification of application to save + @param appName name of the application + @param lifetime lifetime of the application + @param queue queue of the application + @return exit code + @throws IOException IOException + @throws YarnException exception in client or server]]> +     +     + + + + + + + Remove the specification and all application data for a YARN application. + The application cannot be running. +

    + + @param appName the name of the application + @return exit code + @throws IOException IOException + @throws YarnException exception in client or server]]> +     +     + + + + + + + + Change the number of running containers for a component of a YARN + application / long-running service. +

    + + @param appName the name of the application + @param componentCounts map of component name to new component count or + amount to change existing component count (e.g. + 5, +5, -5) + @return exit code + @throws IOException IOException + @throws YarnException exception in client or server]]> +     +     + + + + + + + Upload AM dependencies to HDFS. This makes future application launches + faster since the dependencies do not have to be uploaded on each launch. +

    + + @param destinationFolder + an optional HDFS folder where dependency tarball will be uploaded + @return exit code + @throws IOException + IOException + @throws YarnException + exception in client or server]]> +     +     + + + + + + + Get detailed app specific status string for a YARN application. +

    + + @param appIdOrName appId or appName + @return status string + @throws IOException IOException + @throws YarnException exception in client or server]]> +     +     + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +     + + + + + + + + + + + + + + + + + + Send the information of a number of conceptual entities to the timeline + server. It is a blocking API. The method will not return until it gets the + response from the timeline server. +

    + + @param entities + the collection of {@link TimelineEntity} + @return the error information if the sent entities are not correctly stored + @throws IOException if there are I/O errors + @throws YarnException if entities are incomplete/invalid]]> +     +     + + + + + + + + + Send the information of a number of conceptual entities to the timeline + server. It is a blocking API. The method will not return until it gets the + response from the timeline server. + + This API is only for timeline service v1.5 +

    + + @param appAttemptId {@link ApplicationAttemptId} + @param groupId {@link TimelineEntityGroupId} + @param entities + the collection of {@link TimelineEntity} + @return the error information if the sent entities are not correctly stored + @throws IOException if there are I/O errors + @throws YarnException if entities are incomplete/invalid]]> +     +     + + + + + + + Send the information of a domain to the timeline server. It is a + blocking API. The method will not return until it gets the response from + the timeline server. +

    + + @param domain + an {@link TimelineDomain} object + @throws IOException + @throws YarnException]]> +     +     + + + + + + + + Send the information of a domain to the timeline server. It is a + blocking API. The method will not return until it gets the response from + the timeline server. + + This API is only for timeline service v1.5 +

    + + @param domain + an {@link TimelineDomain} object + @param appAttemptId {@link ApplicationAttemptId} + @throws IOException + @throws YarnException]]> +     +     + + + + + + + Get a delegation token so as to be able to talk to the timeline server in a + secure way. +

    + + @param renewer + Address of the renewer who can renew these tokens when needed by + securely talking to the timeline server + @return a delegation token ({@link Token}) that can be used to talk to the + timeline server + @throws IOException + @throws YarnException]]> +     +     + + + + + + + Renew a timeline delegation token. +

    + + @param timelineDT + the delegation token to renew + @return the new expiration time + @throws IOException + @throws YarnException]]> +     +     + + + + + + + Cancel a timeline delegation token. +

    + + @param timelineDT + the delegation token to cancel + @throws IOException + @throws YarnException]]> +     +     + + + +     + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +     + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + parameterized event of type T]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + InputStream to be checksumed + @return the message digest of the input stream + @throws IOException]]> + + + + + + + + + + + + SharedCacheChecksum object based on the configurable + algorithm implementation + (see yarn.sharedcache.checksum.algo.impl) + + @return SharedCacheChecksum object]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + The object type on which this state machine operates. + @param The state of the entity. + @param The external eventType to be handled. + @param The event object.]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + When {@link #limit} would be reached on append, past messages will be + truncated from head, and a header telling the user about truncation will be + prepended, with ellipses in between header and messages. +

    + Note that header and ellipses are not counted against {@link #limit}. +

    + An example: + + 

    + {@code
+   // At the beginning it's an empty string
+   final Appendable shortAppender = new BoundedAppender(80);
+   // The whole message fits into limit
+   shortAppender.append(
+       "message1 this is a very long message but fitting into limit\n");
+   // The first message is truncated, the second not
+   shortAppender.append("message2 this is shorter than the previous one\n");
+   // The first message is deleted, the second truncated, the third
+   // preserved
+   shortAppender.append("message3 this is even shorter message, maybe.\n");
+   // The first two are deleted, the third one truncated, the last preserved
+   shortAppender.append("message4 the shortest one, yet the greatest :)");
+   // Current contents are like this:
+   // Diagnostic messages truncated, showing last 80 chars out of 199:
+   // ...s is even shorter message, maybe.
+   // message4 the shortest one, yet the greatest :)
+ }
+
    +

    + Note that null values are {@link #append(CharSequence) append}ed + just like in {@link StringBuilder#append(CharSequence) original + implementation}. +

    + Note that this class is not thread safe.]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/hadoop-yarn-project/hadoop-yarn/dev-support/jdiff/Apache_Hadoop_YARN_Server_Common_3.2.4.xml b/hadoop-yarn-project/hadoop-yarn/dev-support/jdiff/Apache_Hadoop_YARN_Server_Common_3.2.4.xml new file mode 100644 index 0000000000000..d06cc2fe00f6f --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/dev-support/jdiff/Apache_Hadoop_YARN_Server_Common_3.2.4.xml @@ -0,0 +1,1412 @@ + + + + + + + + + + + + + + + + + + + + + + + + true if the node is healthy, else false]]> + + + + + diagnostic health report of the node. + @return diagnostic health report of the node]]> + + + + + last timestamp at which the health report was received. + @return last timestamp at which the health report was received]]> + + + + + It includes information such as: +

      +
    • + An indicator of whether the node is healthy, as determined by the + health-check script. +
      • +
    • The previous time at which the health status was reported.
      • +
    • A diagnostic report on the health status.
      • +
    + + @see NodeReport + @see ApplicationClientProtocol#getClusterNodes(org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodesRequest)]]> +     +     + +     + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + type of the proxy + @return the proxy instance + @throws IOException if fails to create the proxy]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + true if the iteration has more elements.]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +     diff --git a/hadoop-yarn-project/hadoop-yarn/dev-support/jdiff/Apache_Hadoop_YARN_Server_Common_3.3.4.xml b/hadoop-yarn-project/hadoop-yarn/dev-support/jdiff/Apache_Hadoop_YARN_Server_Common_3.3.4.xml new file mode 100644 index 0000000000000..1f525eb844ab8 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/dev-support/jdiff/Apache_Hadoop_YARN_Server_Common_3.3.4.xml @@ -0,0 +1,1456 @@ + + + + + + + + + + + + + + + + + + + + + + + + true if the node is healthy, else false]]> + + + + + diagnostic health report of the node. + @return diagnostic health report of the node]]> + + + + + last timestamp at which the health report was received. + @return last timestamp at which the health report was received]]> + + + + + It includes information such as: +
      +
    • + An indicator of whether the node is healthy, as determined by the + health-check script. +
      • +
    • The previous time at which the health status was reported.
      • +
    • A diagnostic report on the health status.
      • +
    + + @see NodeReport + @see ApplicationClientProtocol#getClusterNodes(org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodesRequest)]]> +     +     + +     + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + type of the proxy + @return the proxy instance + @throws IOException if fails to create the proxy]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + true if the iteration has more elements.]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +     diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/pom.xml index 6e7c27a23ea85..61747c2cd8028 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/pom.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/pom.xml @@ -115,10 +115,6 @@ com.fasterxml.jackson.core jackson-annotations     - - javax.ws.rs - javax.ws.rs-api - diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/HAUtil.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/HAUtil.java index d256252601e0d..1360359a9a649 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/HAUtil.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/HAUtil.java @@ -64,8 +64,12 @@ public static boolean isFederationEnabled(Configuration conf) { * configuration; else false. */ public static boolean isFederationFailoverEnabled(Configuration conf) { - return conf.getBoolean(YarnConfiguration.FEDERATION_FAILOVER_ENABLED, - YarnConfiguration.DEFAULT_FEDERATION_FAILOVER_ENABLED); + // Federation failover is not enabled unless federation is enabled. This previously caused + // YARN RMProxy to use the HA Retry policy in a non-HA & non-federation environments because + // the default federation failover enabled value is true. + return isFederationEnabled(conf) && + conf.getBoolean(YarnConfiguration.FEDERATION_FAILOVER_ENABLED, + YarnConfiguration.DEFAULT_FEDERATION_FAILOVER_ENABLED); } /** diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java index d42562cf6140a..fd4cb0c5ec9d2 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java @@ -793,6 +793,10 @@ public static boolean isAclEnabled(Configuration conf) { RM_PREFIX + "delegation.token.max-lifetime"; public static final long RM_DELEGATION_TOKEN_MAX_LIFETIME_DEFAULT = 7*24*60*60*1000; // 7 days + public static final String RM_DELEGATION_TOKEN_REMOVE_SCAN_INTERVAL_KEY = + RM_PREFIX + "delegation.token.remove-scan-interval"; + public static final long RM_DELEGATION_TOKEN_REMOVE_SCAN_INTERVAL_DEFAULT = + 60*60*1000; // 1 hour public static final String RM_DELEGATION_TOKEN_MAX_CONF_SIZE = RM_PREFIX + "delegation-token.max-conf-size-bytes"; @@ -1548,6 +1552,13 @@ public static boolean isAclEnabled(Configuration conf) { public static final long DEFAULT_LOG_AGGREGATION_STATUS_TIME_OUT_MS = 10 * 60 * 1000; + /** + * Whether to clean up nodemanager logs when log aggregation is enabled. + */ + public static final String LOG_AGGREGATION_ENABLE_LOCAL_CLEANUP = + YARN_PREFIX + "log-aggregation.enable-local-cleanup"; + public static final boolean DEFAULT_LOG_AGGREGATION_ENABLE_LOCAL_CLEANUP = true; + /** * Number of seconds to retain logs on the NodeManager. Only applicable if Log * aggregation is disabled @@ -2983,6 +2994,51 @@ public static boolean isAclEnabled(Configuration conf) { public static final int DEFAULT_YARN_DISPATCHER_CPU_MONITOR_SAMPLES_PER_MIN = 60; + /** + * Resource manager dispatcher has a thread pool that prints EventQueue, + * configure the corePoolSize of this thread pool, + * the meaning of corePoolSize is the number of threads to keep in the pool. + */ + public static final String YARN_DISPATCHER_PRINT_THREAD_POOL_CORE_POOL_SIZE = + YARN_PREFIX + "dispatcher.print-thread-pool.core-pool-size"; + + /** + * The minimum number of core threads for the + * Resource manager dispatcher is 1. + */ + public static final int DEFAULT_YARN_DISPATCHER_PRINT_THREAD_POOL_CORE_POOL_SIZE = 1; + + /** + * Resource manager dispatcher has a thread pool that prints EventQueue, + * configure the maximumPoolSize of this thread pool, + * the meaning of maximumPoolSize is the maximum number of threads to allow in the pool. + */ + public static final String YARN_DISPATCHER_PRINT_THREAD_POOL_MAXIMUM_POOL_SIZE = + YARN_PREFIX + "dispatcher.print-thread-pool.maximum-pool-size"; + + /** + * The maximum number of core threads for the + * Resource manager dispatcher is 5. + */ + public static final int DEFAULT_YARN_DISPATCHER_PRINT_THREAD_POOL_MAXIMUM_POOL_SIZE = 5; + + /** + * Resource manager dispatcher has a thread pool that prints EventQueue, + * configure the keepAliveTime of this thread pool, + * The meaning of keepAliveTime is as follows + * when the number of threads is greater than the core, + * this is the maximum time that excess idle threads will wait for new tasks before terminating. + */ + public static final String YARN_DISPATCHER_PRINT_THREAD_POOL_KEEP_ALIVE_TIME = + YARN_PREFIX + "dispatcher.print-thread-pool.keep-alive-time"; + + /** + * The keep alive time of core threads for the + * Resource manager dispatcher is 10s. + */ + public static final long DEFAULT_YARN_DISPATCHER_PRINT_THREAD_POOL_KEEP_ALIVE_TIME = + 10*1000; // 10s + /** * CLASSPATH for YARN applications. A comma-separated list of CLASSPATH * entries @@ -3920,6 +3976,13 @@ public static boolean isAclEnabled(Configuration conf) { public static final String DEFAULT_FEDERATION_REGISTRY_BASE_KEY = "yarnfederation/"; + public static final String FEDERATION_STATESTORE_HEARTBEAT_INITIAL_DELAY = + FEDERATION_PREFIX + "state-store.heartbeat.initial-delay"; + + // 30 secs + public static final int + DEFAULT_FEDERATION_STATESTORE_HEARTBEAT_INITIAL_DELAY = 30; + public static final String FEDERATION_STATESTORE_HEARTBEAT_INTERVAL_SECS = FEDERATION_PREFIX + "state-store.heartbeat-interval-secs"; @@ -4107,6 +4170,25 @@ public static boolean isAclEnabled(Configuration conf) { public static final long DEFAULT_ROUTER_WEBAPP_READ_TIMEOUT = TimeUnit.SECONDS.toMillis(30); + /** The Kerberos keytab for the yarn router.*/ + public static final String ROUTER_KEYTAB = ROUTER_PREFIX + "keytab.file"; + + /** The Kerberos principal for the yarn router.*/ + public static final String ROUTER_PRINCIPAL = ROUTER_PREFIX + "kerberos.principal"; + + /** The Kerberos principal hostname for the yarn router.*/ + public static final String ROUTER_KERBEROS_PRINCIPAL_HOSTNAME_KEY = ROUTER_PREFIX + + "kerberos.principal.hostname"; + + /** Router enable AppsInfo Cache. **/ + public static final String ROUTER_APPSINFO_ENABLED = ROUTER_WEBAPP_PREFIX + "appsinfo-enabled"; + public static final boolean DEFAULT_ROUTER_APPSINFO_ENABLED = false; + + /** Router AppsInfo Cache Count. **/ + public static final String ROUTER_APPSINFO_CACHED_COUNT = + ROUTER_WEBAPP_PREFIX + "appsinfo-cached-count"; + public static final int DEFAULT_ROUTER_APPSINFO_CACHED_COUNT = 100; + //////////////////////////////// // CSI Volume configs //////////////////////////////// diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceUtils.java index d4d14d3b3ddda..1433c24e9e10c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceUtils.java @@ -115,8 +115,7 @@ private static void checkSpecialResources( * Supporting 'memory', 'memory-mb', 'vcores' also as invalid resource * names, in addition to 'MEMORY' for historical reasons */ - String[] keys = { "memory", ResourceInformation.MEMORY_URI, - ResourceInformation.VCORES_URI }; + String[] keys = {"memory", ResourceInformation.MEMORY_URI, ResourceInformation.VCORES_URI}; for(String key : keys) { if (resourceInformationMap.containsKey(key)) { LOG.warn("Attempt to define resource '" + key + "', but it is not allowed."); @@ -234,7 +233,8 @@ static void validateNameOfResourceNameAndThrowException(String resourceName) } /** - * Get maximum allocation from config, *THIS WILL NOT UPDATE INTERNAL DATA* + * Get maximum allocation from config, *THIS WILL NOT UPDATE INTERNAL DATA. + * * @param conf config * @return maximum allocation */ @@ -379,7 +379,7 @@ public static Map getResourceTypeIndex() { /** * Get the resource types to be supported by the system. - * @return A map of the resource name to a ResouceInformation object + * @return A map of the resource name to a ResourceInformation object * which contains details such as the unit. */ public static Map getResourceTypes() { @@ -473,10 +473,10 @@ private static void addResourcesFileToConf(String resourceFile, LOG.debug("Found {}, adding to configuration", resourceFile); conf.addResource(ris); } catch (FileNotFoundException fe) { - LOG.info("Unable to find '" + resourceFile + "'."); + LOG.info("Unable to find '{}'.", resourceFile); } catch (IOException | YarnException ex) { - LOG.error("Exception trying to read resource types configuration '" - + resourceFile + "'.", ex); + LOG.error("Exception trying to read resource types configuration '{}'.", + resourceFile, ex); throw new YarnRuntimeException(ex); } } @@ -668,7 +668,7 @@ public static List getResourcesTypeInfo() { /** * Reinitialize all resource types from external source (in case of client, * server will send the updated list and local resourceutils cache will be - * updated as per server's list of resources) + * updated as per server's list of resources). * * @param resourceTypeInfo * List of resource types @@ -857,6 +857,7 @@ private static Map parseResourcesString(String resourcesStr) { units = "Gi"; } else if (units.isEmpty()) { // do nothing; + LOG.debug("units is empty."); } else { throw new IllegalArgumentException("Acceptable units are M/G or empty"); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-catalog/hadoop-yarn-applications-catalog-webapp/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-catalog/hadoop-yarn-applications-catalog-webapp/pom.xml index 8ad0fc6931679..21b3172a59f22 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-catalog/hadoop-yarn-applications-catalog-webapp/pom.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-catalog/hadoop-yarn-applications-catalog-webapp/pom.xml @@ -261,11 +261,6 @@ jackson-jaxrs-base ${jackson2.version} - - - javax.ws.rs - javax.ws.rs-api - ${artifact.name} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-catalog/hadoop-yarn-applications-catalog-webapp/src/main/java/org/apache/hadoop/yarn/appcatalog/application/YarnServiceClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-catalog/hadoop-yarn-applications-catalog-webapp/src/main/java/org/apache/hadoop/yarn/appcatalog/application/YarnServiceClient.java index 3a6c67d73700f..79838a9e4fb61 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-catalog/hadoop-yarn-applications-catalog-webapp/src/main/java/org/apache/hadoop/yarn/appcatalog/application/YarnServiceClient.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-catalog/hadoop-yarn-applications-catalog-webapp/src/main/java/org/apache/hadoop/yarn/appcatalog/application/YarnServiceClient.java @@ -61,7 +61,7 @@ public YarnServiceClient() { try { asc = new ApiServiceClient(conf); } catch (Exception e) { - LOG.error("Error initialize YARN Service Client: {}", e); + LOG.error("Error initialize YARN Service Client.", e); } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/pom.xml index 7e468bc75684e..d1cd362c7d65d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/pom.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/pom.xml @@ -166,10 +166,6 @@ - - javax.ws.rs - javax.ws.rs-api - diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/pom.xml index 4daac9fa235ae..746abf385e6bd 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/pom.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/pom.xml @@ -137,11 +137,6 @@ jackson-annotations - - javax.ws.rs - javax.ws.rs-api - - org.apache.hadoop hadoop-hdfs-client diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceScheduler.java index 6d143a3bd4aa0..9da8f31fe4b8b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceScheduler.java @@ -1139,7 +1139,7 @@ public void syncSysFs(Service yarnApp) { LOG.info("YARN sysfs synchronized."); } } catch (IOException | URISyntaxException | InterruptedException e) { - LOG.error("Fail to sync service spec: {}", e); + LOG.error("Fail to sync service spec.", e); } } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/TestServiceAM.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/TestServiceAM.java index c9886584bf27b..69c0c2cee57f9 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/TestServiceAM.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/TestServiceAM.java @@ -553,7 +553,7 @@ public void testSyncSysFS() { am.stop(); am.close(); } catch (Exception e) { - LOG.error("Fail to sync sysfs: {}", e); + LOG.error("Fail to sync sysfs.", e); Assert.fail("Fail to sync sysfs."); } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/TestFederationRMFailoverProxyProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/TestFederationRMFailoverProxyProvider.java index 0a7ee3ff734c1..74cf5152d5a88 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/TestFederationRMFailoverProxyProvider.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/TestFederationRMFailoverProxyProvider.java @@ -107,7 +107,8 @@ private void testProxyProvider(boolean facadeFlushCache) throws Exception { conf.setBoolean(YarnConfiguration.FEDERATION_FLUSH_CACHE_FOR_RM_ADDR, facadeFlushCache); - conf.setBoolean(YarnConfiguration.RM_HA_ENABLED, true); + conf.setBoolean(YarnConfiguration.FEDERATION_ENABLED, true); + conf.setBoolean(YarnConfiguration.FEDERATION_FAILOVER_ENABLED, true); conf.setBoolean(YarnConfiguration.AUTO_FAILOVER_ENABLED, false); conf.set(YarnConfiguration.RM_CLUSTER_ID, "cluster1"); conf.set(YarnConfiguration.RM_HA_IDS, "rm1,rm2,rm3"); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/pom.xml index c6df7594fcc3f..a04ff52be9749 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/pom.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/pom.xml @@ -197,10 +197,6 @@ com.fasterxml.jackson.jaxrs jackson-jaxrs-json-provider - - javax.ws.rs - javax.ws.rs-api - diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/RMProxy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/RMProxy.java index e3be31c10a1c3..95c8f2a01cef1 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/RMProxy.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/RMProxy.java @@ -98,8 +98,7 @@ protected static T createRMProxy(final Configuration configuration, YarnConfiguration conf = (configuration instanceof YarnConfiguration) ? (YarnConfiguration) configuration : new YarnConfiguration(configuration); - RetryPolicy retryPolicy = createRetryPolicy(conf, - (HAUtil.isHAEnabled(conf) || HAUtil.isFederationFailoverEnabled(conf))); + RetryPolicy retryPolicy = createRetryPolicy(conf, isFailoverEnabled(conf)); return newProxyInstance(conf, protocol, instance, retryPolicy); } @@ -126,7 +125,7 @@ private static T newProxyInstance(final YarnConfiguration conf, final Class protocol, RMProxy instance, RetryPolicy retryPolicy) throws IOException{ RMFailoverProxyProvider provider; - if (HAUtil.isHAEnabled(conf) || HAUtil.isFederationEnabled(conf)) { + if (isFailoverEnabled(conf)) { provider = instance.createRMFailoverProxyProvider(conf, protocol); } else { provider = instance.createNonHaRMFailoverProxyProvider(conf, protocol); @@ -300,7 +299,20 @@ protected static RetryPolicy createRetryPolicy(Configuration conf, // YARN-4288: local IOException is also possible. exceptionToPolicyMap.put(IOException.class, retryPolicy); // Not retry on remote IO exception. - return RetryPolicies.retryOtherThanRemoteException( + return RetryPolicies.retryOtherThanRemoteAndSaslException( RetryPolicies.TRY_ONCE_THEN_FAIL, exceptionToPolicyMap); } + + private static boolean isFailoverEnabled(YarnConfiguration conf) { + if (HAUtil.isHAEnabled(conf)) { + // Considering Resource Manager HA is enabled. + return true; + } + if (HAUtil.isFederationEnabled(conf) && HAUtil.isFederationFailoverEnabled(conf)) { + // Considering both federation and federation failover are enabled. + return true; + } + return false; + } + } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/event/AsyncDispatcher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/event/AsyncDispatcher.java index 1c4ed24b47d78..eee927c8c7ce5 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/event/AsyncDispatcher.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/event/AsyncDispatcher.java @@ -191,8 +191,21 @@ protected void serviceInit(Configuration conf) throws Exception{ .build(); // Thread pool for async print event details, // to prevent wasting too much time for RM. + int numCorePoolSizeThreads = getConfig().getInt( + YarnConfiguration.YARN_DISPATCHER_PRINT_THREAD_POOL_CORE_POOL_SIZE, + YarnConfiguration.DEFAULT_YARN_DISPATCHER_PRINT_THREAD_POOL_CORE_POOL_SIZE); + + int numMaximumPoolSizeThreads = getConfig().getInt( + YarnConfiguration.YARN_DISPATCHER_PRINT_THREAD_POOL_MAXIMUM_POOL_SIZE, + YarnConfiguration.DEFAULT_YARN_DISPATCHER_PRINT_THREAD_POOL_MAXIMUM_POOL_SIZE); + + long keepAliveTime = + conf.getTimeDuration(YarnConfiguration.YARN_DISPATCHER_PRINT_THREAD_POOL_KEEP_ALIVE_TIME, + YarnConfiguration.DEFAULT_YARN_DISPATCHER_PRINT_THREAD_POOL_KEEP_ALIVE_TIME, + TimeUnit.SECONDS); + printEventDetailsExecutor = new ThreadPoolExecutor( - 1, 5, 10, TimeUnit.SECONDS, + numCorePoolSizeThreads, numMaximumPoolSizeThreads, keepAliveTime, TimeUnit.SECONDS, new LinkedBlockingQueue<>(), threadFactory); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml index 407ef74d3d062..92a7298373879 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml @@ -133,6 +133,41 @@ 60 + + + Resource manager dispatcher has a thread pool that prints EventQueue, + configure the corePoolSize of this thread pool, + the meaning of corePoolSize is the number of threads to keep in the pool. + the default value is 1. + + yarn.dispatcher.print-thread-pool.core-pool-size + 1 + + + + + Resource manager dispatcher has a thread pool that prints EventQueue, + configure the maximumPoolSize of this thread pool, + the meaning of maximumPoolSize is the maximum number of threads to allow in the pool. + the default value is 5. + + yarn.dispatcher.print-thread-pool.maximum-pool-size + 5 + + + + + Resource manager dispatcher has a thread pool that prints EventQueue, + configure the keepAliveTime of this thread pool, + The meaning of keepAliveTime is as follows + when the number of threads is greater than the core, + this is the maximum time that excess idle threads will wait for new tasks before terminating. + the default value is 10s. + + yarn.dispatcher.print-thread-pool.keep-alive-time + 10s + + The expiry interval for application master reporting. yarn.am.liveness-monitor.expiry-interval-ms @@ -1077,6 +1112,18 @@ 86400000 + + + This configuration is used for + how often the tokens are scanned for expired tokens in milliseconds. + the background thread(delegation token remover thread) + will delete expired tokens after the configured time. + the default value is 1h. + + yarn.resourcemanager.delegation.token.remove-scan-interval + 1h + + RM DelegationTokenRenewer thread timeout @@ -1548,6 +1595,15 @@ 600000 + + Whether to clean up nodemanager logs when log aggregation is enabled. Setting to + false disables the cleanup nodemanager logging, and it causes disk full in the long run. Users + can set to false for test-only purpose. + + yarn.log-aggregation.enable-local-cleanup + true + + Time in seconds to retain user logs. Only applicable if log aggregation is disabled @@ -3624,6 +3680,16 @@ yarn.federation.enabled false + + + Initial delay for federation state-store heartbeat service. Value is followed by a unit + specifier: ns, us, ms, s, m, h, d for nanoseconds, microseconds, milliseconds, seconds, + minutes, hours, days respectively. Values should provide units, + but seconds are assumed + + yarn.federation.state-store.heartbeat.initial-delay + 30s + Machine list file to be loaded by the FederationSubCluster Resolver @@ -4888,4 +4954,57 @@ default implementation LocalityAppPlacementAllocator is used. + + + yarn.router.keytab.file + + + The keytab file used by router to login as its + service principal. The principal name is configured with + dfs.federation.router.kerberos.principal. + + + + + yarn.router.kerberos.principal + + + The Router service principal. This is typically set to + router/_HOST@REALM.TLD. Each Router will substitute _HOST with its + own fully qualified hostname at startup. The _HOST placeholder + allows using the same configuration setting on both Router setup. + + + + + yarn.router.kerberos.principal.hostname + + + Optional. + The hostname for the Router containing this + configuration file. Will be different for each machine. + Defaults to current hostname. + + + + + yarn.router.webapp.appsinfo-enabled + false + + This configuration is used to enable the cache of AppsInfo. + If it is set to true, the cache is enabled. + If it is set to false, the cache is not enabled. + + + + + yarn.router.webapp.appsinfo-cached-count + 100 + + When yarn.router.appsinfo-enabled is set to true, + the number of cached appsInfo. + Default is 100 + + + diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-csi/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-csi/pom.xml index 75165f696a169..953791e01e9ff 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-csi/pom.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-csi/pom.xml @@ -91,6 +91,16 @@ assertj-core test + + org.junit.jupiter + junit-jupiter-api + test + + + org.junit.jupiter + junit-jupiter-engine + test + org.apache.hadoop hadoop-common diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-csi/src/test/java/org/apache/hadoop/yarn/csi/adaptor/TestCsiAdaptorService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-csi/src/test/java/org/apache/hadoop/yarn/csi/adaptor/TestCsiAdaptorService.java index 206fb1433a1d6..cc14711397366 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-csi/src/test/java/org/apache/hadoop/yarn/csi/adaptor/TestCsiAdaptorService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-csi/src/test/java/org/apache/hadoop/yarn/csi/adaptor/TestCsiAdaptorService.java @@ -42,10 +42,9 @@ import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.ipc.YarnRPC; -import org.junit.AfterClass; -import org.junit.Assert; -import org.junit.BeforeClass; -import org.junit.Test; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; import java.io.File; import java.io.IOException; @@ -53,6 +52,9 @@ import java.net.ServerSocket; import static org.assertj.core.api.Assertions.assertThat; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; import static org.apache.hadoop.yarn.api.protocolrecords.ValidateVolumeCapabilitiesRequest.AccessMode.MULTI_NODE_MULTI_WRITER; import static org.apache.hadoop.yarn.api.protocolrecords.ValidateVolumeCapabilitiesRequest.VolumeType.FILE_SYSTEM; @@ -64,7 +66,7 @@ public class TestCsiAdaptorService { private static File testRoot = null; private static String domainSocket = null; - @BeforeClass + @BeforeAll public static void setUp() throws IOException { testRoot = GenericTestUtils.getTestDir("csi-test"); File socketPath = new File(testRoot, "csi.sock"); @@ -72,7 +74,7 @@ public static void setUp() throws IOException { domainSocket = "unix://" + socketPath.getAbsolutePath(); } - @AfterClass + @AfterAll public static void tearDown() throws IOException { if (testRoot != null) { FileUtils.deleteDirectory(testRoot); @@ -113,7 +115,7 @@ default NodeUnpublishVolumeResponse nodeUnpublishVolume( } @Test - public void testValidateVolume() throws IOException, YarnException { + void testValidateVolume() throws IOException, YarnException { ServerSocket ss = new ServerSocket(0); ss.close(); InetSocketAddress address = new InetSocketAddress(ss.getLocalPort()); @@ -145,20 +147,20 @@ public ValidateVolumeCapabilitiesResponse validateVolumeCapacity( ValidateVolumeCapabilitiesRequest request) throws YarnException, IOException { // validate we get all info from the request - Assert.assertEquals("volume-id-0000123", request.getVolumeId()); - Assert.assertEquals(1, request.getVolumeCapabilities().size()); - Assert.assertEquals(Csi.VolumeCapability.AccessMode + assertEquals("volume-id-0000123", request.getVolumeId()); + assertEquals(1, request.getVolumeCapabilities().size()); + assertEquals(Csi.VolumeCapability.AccessMode .newBuilder().setModeValue(5).build().getMode().name(), request.getVolumeCapabilities().get(0).getAccessMode().name()); - Assert.assertEquals(2, request.getVolumeCapabilities().get(0) + assertEquals(2, request.getVolumeCapabilities().get(0) .getMountFlags().size()); - Assert.assertTrue(request.getVolumeCapabilities().get(0) + assertTrue(request.getVolumeCapabilities().get(0) .getMountFlags().contains("mountFlag1")); - Assert.assertTrue(request.getVolumeCapabilities().get(0) + assertTrue(request.getVolumeCapabilities().get(0) .getMountFlags().contains("mountFlag2")); - Assert.assertEquals(2, request.getVolumeAttributes().size()); - Assert.assertEquals("v1", request.getVolumeAttributes().get("k1")); - Assert.assertEquals("v2", request.getVolumeAttributes().get("k2")); + assertEquals(2, request.getVolumeAttributes().size()); + assertEquals("v1", request.getVolumeAttributes().get("k1")); + assertEquals("v2", request.getVolumeAttributes().get("k2")); // return a fake result return ValidateVolumeCapabilitiesResponse .newInstance(false, "this is a test"); @@ -178,22 +180,22 @@ public ValidateVolumeCapabilitiesResponse validateVolumeCapacity( ImmutableList.of( new ValidateVolumeCapabilitiesRequest .VolumeCapability( - MULTI_NODE_MULTI_WRITER, FILE_SYSTEM, + MULTI_NODE_MULTI_WRITER, FILE_SYSTEM, ImmutableList.of("mountFlag1", "mountFlag2"))), - ImmutableMap.of("k1", "v1", "k2", "v2")); + ImmutableMap.of("k1", "v1", "k2", "v2")); ValidateVolumeCapabilitiesResponse response = client .validateVolumeCapacity(request); - Assert.assertEquals(false, response.isSupported()); - Assert.assertEquals("this is a test", response.getResponseMessage()); + assertEquals(false, response.isSupported()); + assertEquals("this is a test", response.getResponseMessage()); } finally { service.stop(); } } @Test - public void testValidateVolumeWithNMProxy() throws Exception { + void testValidateVolumeWithNMProxy() throws Exception { ServerSocket ss = new ServerSocket(0); ss.close(); InetSocketAddress address = new InetSocketAddress(ss.getLocalPort()); @@ -225,21 +227,21 @@ public ValidateVolumeCapabilitiesResponse validateVolumeCapacity( ValidateVolumeCapabilitiesRequest request) throws YarnException, IOException { // validate we get all info from the request - Assert.assertEquals("volume-id-0000123", request.getVolumeId()); - Assert.assertEquals(1, request.getVolumeCapabilities().size()); - Assert.assertEquals( + assertEquals("volume-id-0000123", request.getVolumeId()); + assertEquals(1, request.getVolumeCapabilities().size()); + assertEquals( Csi.VolumeCapability.AccessMode.newBuilder().setModeValue(5) .build().getMode().name(), request.getVolumeCapabilities().get(0).getAccessMode().name()); - Assert.assertEquals(2, + assertEquals(2, request.getVolumeCapabilities().get(0).getMountFlags().size()); - Assert.assertTrue(request.getVolumeCapabilities().get(0).getMountFlags() + assertTrue(request.getVolumeCapabilities().get(0).getMountFlags() .contains("mountFlag1")); - Assert.assertTrue(request.getVolumeCapabilities().get(0).getMountFlags() + assertTrue(request.getVolumeCapabilities().get(0).getMountFlags() .contains("mountFlag2")); - Assert.assertEquals(2, request.getVolumeAttributes().size()); - Assert.assertEquals("v1", request.getVolumeAttributes().get("k1")); - Assert.assertEquals("v2", request.getVolumeAttributes().get("k2")); + assertEquals(2, request.getVolumeAttributes().size()); + assertEquals("v1", request.getVolumeAttributes().get("k1")); + assertEquals("v2", request.getVolumeAttributes().get("k2")); // return a fake result return ValidateVolumeCapabilitiesResponse .newInstance(false, "this is a test"); @@ -261,50 +263,59 @@ public ValidateVolumeCapabilitiesResponse validateVolumeCapacity( .newInstance("volume-id-0000123", ImmutableList.of(new ValidateVolumeCapabilitiesRequest .VolumeCapability( - MULTI_NODE_MULTI_WRITER, FILE_SYSTEM, + MULTI_NODE_MULTI_WRITER, FILE_SYSTEM, ImmutableList.of("mountFlag1", "mountFlag2"))), ImmutableMap.of("k1", "v1", "k2", "v2")); ValidateVolumeCapabilitiesResponse response = adaptorClient .validateVolumeCapacity(request); - Assert.assertEquals(false, response.isSupported()); - Assert.assertEquals("this is a test", response.getResponseMessage()); + assertEquals(false, response.isSupported()); + assertEquals("this is a test", response.getResponseMessage()); service.stop(); } - @Test (expected = ServiceStateException.class) - public void testMissingConfiguration() { - Configuration conf = new Configuration(); - CsiAdaptorProtocolService service = - new CsiAdaptorProtocolService(new FakeCsiAdaptor() {}); - service.init(conf); + @Test + void testMissingConfiguration() { + assertThrows(ServiceStateException.class, () -> { + Configuration conf = new Configuration(); + CsiAdaptorProtocolService service = + new CsiAdaptorProtocolService(new FakeCsiAdaptor() { + }); + service.init(conf); + }); } - @Test (expected = ServiceStateException.class) - public void testInvalidServicePort() { - Configuration conf = new Configuration(); - conf.set(YarnConfiguration.NM_CSI_ADAPTOR_PREFIX - + "test-driver-0001.address", - "0.0.0.0:-100"); // this is an invalid address - CsiAdaptorProtocolService service = - new CsiAdaptorProtocolService(new FakeCsiAdaptor() {}); - service.init(conf); + @Test + void testInvalidServicePort() { + assertThrows(ServiceStateException.class, () -> { + Configuration conf = new Configuration(); + conf.set(YarnConfiguration.NM_CSI_ADAPTOR_PREFIX + + "test-driver-0001.address", + "0.0.0.0:-100"); // this is an invalid address + CsiAdaptorProtocolService service = + new CsiAdaptorProtocolService(new FakeCsiAdaptor() { + }); + service.init(conf); + }); } - @Test (expected = ServiceStateException.class) - public void testInvalidHost() { - Configuration conf = new Configuration(); - conf.set(YarnConfiguration.NM_CSI_ADAPTOR_PREFIX - + "test-driver-0001.address", - "192.0.1:8999"); // this is an invalid ip address - CsiAdaptorProtocolService service = - new CsiAdaptorProtocolService(new FakeCsiAdaptor() {}); - service.init(conf); + @Test + void testInvalidHost() { + assertThrows(ServiceStateException.class, () -> { + Configuration conf = new Configuration(); + conf.set(YarnConfiguration.NM_CSI_ADAPTOR_PREFIX + + "test-driver-0001.address", + "192.0.1:8999"); // this is an invalid ip address + CsiAdaptorProtocolService service = + new CsiAdaptorProtocolService(new FakeCsiAdaptor() { + }); + service.init(conf); + }); } @Test - public void testCustomizedAdaptor() throws IOException, YarnException { + void testCustomizedAdaptor() throws IOException, YarnException { ServerSocket ss = new ServerSocket(0); ss.close(); InetSocketAddress address = new InetSocketAddress(ss.getLocalPort()); @@ -349,15 +360,15 @@ public void testCustomizedAdaptor() throws IOException, YarnException { ValidateVolumeCapabilitiesResponse response = adaptorClient .validateVolumeCapacity(request); - Assert.assertEquals(true, response.isSupported()); - Assert.assertEquals("verified via MockCsiAdaptor", + assertEquals(true, response.isSupported()); + assertEquals("verified via MockCsiAdaptor", response.getResponseMessage()); services.stop(); } @Test - public void testMultipleCsiAdaptors() throws IOException, YarnException { + void testMultipleCsiAdaptors() throws IOException, YarnException { ServerSocket driver1Addr = new ServerSocket(0); ServerSocket driver2Addr = new ServerSocket(0); @@ -374,22 +385,22 @@ public void testMultipleCsiAdaptors() throws IOException, YarnException { // customized-driver-1 conf.setSocketAddr(YarnConfiguration.NM_CSI_ADAPTOR_PREFIX - + "customized-driver-1.address", address1); + + "customized-driver-1.address", address1); conf.set(YarnConfiguration.NM_CSI_ADAPTOR_PREFIX - + "customized-driver-1.class", + + "customized-driver-1.class", "org.apache.hadoop.yarn.csi.adaptor.MockCsiAdaptor"); conf.set(YarnConfiguration.NM_CSI_DRIVER_PREFIX - + "customized-driver-1.endpoint", + + "customized-driver-1.endpoint", "unix:///tmp/customized-driver-1.sock"); // customized-driver-2 conf.setSocketAddr(YarnConfiguration.NM_CSI_ADAPTOR_PREFIX + "customized-driver-2.address", address2); conf.set(YarnConfiguration.NM_CSI_ADAPTOR_PREFIX - + "customized-driver-2.class", + + "customized-driver-2.class", "org.apache.hadoop.yarn.csi.adaptor.MockCsiAdaptor"); conf.set(YarnConfiguration.NM_CSI_DRIVER_PREFIX - + "customized-driver-2.endpoint", + + "customized-driver-2.endpoint", "unix:///tmp/customized-driver-2.sock"); driver1Addr.close(); @@ -427,8 +438,8 @@ public void testMultipleCsiAdaptors() throws IOException, YarnException { ValidateVolumeCapabilitiesResponse response = client1 .validateVolumeCapacity(request); - Assert.assertEquals(true, response.isSupported()); - Assert.assertEquals("verified via MockCsiAdaptor", + assertEquals(true, response.isSupported()); + assertEquals("verified via MockCsiAdaptor", response.getResponseMessage()); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-csi/src/test/java/org/apache/hadoop/yarn/csi/adaptor/TestGetPluginInfoRequestResponse.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-csi/src/test/java/org/apache/hadoop/yarn/csi/adaptor/TestGetPluginInfoRequestResponse.java index f1734c84b3774..a6591091e5a5d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-csi/src/test/java/org/apache/hadoop/yarn/csi/adaptor/TestGetPluginInfoRequestResponse.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-csi/src/test/java/org/apache/hadoop/yarn/csi/adaptor/TestGetPluginInfoRequestResponse.java @@ -21,8 +21,10 @@ import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetPluginInfoRequestPBImpl; import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetPluginInfoResponsePBImpl; import org.apache.hadoop.yarn.proto.CsiAdaptorProtos; -import org.junit.Assert; -import org.junit.Test; +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; /** * Verify the integrity of GetPluginInfoRequest and GetPluginInfoResponse. @@ -30,37 +32,37 @@ public class TestGetPluginInfoRequestResponse { @Test - public void testGetPluginInfoRequestPBRecord() { + void testGetPluginInfoRequestPBRecord() { CsiAdaptorProtos.GetPluginInfoRequest requestProto = CsiAdaptorProtos.GetPluginInfoRequest.newBuilder().build(); GetPluginInfoRequestPBImpl pbImpl = new GetPluginInfoRequestPBImpl(requestProto); - Assert.assertNotNull(pbImpl); - Assert.assertEquals(requestProto, pbImpl.getProto()); + assertNotNull(pbImpl); + assertEquals(requestProto, pbImpl.getProto()); } @Test - public void testGetPluginInfoResponsePBRecord() { + void testGetPluginInfoResponsePBRecord() { CsiAdaptorProtos.GetPluginInfoResponse responseProto = CsiAdaptorProtos.GetPluginInfoResponse.newBuilder() - .setName("test-driver") - .setVendorVersion("1.0.1") - .build(); + .setName("test-driver") + .setVendorVersion("1.0.1") + .build(); GetPluginInfoResponsePBImpl pbImpl = new GetPluginInfoResponsePBImpl(responseProto); - Assert.assertEquals("test-driver", pbImpl.getDriverName()); - Assert.assertEquals("1.0.1", pbImpl.getVersion()); - Assert.assertEquals(responseProto, pbImpl.getProto()); + assertEquals("test-driver", pbImpl.getDriverName()); + assertEquals("1.0.1", pbImpl.getVersion()); + assertEquals(responseProto, pbImpl.getProto()); GetPluginInfoResponse pbImpl2 = GetPluginInfoResponsePBImpl .newInstance("test-driver", "1.0.1"); - Assert.assertEquals("test-driver", pbImpl2.getDriverName()); - Assert.assertEquals("1.0.1", pbImpl2.getVersion()); + assertEquals("test-driver", pbImpl2.getDriverName()); + assertEquals("1.0.1", pbImpl2.getVersion()); CsiAdaptorProtos.GetPluginInfoResponse proto = ((GetPluginInfoResponsePBImpl) pbImpl2).getProto(); - Assert.assertEquals("test-driver", proto.getName()); - Assert.assertEquals("1.0.1", proto.getVendorVersion()); + assertEquals("test-driver", proto.getName()); + assertEquals("1.0.1", proto.getVendorVersion()); } } \ No newline at end of file diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-csi/src/test/java/org/apache/hadoop/yarn/csi/adaptor/TestNodePublishVolumeRequest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-csi/src/test/java/org/apache/hadoop/yarn/csi/adaptor/TestNodePublishVolumeRequest.java index 5f7fa8774a4a8..77c60fd6d1102 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-csi/src/test/java/org/apache/hadoop/yarn/csi/adaptor/TestNodePublishVolumeRequest.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-csi/src/test/java/org/apache/hadoop/yarn/csi/adaptor/TestNodePublishVolumeRequest.java @@ -21,8 +21,10 @@ import org.apache.hadoop.yarn.proto.CsiAdaptorProtos; import org.apache.hadoop.yarn.proto.CsiAdaptorProtos.VolumeCapability.AccessMode; import org.apache.hadoop.yarn.proto.CsiAdaptorProtos.VolumeCapability.VolumeType; -import org.junit.Assert; -import org.junit.Test; +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; /** * UT for NodePublishVolumeRequest. @@ -30,7 +32,7 @@ public class TestNodePublishVolumeRequest { @Test - public void testPBRecord() { + void testPBRecord() { CsiAdaptorProtos.VolumeCapability capability = CsiAdaptorProtos.VolumeCapability.newBuilder() .setAccessMode(AccessMode.MULTI_NODE_READER_ONLY) @@ -47,9 +49,9 @@ public void testPBRecord() { NodePublishVolumeRequestPBImpl pbImpl = new NodePublishVolumeRequestPBImpl(proto); - Assert.assertEquals("test-vol-000001", pbImpl.getVolumeId()); - Assert.assertEquals("/mnt/data", pbImpl.getTargetPath()); - Assert.assertEquals("/mnt/staging", pbImpl.getStagingPath()); - Assert.assertFalse(pbImpl.getReadOnly()); + assertEquals("test-vol-000001", pbImpl.getVolumeId()); + assertEquals("/mnt/data", pbImpl.getTargetPath()); + assertEquals("/mnt/staging", pbImpl.getStagingPath()); + assertFalse(pbImpl.getReadOnly()); } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-csi/src/test/java/org/apache/hadoop/yarn/csi/adaptor/TestValidateVolumeCapabilityRequest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-csi/src/test/java/org/apache/hadoop/yarn/csi/adaptor/TestValidateVolumeCapabilityRequest.java index 127ddb3383d91..0c1d243a62552 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-csi/src/test/java/org/apache/hadoop/yarn/csi/adaptor/TestValidateVolumeCapabilityRequest.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-csi/src/test/java/org/apache/hadoop/yarn/csi/adaptor/TestValidateVolumeCapabilityRequest.java @@ -26,11 +26,11 @@ import org.apache.hadoop.yarn.proto.CsiAdaptorProtos.VolumeCapability.AccessMode; import org.apache.hadoop.yarn.proto.CsiAdaptorProtos.VolumeCapability.VolumeType; import org.apache.hadoop.yarn.proto.YarnProtos; -import org.junit.Assert; -import org.junit.Test; +import org.junit.jupiter.api.Test; import static org.apache.hadoop.yarn.api.protocolrecords.ValidateVolumeCapabilitiesRequest.AccessMode.MULTI_NODE_MULTI_WRITER; import static org.apache.hadoop.yarn.api.protocolrecords.ValidateVolumeCapabilitiesRequest.VolumeType.FILE_SYSTEM; +import static org.junit.jupiter.api.Assertions.assertEquals; /** * UT for message exchanges. @@ -38,7 +38,7 @@ public class TestValidateVolumeCapabilityRequest { @Test - public void testPBRecord() { + void testPBRecord() { CsiAdaptorProtos.VolumeCapability vcProto = CsiAdaptorProtos.VolumeCapability.newBuilder() .setAccessMode(AccessMode.MULTI_NODE_MULTI_WRITER) @@ -64,22 +64,22 @@ public void testPBRecord() { ValidateVolumeCapabilitiesRequestPBImpl request = new ValidateVolumeCapabilitiesRequestPBImpl(requestProto); - Assert.assertEquals("volume-id-0000001", request.getVolumeId()); - Assert.assertEquals(2, request.getVolumeAttributes().size()); - Assert.assertEquals("value0", request.getVolumeAttributes().get("attr0")); - Assert.assertEquals("value1", request.getVolumeAttributes().get("attr1")); - Assert.assertEquals(1, request.getVolumeCapabilities().size()); + assertEquals("volume-id-0000001", request.getVolumeId()); + assertEquals(2, request.getVolumeAttributes().size()); + assertEquals("value0", request.getVolumeAttributes().get("attr0")); + assertEquals("value1", request.getVolumeAttributes().get("attr1")); + assertEquals(1, request.getVolumeCapabilities().size()); VolumeCapability vc = request.getVolumeCapabilities().get(0); - Assert.assertEquals(MULTI_NODE_MULTI_WRITER, vc.getAccessMode()); - Assert.assertEquals(FILE_SYSTEM, vc.getVolumeType()); - Assert.assertEquals(2, vc.getMountFlags().size()); + assertEquals(MULTI_NODE_MULTI_WRITER, vc.getAccessMode()); + assertEquals(FILE_SYSTEM, vc.getVolumeType()); + assertEquals(2, vc.getMountFlags().size()); - Assert.assertEquals(requestProto, request.getProto()); + assertEquals(requestProto, request.getProto()); } @Test - public void testNewInstance() { + void testNewInstance() { ValidateVolumeCapabilitiesRequest pbImpl = ValidateVolumeCapabilitiesRequestPBImpl .newInstance("volume-id-0000123", @@ -89,25 +89,25 @@ public void testNewInstance() { ImmutableList.of("mountFlag1", "mountFlag2"))), ImmutableMap.of("k1", "v1", "k2", "v2")); - Assert.assertEquals("volume-id-0000123", pbImpl.getVolumeId()); - Assert.assertEquals(1, pbImpl.getVolumeCapabilities().size()); - Assert.assertEquals(FILE_SYSTEM, + assertEquals("volume-id-0000123", pbImpl.getVolumeId()); + assertEquals(1, pbImpl.getVolumeCapabilities().size()); + assertEquals(FILE_SYSTEM, pbImpl.getVolumeCapabilities().get(0).getVolumeType()); - Assert.assertEquals(MULTI_NODE_MULTI_WRITER, + assertEquals(MULTI_NODE_MULTI_WRITER, pbImpl.getVolumeCapabilities().get(0).getAccessMode()); - Assert.assertEquals(2, pbImpl.getVolumeAttributes().size()); + assertEquals(2, pbImpl.getVolumeAttributes().size()); CsiAdaptorProtos.ValidateVolumeCapabilitiesRequest proto = ((ValidateVolumeCapabilitiesRequestPBImpl) pbImpl).getProto(); - Assert.assertEquals("volume-id-0000123", proto.getVolumeId()); - Assert.assertEquals(1, proto.getVolumeCapabilitiesCount()); - Assert.assertEquals(AccessMode.MULTI_NODE_MULTI_WRITER, + assertEquals("volume-id-0000123", proto.getVolumeId()); + assertEquals(1, proto.getVolumeCapabilitiesCount()); + assertEquals(AccessMode.MULTI_NODE_MULTI_WRITER, proto.getVolumeCapabilities(0).getAccessMode()); - Assert.assertEquals(VolumeType.FILE_SYSTEM, + assertEquals(VolumeType.FILE_SYSTEM, proto.getVolumeCapabilities(0).getVolumeType()); - Assert.assertEquals(2, proto.getVolumeCapabilities(0) + assertEquals(2, proto.getVolumeCapabilities(0) .getMountFlagsCount()); - Assert.assertEquals(2, proto.getVolumeCapabilities(0) + assertEquals(2, proto.getVolumeCapabilities(0) .getMountFlagsList().size()); } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-csi/src/test/java/org/apache/hadoop/yarn/csi/adaptor/TestValidateVolumeCapabilityResponse.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-csi/src/test/java/org/apache/hadoop/yarn/csi/adaptor/TestValidateVolumeCapabilityResponse.java index 97f116af0d5cc..ee80bf4d0c648 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-csi/src/test/java/org/apache/hadoop/yarn/csi/adaptor/TestValidateVolumeCapabilityResponse.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-csi/src/test/java/org/apache/hadoop/yarn/csi/adaptor/TestValidateVolumeCapabilityResponse.java @@ -20,8 +20,9 @@ import org.apache.hadoop.yarn.api.protocolrecords.ValidateVolumeCapabilitiesResponse; import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.ValidateVolumeCapabilitiesResponsePBImpl; import org.apache.hadoop.yarn.proto.CsiAdaptorProtos; -import org.junit.Assert; -import org.junit.Test; +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertEquals; /** * UT for message exchanges. @@ -29,33 +30,33 @@ public class TestValidateVolumeCapabilityResponse { @Test - public void testPBRecord() { + void testPBRecord() { CsiAdaptorProtos.ValidateVolumeCapabilitiesResponse proto = CsiAdaptorProtos.ValidateVolumeCapabilitiesResponse.newBuilder() - .setSupported(true) - .setMessage("capability is supported") - .build(); + .setSupported(true) + .setMessage("capability is supported") + .build(); ValidateVolumeCapabilitiesResponsePBImpl pbImpl = new ValidateVolumeCapabilitiesResponsePBImpl(proto); - Assert.assertEquals(true, pbImpl.isSupported()); - Assert.assertEquals("capability is supported", pbImpl.getResponseMessage()); - Assert.assertEquals(proto, pbImpl.getProto()); + assertEquals(true, pbImpl.isSupported()); + assertEquals("capability is supported", pbImpl.getResponseMessage()); + assertEquals(proto, pbImpl.getProto()); } @Test - public void testNewInstance() { + void testNewInstance() { ValidateVolumeCapabilitiesResponse pbImpl = ValidateVolumeCapabilitiesResponsePBImpl .newInstance(false, "capability not supported"); - Assert.assertEquals(false, pbImpl.isSupported()); - Assert.assertEquals("capability not supported", + assertEquals(false, pbImpl.isSupported()); + assertEquals("capability not supported", pbImpl.getResponseMessage()); CsiAdaptorProtos.ValidateVolumeCapabilitiesResponse proto = ((ValidateVolumeCapabilitiesResponsePBImpl) pbImpl).getProto(); - Assert.assertEquals(false, proto.getSupported()); - Assert.assertEquals("capability not supported", proto.getMessage()); + assertEquals(false, proto.getSupported()); + assertEquals("capability not supported", proto.getMessage()); } } \ No newline at end of file diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-csi/src/test/java/org/apache/hadoop/yarn/csi/client/TestCsiClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-csi/src/test/java/org/apache/hadoop/yarn/csi/client/TestCsiClient.java index ffbfa46590e09..059cc90c4f36e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-csi/src/test/java/org/apache/hadoop/yarn/csi/client/TestCsiClient.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-csi/src/test/java/org/apache/hadoop/yarn/csi/client/TestCsiClient.java @@ -20,12 +20,11 @@ import csi.v0.Csi; import org.apache.commons.io.FileUtils; -import org.junit.AfterClass; -import org.junit.Assert; -import org.junit.Assume; -import org.junit.Before; -import org.junit.BeforeClass; -import org.junit.Test; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.Assumptions; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -33,6 +32,8 @@ import java.io.IOException; import java.nio.file.Files; +import static org.junit.jupiter.api.Assertions.assertEquals; + /** * Test class for CSI client. */ @@ -42,7 +43,7 @@ public class TestCsiClient { private static String domainSocket = null; private static FakeCsiDriver driver = null; - @BeforeClass + @BeforeAll public static void setUp() throws IOException { // Use /tmp to fix bind failure caused by the long file name File tmpDir = new File(System.getProperty("java.io.tmpdir")); @@ -55,27 +56,27 @@ public static void setUp() throws IOException { driver = new FakeCsiDriver(domainSocket); } - @AfterClass + @AfterAll public static void tearDown() throws IOException { if (testRoot != null) { FileUtils.deleteDirectory(testRoot); } } - @Before + @BeforeEach public void beforeMethod() { // Skip tests on non-linux systems String osName = System.getProperty("os.name").toLowerCase(); - Assume.assumeTrue(osName.contains("nix") || osName.contains("nux")); + Assumptions.assumeTrue(osName.contains("nix") || osName.contains("nux")); } @Test - public void testIdentityService() throws IOException { + void testIdentityService() throws IOException { try { driver.start(); CsiClient client = new CsiClientImpl(domainSocket); Csi.GetPluginInfoResponse response = client.getPluginInfo(); - Assert.assertEquals("fake-csi-identity-service", response.getName()); + assertEquals("fake-csi-identity-service", response.getName()); } finally { driver.stop(); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/pom.xml index 2a21c0dc092fc..d0fd79aaa78e1 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/pom.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/pom.xml @@ -34,6 +34,8 @@ ${project.parent.parent.basedir} + + javax.servlet @@ -194,6 +196,26 @@ bcprov-jdk15on test + + org.junit.jupiter + junit-jupiter-api + test + + + org.junit.jupiter + junit-jupiter-engine + test + + + org.junit.jupiter + junit-jupiter-params + test + + + org.junit.platform + junit-platform-launcher + test + de.ruedigermoeller fst @@ -205,11 +227,6 @@ - - - javax.ws.rs - javax.ws.rs-api - diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryClientService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryClientService.java index 7ef6eca0dcad6..86b15b3412833 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryClientService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryClientService.java @@ -21,6 +21,9 @@ import java.io.IOException; import java.util.List; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; + import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationAttemptReportRequest; import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationAttemptReportResponse; @@ -49,9 +52,11 @@ import org.apache.hadoop.yarn.server.timeline.TimelineDataManager; import org.apache.hadoop.yarn.server.timeline.TimelineStore; import org.apache.hadoop.yarn.server.timeline.security.TimelineACLsManager; -import org.junit.Assert; -import org.junit.BeforeClass; -import org.junit.Test; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.fail; public class TestApplicationHistoryClientService { @@ -59,7 +64,7 @@ public class TestApplicationHistoryClientService { private static TimelineDataManager dataManager; private final static int MAX_APPS = 2; - @BeforeClass + @BeforeAll public static void setup() throws Exception { Configuration conf = new YarnConfiguration(); TimelineStore store = @@ -78,7 +83,7 @@ public static void setup() throws Exception { } @Test - public void testApplicationNotFound() throws IOException, YarnException { + void testApplicationNotFound() throws IOException, YarnException { ApplicationId appId = null; appId = ApplicationId.newInstance(0, MAX_APPS + 1); GetApplicationReportRequest request = @@ -87,18 +92,18 @@ public void testApplicationNotFound() throws IOException, YarnException { @SuppressWarnings("unused") GetApplicationReportResponse response = clientService.getApplicationReport(request); - Assert.fail("Exception should have been thrown before we reach here."); + fail("Exception should have been thrown before we reach here."); } catch (ApplicationNotFoundException e) { //This exception is expected. - Assert.assertTrue(e.getMessage().contains( + assertTrue(e.getMessage().contains( "doesn't exist in the timeline store")); } catch (Exception e) { - Assert.fail("Undesired exception caught"); + fail("Undesired exception caught"); } } @Test - public void testApplicationAttemptNotFound() throws IOException, YarnException { + void testApplicationAttemptNotFound() throws IOException, YarnException { ApplicationId appId = ApplicationId.newInstance(0, 1); ApplicationAttemptId appAttemptId = ApplicationAttemptId.newInstance(appId, MAX_APPS + 1); @@ -108,41 +113,41 @@ public void testApplicationAttemptNotFound() throws IOException, YarnException { @SuppressWarnings("unused") GetApplicationAttemptReportResponse response = clientService.getApplicationAttemptReport(request); - Assert.fail("Exception should have been thrown before we reach here."); + fail("Exception should have been thrown before we reach here."); } catch (ApplicationAttemptNotFoundException e) { //This Exception is expected System.out.println(e.getMessage()); - Assert.assertTrue(e.getMessage().contains( + assertTrue(e.getMessage().contains( "doesn't exist in the timeline store")); } catch (Exception e) { - Assert.fail("Undesired exception caught"); + fail("Undesired exception caught"); } } @Test - public void testContainerNotFound() throws IOException, YarnException { - ApplicationId appId = ApplicationId.newInstance(0, 1); - ApplicationAttemptId appAttemptId = - ApplicationAttemptId.newInstance(appId, 1); - ContainerId containerId = ContainerId.newContainerId(appAttemptId, - MAX_APPS + 1); - GetContainerReportRequest request = - GetContainerReportRequest.newInstance(containerId); - try { - @SuppressWarnings("unused") - GetContainerReportResponse response = - clientService.getContainerReport(request); - } catch (ContainerNotFoundException e) { - //This exception is expected - Assert.assertTrue(e.getMessage().contains( - "doesn't exist in the timeline store")); - } catch (Exception e) { - Assert.fail("Undesired exception caught"); - } - } + void testContainerNotFound() throws IOException, YarnException { + ApplicationId appId = ApplicationId.newInstance(0, 1); + ApplicationAttemptId appAttemptId = + ApplicationAttemptId.newInstance(appId, 1); + ContainerId containerId = ContainerId.newContainerId(appAttemptId, + MAX_APPS + 1); + GetContainerReportRequest request = + GetContainerReportRequest.newInstance(containerId); + try { + @SuppressWarnings("unused") + GetContainerReportResponse response = + clientService.getContainerReport(request); + } catch (ContainerNotFoundException e) { + //This exception is expected + assertTrue(e.getMessage().contains( + "doesn't exist in the timeline store")); + } catch (Exception e) { + fail("Undesired exception caught"); + } + } @Test - public void testApplicationReport() throws IOException, YarnException { + void testApplicationReport() throws IOException, YarnException { ApplicationId appId = null; appId = ApplicationId.newInstance(0, 1); GetApplicationReportRequest request = @@ -150,20 +155,20 @@ public void testApplicationReport() throws IOException, YarnException { GetApplicationReportResponse response = clientService.getApplicationReport(request); ApplicationReport appReport = response.getApplicationReport(); - Assert.assertNotNull(appReport); - Assert.assertEquals(123, appReport.getApplicationResourceUsageReport() + assertNotNull(appReport); + assertEquals(123, appReport.getApplicationResourceUsageReport() .getMemorySeconds()); - Assert.assertEquals(345, appReport.getApplicationResourceUsageReport() + assertEquals(345, appReport.getApplicationResourceUsageReport() .getVcoreSeconds()); - Assert.assertEquals("application_0_0001", appReport.getApplicationId() - .toString()); - Assert.assertEquals("test app type", + assertEquals("application_0_0001", appReport.getApplicationId() + .toString()); + assertEquals("test app type", appReport.getApplicationType().toString()); - Assert.assertEquals("test queue", appReport.getQueue().toString()); + assertEquals("test queue", appReport.getQueue().toString()); } @Test - public void testApplications() throws IOException, YarnException { + void testApplications() throws IOException, YarnException { ApplicationId appId = null; appId = ApplicationId.newInstance(0, 1); ApplicationId appId1 = ApplicationId.newInstance(0, 2); @@ -171,9 +176,9 @@ public void testApplications() throws IOException, YarnException { GetApplicationsResponse response = clientService.getApplications(request); List appReport = response.getApplicationList(); - Assert.assertNotNull(appReport); - Assert.assertEquals(appId, appReport.get(1).getApplicationId()); - Assert.assertEquals(appId1, appReport.get(0).getApplicationId()); + assertNotNull(appReport); + assertEquals(appId, appReport.get(1).getApplicationId()); + assertEquals(appId1, appReport.get(0).getApplicationId()); // Create a historyManager, and set the max_apps can be loaded // as 1. @@ -181,7 +186,7 @@ public void testApplications() throws IOException, YarnException { conf.setLong(YarnConfiguration.APPLICATION_HISTORY_MAX_APPS, 1); ApplicationHistoryManagerOnTimelineStore historyManager2 = new ApplicationHistoryManagerOnTimelineStore(dataManager, - new ApplicationACLsManager(conf)); + new ApplicationACLsManager(conf)); historyManager2.init(conf); historyManager2.start(); @SuppressWarnings("resource") @@ -189,14 +194,14 @@ public void testApplications() throws IOException, YarnException { new ApplicationHistoryClientService(historyManager2); response = clientService2.getApplications(request); appReport = response.getApplicationList(); - Assert.assertNotNull(appReport); - Assert.assertTrue(appReport.size() == 1); + assertNotNull(appReport); + assertTrue(appReport.size() == 1); // Expected to get the appReport for application with appId1 - Assert.assertEquals(appId1, appReport.get(0).getApplicationId()); + assertEquals(appId1, appReport.get(0).getApplicationId()); } @Test - public void testApplicationAttemptReport() throws IOException, YarnException { + void testApplicationAttemptReport() throws IOException, YarnException { ApplicationId appId = ApplicationId.newInstance(0, 1); ApplicationAttemptId appAttemptId = ApplicationAttemptId.newInstance(appId, 1); @@ -206,13 +211,13 @@ public void testApplicationAttemptReport() throws IOException, YarnException { clientService.getApplicationAttemptReport(request); ApplicationAttemptReport attemptReport = response.getApplicationAttemptReport(); - Assert.assertNotNull(attemptReport); - Assert.assertEquals("appattempt_0_0001_000001", attemptReport - .getApplicationAttemptId().toString()); + assertNotNull(attemptReport); + assertEquals("appattempt_0_0001_000001", attemptReport + .getApplicationAttemptId().toString()); } @Test - public void testApplicationAttempts() throws IOException, YarnException { + void testApplicationAttempts() throws IOException, YarnException { ApplicationId appId = ApplicationId.newInstance(0, 1); ApplicationAttemptId appAttemptId = ApplicationAttemptId.newInstance(appId, 1); @@ -224,15 +229,15 @@ public void testApplicationAttempts() throws IOException, YarnException { clientService.getApplicationAttempts(request); List attemptReports = response.getApplicationAttemptList(); - Assert.assertNotNull(attemptReports); - Assert.assertEquals(appAttemptId, attemptReports.get(0) - .getApplicationAttemptId()); - Assert.assertEquals(appAttemptId1, attemptReports.get(1) - .getApplicationAttemptId()); + assertNotNull(attemptReports); + assertEquals(appAttemptId, attemptReports.get(0) + .getApplicationAttemptId()); + assertEquals(appAttemptId1, attemptReports.get(1) + .getApplicationAttemptId()); } @Test - public void testContainerReport() throws IOException, YarnException { + void testContainerReport() throws IOException, YarnException { ApplicationId appId = ApplicationId.newInstance(0, 1); ApplicationAttemptId appAttemptId = ApplicationAttemptId.newInstance(appId, 1); @@ -242,15 +247,15 @@ public void testContainerReport() throws IOException, YarnException { GetContainerReportResponse response = clientService.getContainerReport(request); ContainerReport container = response.getContainerReport(); - Assert.assertNotNull(container); - Assert.assertEquals(containerId, container.getContainerId()); - Assert.assertEquals("http://0.0.0.0:8188/applicationhistory/logs/" + + assertNotNull(container); + assertEquals(containerId, container.getContainerId()); + assertEquals("http://0.0.0.0:8188/applicationhistory/logs/" + "test host:100/container_0_0001_01_000001/" + "container_0_0001_01_000001/user1", container.getLogUrl()); } @Test - public void testContainers() throws IOException, YarnException { + void testContainers() throws IOException, YarnException { ApplicationId appId = ApplicationId.newInstance(0, 1); ApplicationAttemptId appAttemptId = ApplicationAttemptId.newInstance(appId, 1); @@ -261,8 +266,8 @@ public void testContainers() throws IOException, YarnException { GetContainersResponse response = clientService.getContainers(request); List containers = response.getContainerList(); - Assert.assertNotNull(containers); - Assert.assertEquals(containerId, containers.get(0).getContainerId()); - Assert.assertEquals(containerId1, containers.get(1).getContainerId()); + assertNotNull(containers); + assertEquals(containerId, containers.get(0).getContainerId()); + assertEquals(containerId1, containers.get(1).getContainerId()); } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryManagerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryManagerImpl.java index da5ddc183b173..87a191d643bff 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryManagerImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryManagerImpl.java @@ -21,39 +21,43 @@ import java.io.IOException; import java.util.Map; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ApplicationReport; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.exceptions.YarnException; -import org.junit.After; -import org.junit.Assert; -import org.junit.Before; -import org.junit.Test; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; public class TestApplicationHistoryManagerImpl extends ApplicationHistoryStoreTestUtils { - ApplicationHistoryManagerImpl applicationHistoryManagerImpl = null; + private ApplicationHistoryManagerImpl applicationHistoryManagerImpl = null; - @Before + @BeforeEach public void setup() throws Exception { Configuration config = new Configuration(); config.setClass(YarnConfiguration.APPLICATION_HISTORY_STORE, - MemoryApplicationHistoryStore.class, ApplicationHistoryStore.class); + MemoryApplicationHistoryStore.class, ApplicationHistoryStore.class); applicationHistoryManagerImpl = new ApplicationHistoryManagerImpl(); applicationHistoryManagerImpl.init(config); applicationHistoryManagerImpl.start(); store = applicationHistoryManagerImpl.getHistoryStore(); } - @After + @AfterEach public void tearDown() throws Exception { applicationHistoryManagerImpl.stop(); } @Test - public void testApplicationReport() throws IOException, YarnException { + void testApplicationReport() throws IOException, YarnException { ApplicationId appId = null; appId = ApplicationId.newInstance(0, 1); writeApplicationStartData(appId); @@ -64,17 +68,17 @@ public void testApplicationReport() throws IOException, YarnException { writeApplicationAttemptFinishData(appAttemptId); ApplicationReport appReport = applicationHistoryManagerImpl.getApplication(appId); - Assert.assertNotNull(appReport); - Assert.assertEquals(appId, appReport.getApplicationId()); - Assert.assertEquals(appAttemptId, - appReport.getCurrentApplicationAttemptId()); - Assert.assertEquals(appAttemptId.toString(), appReport.getHost()); - Assert.assertEquals("test type", appReport.getApplicationType().toString()); - Assert.assertEquals("test queue", appReport.getQueue().toString()); + assertNotNull(appReport); + assertEquals(appId, appReport.getApplicationId()); + assertEquals(appAttemptId, + appReport.getCurrentApplicationAttemptId()); + assertEquals(appAttemptId.toString(), appReport.getHost()); + assertEquals("test type", appReport.getApplicationType().toString()); + assertEquals("test queue", appReport.getQueue().toString()); } @Test - public void testApplications() throws IOException { + void testApplications() throws IOException { ApplicationId appId1 = ApplicationId.newInstance(0, 1); ApplicationId appId2 = ApplicationId.newInstance(0, 2); ApplicationId appId3 = ApplicationId.newInstance(0, 3); @@ -86,10 +90,10 @@ public void testApplications() throws IOException { writeApplicationFinishData(appId3); Map reports = applicationHistoryManagerImpl.getApplications(2, 2000L, 5000L); - Assert.assertNotNull(reports); - Assert.assertEquals(2, reports.size()); - Assert.assertNull(reports.get("1")); - Assert.assertNull(reports.get("2")); - Assert.assertNull(reports.get("3")); + assertNotNull(reports); + assertEquals(2, reports.size()); + assertNull(reports.get("1")); + assertNull(reports.get("2")); + assertNull(reports.get("3")); } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryManagerOnTimelineStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryManagerOnTimelineStore.java index 96c81315704d8..a31d9a92419f4 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryManagerOnTimelineStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryManagerOnTimelineStore.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -26,6 +26,11 @@ import java.util.Map; import java.util.Set; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.MethodSource; + import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.security.SaslRpcServer.AuthMethod; import org.apache.hadoop.security.UserGroupInformation; @@ -56,16 +61,13 @@ import org.apache.hadoop.yarn.server.timeline.TimelineDataManager; import org.apache.hadoop.yarn.server.timeline.TimelineStore; import org.apache.hadoop.yarn.server.timeline.security.TimelineACLsManager; -import org.junit.After; -import org.junit.Assert; -import org.junit.Before; -import org.junit.BeforeClass; -import org.junit.Test; -import org.junit.runner.RunWith; -import org.junit.runners.Parameterized; -import org.junit.runners.Parameterized.Parameters; -@RunWith(Parameterized.class) +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.fail; + public class TestApplicationHistoryManagerOnTimelineStore { private static final int SCALE = 5; @@ -75,91 +77,78 @@ public class TestApplicationHistoryManagerOnTimelineStore { private UserGroupInformation callerUGI; private Configuration conf; - @BeforeClass + @BeforeAll public static void prepareStore() throws Exception { store = createStore(SCALE); TimelineEntities entities = new TimelineEntities(); - entities.addEntity(createApplicationTimelineEntity( - ApplicationId.newInstance(0, SCALE + 1), true, true, false, false, - YarnApplicationState.FINISHED)); - entities.addEntity(createApplicationTimelineEntity( - ApplicationId.newInstance(0, SCALE + 2), true, false, true, false, - YarnApplicationState.FINISHED)); + entities.addEntity( + createApplicationTimelineEntity(ApplicationId.newInstance(0, SCALE + 1), true, true, false, + false, YarnApplicationState.FINISHED)); + entities.addEntity( + createApplicationTimelineEntity(ApplicationId.newInstance(0, SCALE + 2), true, false, true, + false, YarnApplicationState.FINISHED)); store.put(entities); } public static TimelineStore createStore(int scale) throws Exception { - TimelineStore store = new MemoryTimelineStore(); - prepareTimelineStore(store, scale); + store = new MemoryTimelineStore(); + prepareTimelineStore(scale); return store; } - @Before - public void setup() throws Exception { - // Only test the ACLs of the generic history - TimelineACLsManager aclsManager = new TimelineACLsManager(new YarnConfiguration()); - aclsManager.setTimelineStore(store); - TimelineDataManager dataManager = - new TimelineDataManager(store, aclsManager); - dataManager.init(conf); - ApplicationACLsManager appAclsManager = new ApplicationACLsManager(conf); - historyManager = - new ApplicationHistoryManagerOnTimelineStore(dataManager, appAclsManager); - historyManager.init(conf); - historyManager.start(); - } - - @After + @AfterEach public void tearDown() { if (historyManager != null) { historyManager.stop(); } } - @Parameters public static Collection callers() { // user1 is the owner // user2 is the authorized user // user3 is the unauthorized user // admin is the admin acl - return Arrays.asList( - new Object[][] { { "" }, { "user1" }, { "user2" }, { "user3" }, { "admin" } }); + return Arrays.asList(new Object[][] {{""}, {"user1"}, {"user2"}, {"user3"}, {"admin"}}); } - public TestApplicationHistoryManagerOnTimelineStore(String caller) { + public void initTestApplicationHistoryManagerOnTimelineStore(String caller) { conf = new YarnConfiguration(); if (!caller.equals("")) { callerUGI = UserGroupInformation.createRemoteUser(caller, AuthMethod.SIMPLE); conf.setBoolean(YarnConfiguration.YARN_ACL_ENABLE, true); conf.set(YarnConfiguration.YARN_ADMIN_ACL, "admin"); } + TimelineACLsManager aclsManager = new TimelineACLsManager(new YarnConfiguration()); + aclsManager.setTimelineStore(store); + TimelineDataManager dataManager = new TimelineDataManager(store, aclsManager); + dataManager.init(conf); + ApplicationACLsManager appAclsManager = new ApplicationACLsManager(conf); + historyManager = new ApplicationHistoryManagerOnTimelineStore(dataManager, appAclsManager); + historyManager.init(conf); + historyManager.start(); } - private static void prepareTimelineStore(TimelineStore store, int scale) - throws Exception { + private static void prepareTimelineStore(int scale) throws Exception { for (int i = 1; i <= scale; ++i) { TimelineEntities entities = new TimelineEntities(); ApplicationId appId = ApplicationId.newInstance(0, i); if (i == 2) { - entities.addEntity(createApplicationTimelineEntity( - appId, true, false, false, true, YarnApplicationState.FINISHED)); + entities.addEntity(createApplicationTimelineEntity(appId, true, false, false, true, + YarnApplicationState.FINISHED)); } else if (i == 3) { - entities.addEntity(createApplicationTimelineEntity( - appId, false, false, false, false, YarnApplicationState.FINISHED, - true, false)); + entities.addEntity(createApplicationTimelineEntity(appId, false, false, false, false, + YarnApplicationState.FINISHED, true, false)); } else if (i == SCALE + 1) { - entities.addEntity(createApplicationTimelineEntity( - appId, false, false, false, false, YarnApplicationState.FINISHED, - false, true)); + entities.addEntity(createApplicationTimelineEntity(appId, false, false, false, false, + YarnApplicationState.FINISHED, false, true)); } else { - entities.addEntity(createApplicationTimelineEntity( - appId, false, false, false, false, YarnApplicationState.FINISHED)); + entities.addEntity(createApplicationTimelineEntity(appId, false, false, false, false, + YarnApplicationState.FINISHED)); } store.put(entities); for (int j = 1; j <= scale; ++j) { entities = new TimelineEntities(); - ApplicationAttemptId appAttemptId = - ApplicationAttemptId.newInstance(appId, j); + ApplicationAttemptId appAttemptId = ApplicationAttemptId.newInstance(appId, j); entities.addEntity(createAppAttemptTimelineEntity(appAttemptId)); store.put(entities); for (int k = 1; k <= scale; ++k) { @@ -172,129 +161,119 @@ private static void prepareTimelineStore(TimelineStore store, int scale) } TimelineEntities entities = new TimelineEntities(); ApplicationId appId = ApplicationId.newInstance(1234, 1); - ApplicationAttemptId appAttemptId = - ApplicationAttemptId.newInstance(appId, 1); + ApplicationAttemptId appAttemptId = ApplicationAttemptId.newInstance(appId, 1); ContainerId containerId = ContainerId.newContainerId(appAttemptId, 1); - entities.addEntity(createApplicationTimelineEntity( - appId, true, false, false, false, YarnApplicationState.RUNNING)); + entities.addEntity(createApplicationTimelineEntity(appId, true, false, false, false, + YarnApplicationState.RUNNING)); entities.addEntity(createAppAttemptTimelineEntity(appAttemptId)); entities.addEntity(createContainerEntity(containerId)); store.put(entities); } - @Test - public void testGetApplicationReport() throws Exception { + @MethodSource("callers") + @ParameterizedTest + void testGetApplicationReport(String caller) throws Exception { + initTestApplicationHistoryManagerOnTimelineStore(caller); for (int i = 1; i <= 3; ++i) { final ApplicationId appId = ApplicationId.newInstance(0, i); ApplicationReport app; if (callerUGI == null) { app = historyManager.getApplication(appId); } else { - app = - callerUGI.doAs(new PrivilegedExceptionAction () { + app = callerUGI.doAs(new PrivilegedExceptionAction() { @Override public ApplicationReport run() throws Exception { return historyManager.getApplication(appId); } }); } - Assert.assertNotNull(app); - Assert.assertEquals(appId, app.getApplicationId()); - Assert.assertEquals("test app", app.getName()); - Assert.assertEquals("test app type", app.getApplicationType()); - Assert.assertEquals("user1", app.getUser()); + assertNotNull(app); + assertEquals(appId, app.getApplicationId()); + assertEquals("test app", app.getName()); + assertEquals("test app type", app.getApplicationType()); + assertEquals("user1", app.getUser()); if (i == 2) { // Change event is fired only in case of app with ID 2, hence verify // with updated changes. And make sure last updated change is accepted. - Assert.assertEquals("changed queue1", app.getQueue()); - Assert.assertEquals(Priority.newInstance(6), app.getPriority()); + assertEquals("changed queue1", app.getQueue()); + assertEquals(Priority.newInstance(6), app.getPriority()); } else { - Assert.assertEquals("test queue", app.getQueue()); - Assert.assertEquals(Priority.newInstance(0), app.getPriority()); + assertEquals("test queue", app.getQueue()); + assertEquals(Priority.newInstance(0), app.getPriority()); } - Assert.assertEquals(Integer.MAX_VALUE + 2L - + app.getApplicationId().getId(), app.getStartTime()); - Assert.assertEquals(Integer.MAX_VALUE + 1L, app.getSubmitTime()); - Assert.assertEquals(Integer.MAX_VALUE + 3L - + +app.getApplicationId().getId(), app.getFinishTime()); - Assert.assertTrue(Math.abs(app.getProgress() - 1.0F) < 0.0001); - Assert.assertEquals(2, app.getApplicationTags().size()); - Assert.assertTrue(app.getApplicationTags().contains("Test_APP_TAGS_1")); - Assert.assertTrue(app.getApplicationTags().contains("Test_APP_TAGS_2")); + assertEquals(Integer.MAX_VALUE + 2L + app.getApplicationId().getId(), app.getStartTime()); + assertEquals(Integer.MAX_VALUE + 1L, app.getSubmitTime()); + assertEquals(Integer.MAX_VALUE + 3L + +app.getApplicationId().getId(), app.getFinishTime()); + assertTrue(Math.abs(app.getProgress() - 1.0F) < 0.0001); + assertEquals(2, app.getApplicationTags().size()); + assertTrue(app.getApplicationTags().contains("Test_APP_TAGS_1")); + assertTrue(app.getApplicationTags().contains("Test_APP_TAGS_2")); // App 2 doesn't have the ACLs, such that the default ACLs " " will be used. // Nobody except admin and owner has access to the details of the app. - if ((i != 2 && callerUGI != null && - callerUGI.getShortUserName().equals("user3")) || - (i == 2 && callerUGI != null && - (callerUGI.getShortUserName().equals("user2") || - callerUGI.getShortUserName().equals("user3")))) { - Assert.assertEquals(ApplicationAttemptId.newInstance(appId, -1), + if ((i != 2 && callerUGI != null && callerUGI.getShortUserName().equals("user3")) || (i == 2 + && callerUGI != null && (callerUGI.getShortUserName().equals("user2") + || callerUGI.getShortUserName().equals("user3")))) { + assertEquals(ApplicationAttemptId.newInstance(appId, -1), app.getCurrentApplicationAttemptId()); - Assert.assertEquals(ApplicationHistoryManagerOnTimelineStore.UNAVAILABLE, - app.getHost()); - Assert.assertEquals(-1, app.getRpcPort()); - Assert.assertEquals(ApplicationHistoryManagerOnTimelineStore.UNAVAILABLE, - app.getTrackingUrl()); - Assert.assertEquals(ApplicationHistoryManagerOnTimelineStore.UNAVAILABLE, + assertEquals(ApplicationHistoryManagerOnTimelineStore.UNAVAILABLE, app.getHost()); + assertEquals(-1, app.getRpcPort()); + assertEquals(ApplicationHistoryManagerOnTimelineStore.UNAVAILABLE, app.getTrackingUrl()); + assertEquals(ApplicationHistoryManagerOnTimelineStore.UNAVAILABLE, app.getOriginalTrackingUrl()); - Assert.assertEquals("", app.getDiagnostics()); + assertEquals("", app.getDiagnostics()); } else { - Assert.assertEquals(ApplicationAttemptId.newInstance(appId, 1), + assertEquals(ApplicationAttemptId.newInstance(appId, 1), app.getCurrentApplicationAttemptId()); - Assert.assertEquals("test host", app.getHost()); - Assert.assertEquals(100, app.getRpcPort()); - Assert.assertEquals("test tracking url", app.getTrackingUrl()); - Assert.assertEquals("test original tracking url", - app.getOriginalTrackingUrl()); - Assert.assertEquals("test diagnostics info", app.getDiagnostics()); + assertEquals("test host", app.getHost()); + assertEquals(100, app.getRpcPort()); + assertEquals("test tracking url", app.getTrackingUrl()); + assertEquals("test original tracking url", app.getOriginalTrackingUrl()); + assertEquals("test diagnostics info", app.getDiagnostics()); } ApplicationResourceUsageReport applicationResourceUsageReport = app.getApplicationResourceUsageReport(); - Assert.assertEquals(123, - applicationResourceUsageReport.getMemorySeconds()); - Assert - .assertEquals(345, applicationResourceUsageReport.getVcoreSeconds()); + assertEquals(123, applicationResourceUsageReport.getMemorySeconds()); + assertEquals(345, applicationResourceUsageReport.getVcoreSeconds()); long expectedPreemptMemSecs = 456; long expectedPreemptVcoreSecs = 789; if (i == 3) { expectedPreemptMemSecs = 0; expectedPreemptVcoreSecs = 0; } - Assert.assertEquals(expectedPreemptMemSecs, + assertEquals(expectedPreemptMemSecs, applicationResourceUsageReport.getPreemptedMemorySeconds()); - Assert - .assertEquals(expectedPreemptVcoreSecs, applicationResourceUsageReport - .getPreemptedVcoreSeconds()); - Assert.assertEquals(FinalApplicationStatus.UNDEFINED, - app.getFinalApplicationStatus()); - Assert.assertEquals(YarnApplicationState.FINISHED, - app.getYarnApplicationState()); + assertEquals(expectedPreemptVcoreSecs, + applicationResourceUsageReport.getPreemptedVcoreSeconds()); + assertEquals(FinalApplicationStatus.UNDEFINED, app.getFinalApplicationStatus()); + assertEquals(YarnApplicationState.FINISHED, app.getYarnApplicationState()); } } - @Test - public void testGetApplicationReportWithNotAttempt() throws Exception { + @MethodSource("callers") + @ParameterizedTest + void testGetApplicationReportWithNotAttempt(String caller) throws Exception { + initTestApplicationHistoryManagerOnTimelineStore(caller); final ApplicationId appId = ApplicationId.newInstance(0, SCALE + 1); ApplicationReport app; if (callerUGI == null) { app = historyManager.getApplication(appId); } else { - app = - callerUGI.doAs(new PrivilegedExceptionAction () { - @Override - public ApplicationReport run() throws Exception { - return historyManager.getApplication(appId); - } - }); + app = callerUGI.doAs(new PrivilegedExceptionAction() { + @Override + public ApplicationReport run() throws Exception { + return historyManager.getApplication(appId); + } + }); } - Assert.assertNotNull(app); - Assert.assertEquals(appId, app.getApplicationId()); - Assert.assertEquals(ApplicationAttemptId.newInstance(appId, -1), - app.getCurrentApplicationAttemptId()); + assertNotNull(app); + assertEquals(appId, app.getApplicationId()); + assertEquals(ApplicationAttemptId.newInstance(appId, -1), app.getCurrentApplicationAttemptId()); } - @Test - public void testGetApplicationAttemptReport() throws Exception { + @MethodSource("callers") + @ParameterizedTest + void testGetApplicationAttemptReport(String caller) throws Exception { + initTestApplicationHistoryManagerOnTimelineStore(caller); final ApplicationAttemptId appAttemptId = ApplicationAttemptId.newInstance(ApplicationId.newInstance(0, 1), 1); ApplicationAttemptReport appAttempt; @@ -302,8 +281,7 @@ public void testGetApplicationAttemptReport() throws Exception { appAttempt = historyManager.getApplicationAttempt(appAttemptId); } else { try { - appAttempt = - callerUGI.doAs(new PrivilegedExceptionAction () { + appAttempt = callerUGI.doAs(new PrivilegedExceptionAction() { @Override public ApplicationAttemptReport run() throws Exception { return historyManager.getApplicationAttempt(appAttemptId); @@ -311,7 +289,7 @@ public ApplicationAttemptReport run() throws Exception { }); if (callerUGI != null && callerUGI.getShortUserName().equals("user3")) { // The exception is expected - Assert.fail(); + fail(); } } catch (AuthorizationException e) { if (callerUGI != null && callerUGI.getShortUserName().equals("user3")) { @@ -321,32 +299,29 @@ public ApplicationAttemptReport run() throws Exception { throw e; } } - Assert.assertNotNull(appAttempt); - Assert.assertEquals(appAttemptId, appAttempt.getApplicationAttemptId()); - Assert.assertEquals(ContainerId.newContainerId(appAttemptId, 1), - appAttempt.getAMContainerId()); - Assert.assertEquals("test host", appAttempt.getHost()); - Assert.assertEquals(100, appAttempt.getRpcPort()); - Assert.assertEquals("test tracking url", appAttempt.getTrackingUrl()); - Assert.assertEquals("test original tracking url", - appAttempt.getOriginalTrackingUrl()); - Assert.assertEquals("test diagnostics info", appAttempt.getDiagnostics()); - Assert.assertEquals(YarnApplicationAttemptState.FINISHED, - appAttempt.getYarnApplicationAttemptState()); + assertNotNull(appAttempt); + assertEquals(appAttemptId, appAttempt.getApplicationAttemptId()); + assertEquals(ContainerId.newContainerId(appAttemptId, 1), appAttempt.getAMContainerId()); + assertEquals("test host", appAttempt.getHost()); + assertEquals(100, appAttempt.getRpcPort()); + assertEquals("test tracking url", appAttempt.getTrackingUrl()); + assertEquals("test original tracking url", appAttempt.getOriginalTrackingUrl()); + assertEquals("test diagnostics info", appAttempt.getDiagnostics()); + assertEquals(YarnApplicationAttemptState.FINISHED, appAttempt.getYarnApplicationAttemptState()); } - @Test - public void testGetContainerReport() throws Exception { - final ContainerId containerId = - ContainerId.newContainerId(ApplicationAttemptId.newInstance( - ApplicationId.newInstance(0, 1), 1), 1); + @MethodSource("callers") + @ParameterizedTest + void testGetContainerReport(String caller) throws Exception { + initTestApplicationHistoryManagerOnTimelineStore(caller); + final ContainerId containerId = ContainerId.newContainerId( + ApplicationAttemptId.newInstance(ApplicationId.newInstance(0, 1), 1), 1); ContainerReport container; if (callerUGI == null) { container = historyManager.getContainer(containerId); } else { try { - container = - callerUGI.doAs(new PrivilegedExceptionAction () { + container = callerUGI.doAs(new PrivilegedExceptionAction() { @Override public ContainerReport run() throws Exception { return historyManager.getContainer(containerId); @@ -354,7 +329,7 @@ public ContainerReport run() throws Exception { }); if (callerUGI != null && callerUGI.getShortUserName().equals("user3")) { // The exception is expected - Assert.fail(); + fail(); } } catch (AuthorizationException e) { if (callerUGI != null && callerUGI.getShortUserName().equals("user3")) { @@ -364,61 +339,59 @@ public ContainerReport run() throws Exception { throw e; } } - Assert.assertNotNull(container); - Assert.assertEquals(Integer.MAX_VALUE + 1L, container.getCreationTime()); - Assert.assertEquals(Integer.MAX_VALUE + 2L, container.getFinishTime()); - Assert.assertEquals(Resource.newInstance(-1, -1), - container.getAllocatedResource()); - Assert.assertEquals(NodeId.newInstance("test host", 100), - container.getAssignedNode()); - Assert.assertEquals(Priority.UNDEFINED, container.getPriority()); - Assert - .assertEquals("test diagnostics info", container.getDiagnosticsInfo()); - Assert.assertEquals(ContainerState.COMPLETE, container.getContainerState()); - Assert.assertEquals(-1, container.getContainerExitStatus()); - Assert.assertEquals("http://0.0.0.0:8188/applicationhistory/logs/" + - "test host:100/container_0_0001_01_000001/" - + "container_0_0001_01_000001/user1", container.getLogUrl()); + assertNotNull(container); + assertEquals(Integer.MAX_VALUE + 1L, container.getCreationTime()); + assertEquals(Integer.MAX_VALUE + 2L, container.getFinishTime()); + assertEquals(Resource.newInstance(-1, -1), container.getAllocatedResource()); + assertEquals(NodeId.newInstance("test host", 100), container.getAssignedNode()); + assertEquals(Priority.UNDEFINED, container.getPriority()); + assertEquals("test diagnostics info", container.getDiagnosticsInfo()); + assertEquals(ContainerState.COMPLETE, container.getContainerState()); + assertEquals(-1, container.getContainerExitStatus()); + assertEquals( + "http://0.0.0.0:8188/applicationhistory/logs/" + "test host:100/container_0_0001_01_000001/" + + "container_0_0001_01_000001/user1", container.getLogUrl()); } - @Test - public void testGetApplications() throws Exception { + @MethodSource("callers") + @ParameterizedTest + void testGetApplications(String caller) throws Exception { + initTestApplicationHistoryManagerOnTimelineStore(caller); Collection apps = - historyManager.getApplications(Long.MAX_VALUE, 0L, Long.MAX_VALUE) - .values(); - Assert.assertNotNull(apps); - Assert.assertEquals(SCALE + 2, apps.size()); + historyManager.getApplications(Long.MAX_VALUE, 0L, Long.MAX_VALUE).values(); + assertNotNull(apps); + assertEquals(SCALE + 2, apps.size()); ApplicationId ignoredAppId = ApplicationId.newInstance(0, SCALE + 2); for (ApplicationReport app : apps) { - Assert.assertNotEquals(ignoredAppId, app.getApplicationId()); + assertNotEquals(ignoredAppId, app.getApplicationId()); } // Get apps by given appStartedTime period - apps = - historyManager.getApplications(Long.MAX_VALUE, 2147483653L, - Long.MAX_VALUE).values(); - Assert.assertNotNull(apps); - Assert.assertEquals(2, apps.size()); + apps = historyManager.getApplications(Long.MAX_VALUE, 2147483653L, Long.MAX_VALUE).values(); + assertNotNull(apps); + assertEquals(2, apps.size()); } - @Test - public void testGetApplicationAttempts() throws Exception { + @MethodSource("callers") + @ParameterizedTest + void testGetApplicationAttempts(String caller) throws Exception { + initTestApplicationHistoryManagerOnTimelineStore(caller); final ApplicationId appId = ApplicationId.newInstance(0, 1); Collection appAttempts; if (callerUGI == null) { appAttempts = historyManager.getApplicationAttempts(appId).values(); } else { try { - appAttempts = callerUGI.doAs( - new PrivilegedExceptionAction> () { - @Override - public Collection run() throws Exception { - return historyManager.getApplicationAttempts(appId).values(); - } - }); + appAttempts = + callerUGI.doAs(new PrivilegedExceptionAction>() { + @Override + public Collection run() throws Exception { + return historyManager.getApplicationAttempts(appId).values(); + } + }); if (callerUGI != null && callerUGI.getShortUserName().equals("user3")) { // The exception is expected - Assert.fail(); + fail(); } } catch (AuthorizationException e) { if (callerUGI != null && callerUGI.getShortUserName().equals("user3")) { @@ -428,12 +401,14 @@ public Collection run() throws Exception { throw e; } } - Assert.assertNotNull(appAttempts); - Assert.assertEquals(SCALE, appAttempts.size()); + assertNotNull(appAttempts); + assertEquals(SCALE, appAttempts.size()); } - @Test - public void testGetContainers() throws Exception { + @MethodSource("callers") + @ParameterizedTest + void testGetContainers(String caller) throws Exception { + initTestApplicationHistoryManagerOnTimelineStore(caller); final ApplicationAttemptId appAttemptId = ApplicationAttemptId.newInstance(ApplicationId.newInstance(0, 1), 1); Collection containers; @@ -441,8 +416,7 @@ public void testGetContainers() throws Exception { containers = historyManager.getContainers(appAttemptId).values(); } else { try { - containers = callerUGI.doAs( - new PrivilegedExceptionAction> () { + containers = callerUGI.doAs(new PrivilegedExceptionAction>() { @Override public Collection run() throws Exception { return historyManager.getContainers(appAttemptId).values(); @@ -450,7 +424,7 @@ public Collection run() throws Exception { }); if (callerUGI != null && callerUGI.getShortUserName().equals("user3")) { // The exception is expected - Assert.fail(); + fail(); } } catch (AuthorizationException e) { if (callerUGI != null && callerUGI.getShortUserName().equals("user3")) { @@ -460,12 +434,14 @@ public Collection run() throws Exception { throw e; } } - Assert.assertNotNull(containers); - Assert.assertEquals(SCALE, containers.size()); + assertNotNull(containers); + assertEquals(SCALE, containers.size()); } - @Test - public void testGetAMContainer() throws Exception { + @MethodSource("callers") + @ParameterizedTest + void testGetAMContainer(String caller) throws Exception { + initTestApplicationHistoryManagerOnTimelineStore(caller); final ApplicationAttemptId appAttemptId = ApplicationAttemptId.newInstance(ApplicationId.newInstance(0, 1), 1); ContainerReport container; @@ -473,8 +449,7 @@ public void testGetAMContainer() throws Exception { container = historyManager.getAMContainer(appAttemptId); } else { try { - container = - callerUGI.doAs(new PrivilegedExceptionAction () { + container = callerUGI.doAs(new PrivilegedExceptionAction() { @Override public ContainerReport run() throws Exception { return historyManager.getAMContainer(appAttemptId); @@ -482,7 +457,7 @@ public ContainerReport run() throws Exception { }); if (callerUGI != null && callerUGI.getShortUserName().equals("user3")) { // The exception is expected - Assert.fail(); + fail(); } } catch (AuthorizationException e) { if (callerUGI != null && callerUGI.getShortUserName().equals("user3")) { @@ -492,24 +467,20 @@ public ContainerReport run() throws Exception { throw e; } } - Assert.assertNotNull(container); - Assert.assertEquals(appAttemptId, container.getContainerId() - .getApplicationAttemptId()); + assertNotNull(container); + assertEquals(appAttemptId, container.getContainerId().getApplicationAttemptId()); } - private static TimelineEntity createApplicationTimelineEntity( - ApplicationId appId, boolean emptyACLs, boolean noAttemptId, - boolean wrongAppId, boolean enableUpdateEvent, + private static TimelineEntity createApplicationTimelineEntity(ApplicationId appId, + boolean emptyACLs, boolean noAttemptId, boolean wrongAppId, boolean enableUpdateEvent, YarnApplicationState state) { - return createApplicationTimelineEntity(appId, emptyACLs, noAttemptId, - wrongAppId, enableUpdateEvent, state, false, false); + return createApplicationTimelineEntity(appId, emptyACLs, noAttemptId, wrongAppId, + enableUpdateEvent, state, false, false); } - private static TimelineEntity createApplicationTimelineEntity( - ApplicationId appId, boolean emptyACLs, boolean noAttemptId, - boolean wrongAppId, boolean enableUpdateEvent, - YarnApplicationState state, boolean missingPreemptMetrics, - boolean missingQueue) { + private static TimelineEntity createApplicationTimelineEntity(ApplicationId appId, + boolean emptyACLs, boolean noAttemptId, boolean wrongAppId, boolean enableUpdateEvent, + YarnApplicationState state, boolean missingPreemptMetrics, boolean missingQueue) { TimelineEntity entity = new TimelineEntity(); entity.setEntityType(ApplicationMetricsConstants.ENTITY_TYPE); if (wrongAppId) { @@ -518,23 +489,17 @@ private static TimelineEntity createApplicationTimelineEntity( entity.setEntityId(appId.toString()); } entity.setDomainId(TimelineDataManager.DEFAULT_DOMAIN_ID); - entity.addPrimaryFilter( - TimelineStore.SystemFilter.ENTITY_OWNER.toString(), "yarn"); + entity.addPrimaryFilter(TimelineStore.SystemFilter.ENTITY_OWNER.toString(), "yarn"); Map entityInfo = new HashMap(); entityInfo.put(ApplicationMetricsConstants.NAME_ENTITY_INFO, "test app"); - entityInfo.put(ApplicationMetricsConstants.TYPE_ENTITY_INFO, - "test app type"); + entityInfo.put(ApplicationMetricsConstants.TYPE_ENTITY_INFO, "test app type"); entityInfo.put(ApplicationMetricsConstants.USER_ENTITY_INFO, "user1"); if (!missingQueue) { - entityInfo.put(ApplicationMetricsConstants.QUEUE_ENTITY_INFO, - "test queue"); + entityInfo.put(ApplicationMetricsConstants.QUEUE_ENTITY_INFO, "test queue"); } - entityInfo.put( - ApplicationMetricsConstants.UNMANAGED_APPLICATION_ENTITY_INFO, "false"); - entityInfo.put(ApplicationMetricsConstants.APPLICATION_PRIORITY_INFO, - Priority.newInstance(0)); - entityInfo.put(ApplicationMetricsConstants.SUBMITTED_TIME_ENTITY_INFO, - Integer.MAX_VALUE + 1L); + entityInfo.put(ApplicationMetricsConstants.UNMANAGED_APPLICATION_ENTITY_INFO, "false"); + entityInfo.put(ApplicationMetricsConstants.APPLICATION_PRIORITY_INFO, Priority.newInstance(0)); + entityInfo.put(ApplicationMetricsConstants.SUBMITTED_TIME_ENTITY_INFO, Integer.MAX_VALUE + 1L); entityInfo.put(ApplicationMetricsConstants.APP_MEM_METRICS, 123); entityInfo.put(ApplicationMetricsConstants.APP_CPU_METRICS, 345); if (!missingPreemptMetrics) { @@ -544,8 +509,7 @@ private static TimelineEntity createApplicationTimelineEntity( if (emptyACLs) { entityInfo.put(ApplicationMetricsConstants.APP_VIEW_ACLS_ENTITY_INFO, ""); } else { - entityInfo.put(ApplicationMetricsConstants.APP_VIEW_ACLS_ENTITY_INFO, - "user2"); + entityInfo.put(ApplicationMetricsConstants.APP_VIEW_ACLS_ENTITY_INFO, "user2"); } Set appTags = new HashSet(); appTags.add("Test_APP_TAGS_1"); @@ -557,16 +521,13 @@ private static TimelineEntity createApplicationTimelineEntity( tEvent.setTimestamp(Integer.MAX_VALUE + 2L + appId.getId()); entity.addEvent(tEvent); tEvent = new TimelineEvent(); - tEvent.setEventType( - ApplicationMetricsConstants.FINISHED_EVENT_TYPE); + tEvent.setEventType(ApplicationMetricsConstants.FINISHED_EVENT_TYPE); tEvent.setTimestamp(Integer.MAX_VALUE + 3L + appId.getId()); Map eventInfo = new HashMap(); - eventInfo.put(ApplicationMetricsConstants.DIAGNOSTICS_INFO_EVENT_INFO, - "test diagnostics info"); + eventInfo.put(ApplicationMetricsConstants.DIAGNOSTICS_INFO_EVENT_INFO, "test diagnostics info"); eventInfo.put(ApplicationMetricsConstants.FINAL_STATUS_EVENT_INFO, FinalApplicationStatus.UNDEFINED.toString()); - eventInfo.put(ApplicationMetricsConstants.STATE_EVENT_INFO, - state.toString()); + eventInfo.put(ApplicationMetricsConstants.STATE_EVENT_INFO, state.toString()); if (!noAttemptId) { eventInfo.put(ApplicationMetricsConstants.LATEST_APP_ATTEMPT_EVENT_INFO, ApplicationAttemptId.newInstance(appId, 1)); @@ -577,60 +538,51 @@ private static TimelineEntity createApplicationTimelineEntity( // after YARN_APPLICATION_FINISHED // The final YarnApplicationState should not be changed tEvent = new TimelineEvent(); - tEvent.setEventType( - ApplicationMetricsConstants.STATE_UPDATED_EVENT_TYPE); + tEvent.setEventType(ApplicationMetricsConstants.STATE_UPDATED_EVENT_TYPE); tEvent.setTimestamp(Integer.MAX_VALUE + 4L + appId.getId()); eventInfo = new HashMap(); - eventInfo.put(ApplicationMetricsConstants.STATE_EVENT_INFO, - YarnApplicationState.KILLED); + eventInfo.put(ApplicationMetricsConstants.STATE_EVENT_INFO, YarnApplicationState.KILLED); tEvent.setEventInfo(eventInfo); entity.addEvent(tEvent); if (enableUpdateEvent) { tEvent = new TimelineEvent(); long updatedTimeIndex = 4L; - createAppModifiedEvent(appId, tEvent, updatedTimeIndex++, "changed queue", - 5); + createAppModifiedEvent(appId, tEvent, updatedTimeIndex++, "changed queue", 5); entity.addEvent(tEvent); // Change priority alone tEvent = new TimelineEvent(); - createAppModifiedEvent(appId, tEvent, updatedTimeIndex++, "changed queue", - 6); + createAppModifiedEvent(appId, tEvent, updatedTimeIndex++, "changed queue", 6); // Now change queue tEvent = new TimelineEvent(); - createAppModifiedEvent(appId, tEvent, updatedTimeIndex++, - "changed queue1", 6); + createAppModifiedEvent(appId, tEvent, updatedTimeIndex++, "changed queue1", 6); entity.addEvent(tEvent); } return entity; } - private static void createAppModifiedEvent(ApplicationId appId, - TimelineEvent tEvent, long updatedTimeIndex, String queue, int priority) { + private static void createAppModifiedEvent(ApplicationId appId, TimelineEvent tEvent, + long updatedTimeIndex, String queue, int priority) { tEvent.setEventType(ApplicationMetricsConstants.UPDATED_EVENT_TYPE); tEvent.setTimestamp(Integer.MAX_VALUE + updatedTimeIndex + appId.getId()); Map eventInfo = new HashMap(); eventInfo.put(ApplicationMetricsConstants.QUEUE_ENTITY_INFO, queue); - eventInfo.put(ApplicationMetricsConstants.APPLICATION_PRIORITY_INFO, - priority); + eventInfo.put(ApplicationMetricsConstants.APPLICATION_PRIORITY_INFO, priority); tEvent.setEventInfo(eventInfo); } - private static TimelineEntity createAppAttemptTimelineEntity( - ApplicationAttemptId appAttemptId) { + private static TimelineEntity createAppAttemptTimelineEntity(ApplicationAttemptId appAttemptId) { TimelineEntity entity = new TimelineEntity(); entity.setEntityType(AppAttemptMetricsConstants.ENTITY_TYPE); entity.setEntityId(appAttemptId.toString()); entity.setDomainId(TimelineDataManager.DEFAULT_DOMAIN_ID); entity.addPrimaryFilter(AppAttemptMetricsConstants.PARENT_PRIMARY_FILTER, appAttemptId.getApplicationId().toString()); - entity.addPrimaryFilter( - TimelineStore.SystemFilter.ENTITY_OWNER.toString(), "yarn"); + entity.addPrimaryFilter(TimelineStore.SystemFilter.ENTITY_OWNER.toString(), "yarn"); TimelineEvent tEvent = new TimelineEvent(); tEvent.setEventType(AppAttemptMetricsConstants.REGISTERED_EVENT_TYPE); tEvent.setTimestamp(Integer.MAX_VALUE + 1L); Map eventInfo = new HashMap(); - eventInfo.put(AppAttemptMetricsConstants.TRACKING_URL_INFO, - "test tracking url"); + eventInfo.put(AppAttemptMetricsConstants.TRACKING_URL_INFO, "test tracking url"); eventInfo.put(AppAttemptMetricsConstants.ORIGINAL_TRACKING_URL_INFO, "test original tracking url"); eventInfo.put(AppAttemptMetricsConstants.HOST_INFO, "test host"); @@ -643,12 +595,10 @@ private static TimelineEntity createAppAttemptTimelineEntity( tEvent.setEventType(AppAttemptMetricsConstants.FINISHED_EVENT_TYPE); tEvent.setTimestamp(Integer.MAX_VALUE + 2L); eventInfo = new HashMap(); - eventInfo.put(AppAttemptMetricsConstants.TRACKING_URL_INFO, - "test tracking url"); + eventInfo.put(AppAttemptMetricsConstants.TRACKING_URL_INFO, "test tracking url"); eventInfo.put(AppAttemptMetricsConstants.ORIGINAL_TRACKING_URL_INFO, "test original tracking url"); - eventInfo.put(AppAttemptMetricsConstants.DIAGNOSTICS_INFO, - "test diagnostics info"); + eventInfo.put(AppAttemptMetricsConstants.DIAGNOSTICS_INFO, "test diagnostics info"); eventInfo.put(AppAttemptMetricsConstants.FINAL_STATUS_INFO, FinalApplicationStatus.UNDEFINED.toString()); eventInfo.put(AppAttemptMetricsConstants.STATE_INFO, @@ -665,35 +615,28 @@ private static TimelineEntity createContainerEntity(ContainerId containerId) { entity.setDomainId(TimelineDataManager.DEFAULT_DOMAIN_ID); entity.addPrimaryFilter(ContainerMetricsConstants.PARENT_PRIMARIY_FILTER, containerId.getApplicationAttemptId().toString()); - entity.addPrimaryFilter( - TimelineStore.SystemFilter.ENTITY_OWNER.toString(), "yarn"); + entity.addPrimaryFilter(TimelineStore.SystemFilter.ENTITY_OWNER.toString(), "yarn"); Map entityInfo = new HashMap(); entityInfo.put(ContainerMetricsConstants.ALLOCATED_MEMORY_INFO, -1); entityInfo.put(ContainerMetricsConstants.ALLOCATED_VCORE_INFO, -1); - entityInfo.put(ContainerMetricsConstants.ALLOCATED_HOST_INFO, - "test host"); + entityInfo.put(ContainerMetricsConstants.ALLOCATED_HOST_INFO, "test host"); entityInfo.put(ContainerMetricsConstants.ALLOCATED_PORT_INFO, 100); - entityInfo - .put(ContainerMetricsConstants.ALLOCATED_PRIORITY_INFO, -1); - entityInfo.put(ContainerMetricsConstants - .ALLOCATED_HOST_HTTP_ADDRESS_INFO, "http://test:1234"); + entityInfo.put(ContainerMetricsConstants.ALLOCATED_PRIORITY_INFO, -1); + entityInfo.put(ContainerMetricsConstants.ALLOCATED_HOST_HTTP_ADDRESS_INFO, "http://test:1234"); entity.setOtherInfo(entityInfo); TimelineEvent tEvent = new TimelineEvent(); tEvent.setEventType(ContainerMetricsConstants.CREATED_EVENT_TYPE); tEvent.setTimestamp(Integer.MAX_VALUE + 1L); entity.addEvent(tEvent); - ; tEvent = new TimelineEvent(); tEvent.setEventType(ContainerMetricsConstants.FINISHED_EVENT_TYPE); tEvent.setTimestamp(Integer.MAX_VALUE + 2L); Map eventInfo = new HashMap(); - eventInfo.put(ContainerMetricsConstants.DIAGNOSTICS_INFO, - "test diagnostics info"); + eventInfo.put(ContainerMetricsConstants.DIAGNOSTICS_INFO, "test diagnostics info"); eventInfo.put(ContainerMetricsConstants.EXIT_STATUS_INFO, -1); - eventInfo.put(ContainerMetricsConstants.STATE_INFO, - ContainerState.COMPLETE.toString()); + eventInfo.put(ContainerMetricsConstants.STATE_INFO, ContainerState.COMPLETE.toString()); tEvent.setEventInfo(eventInfo); entity.addEvent(tEvent); return entity; } -} +} \ No newline at end of file diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryServer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryServer.java index b9d1c5e492f12..eb3db5eee190c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryServer.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryServer.java @@ -18,9 +18,17 @@ package org.apache.hadoop.yarn.server.applicationhistoryservice; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.fail; +import java.io.ByteArrayOutputStream; +import java.io.File; +import java.io.FileInputStream; +import java.io.InputStream; +import java.net.HttpURLConnection; +import java.net.URL; +import java.util.HashMap; +import java.util.Map; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.Timeout; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.http.lib.StaticUserWebFilter; @@ -33,23 +41,18 @@ import org.apache.hadoop.yarn.server.timeline.recovery.MemoryTimelineStateStore; import org.apache.hadoop.yarn.server.timeline.recovery.TimelineStateStore; import org.apache.hadoop.yarn.server.timeline.security.TimelineAuthenticationFilterInitializer; -import org.junit.Assert; -import org.junit.Test; -import java.io.ByteArrayOutputStream; -import java.io.File; -import java.io.FileInputStream; -import java.io.InputStream; -import java.net.HttpURLConnection; -import java.net.URL; -import java.util.HashMap; -import java.util.Map; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.fail; public class TestApplicationHistoryServer { // simple test init/start/stop ApplicationHistoryServer. Status should change. - @Test(timeout = 60000) - public void testStartStopServer() throws Exception { + @Test + @Timeout(60000) + void testStartStopServer() throws Exception { ApplicationHistoryServer historyServer = new ApplicationHistoryServer(); Configuration config = new YarnConfiguration(); config.setClass(YarnConfiguration.TIMELINE_SERVICE_STORE, @@ -65,7 +68,7 @@ public void testStartStopServer() throws Exception { historyServer.start(); fail(); } catch (IllegalArgumentException e) { - Assert.assertTrue(e.getMessage().contains( + assertTrue(e.getMessage().contains( YarnConfiguration.TIMELINE_SERVICE_HANDLER_THREAD_COUNT)); } config.setInt(YarnConfiguration.TIMELINE_SERVICE_HANDLER_THREAD_COUNT, @@ -89,8 +92,9 @@ public void testStartStopServer() throws Exception { } // test launch method - @Test(timeout = 60000) - public void testLaunch() throws Exception { + @Test + @Timeout(60000) + void testLaunch() throws Exception { ExitUtil.disableSystemExit(); ApplicationHistoryServer historyServer = null; try { @@ -109,34 +113,37 @@ public void testLaunch() throws Exception { } } - //test launch method with -D arguments - @Test(timeout = 60000) - public void testLaunchWithArguments() throws Exception { - ExitUtil.disableSystemExit(); - ApplicationHistoryServer historyServer = null; - try { - // Not able to modify the config of this test case, - // but others have been customized to avoid conflicts - String[] args = new String[2]; - args[0]="-D" + YarnConfiguration.TIMELINE_SERVICE_LEVELDB_TTL_INTERVAL_MS + "=4000"; - args[1]="-D" + YarnConfiguration.TIMELINE_SERVICE_TTL_MS + "=200"; - historyServer = - ApplicationHistoryServer.launchAppHistoryServer(args); - Configuration conf = historyServer.getConfig(); - assertEquals("4000", conf.get(YarnConfiguration.TIMELINE_SERVICE_LEVELDB_TTL_INTERVAL_MS)); - assertEquals("200", conf.get(YarnConfiguration.TIMELINE_SERVICE_TTL_MS)); - } catch (ExitUtil.ExitException e) { - assertEquals(0, e.status); - ExitUtil.resetFirstExitException(); - fail(); - } finally { - if (historyServer != null) { - historyServer.stop(); - } - } - } - @Test(timeout = 240000) - public void testFilterOverrides() throws Exception { + //test launch method with -D arguments + @Test + @Timeout(60000) + void testLaunchWithArguments() throws Exception { + ExitUtil.disableSystemExit(); + ApplicationHistoryServer historyServer = null; + try { + // Not able to modify the config of this test case, + // but others have been customized to avoid conflicts + String[] args = new String[2]; + args[0] = "-D" + YarnConfiguration.TIMELINE_SERVICE_LEVELDB_TTL_INTERVAL_MS + "=4000"; + args[1] = "-D" + YarnConfiguration.TIMELINE_SERVICE_TTL_MS + "=200"; + historyServer = + ApplicationHistoryServer.launchAppHistoryServer(args); + Configuration conf = historyServer.getConfig(); + assertEquals("4000", conf.get(YarnConfiguration.TIMELINE_SERVICE_LEVELDB_TTL_INTERVAL_MS)); + assertEquals("200", conf.get(YarnConfiguration.TIMELINE_SERVICE_TTL_MS)); + } catch (ExitUtil.ExitException e) { + assertEquals(0, e.status); + ExitUtil.resetFirstExitException(); + fail(); + } finally { + if (historyServer != null) { + historyServer.stop(); + } + } + } + + @Test + @Timeout(240000) + void testFilterOverrides() throws Exception { HashMap driver = new HashMap(); driver.put("", TimelineAuthenticationFilterInitializer.class.getName()); @@ -144,15 +151,15 @@ public void testFilterOverrides() throws Exception { StaticUserWebFilter.class.getName() + "," + TimelineAuthenticationFilterInitializer.class.getName()); driver.put(AuthenticationFilterInitializer.class.getName(), - TimelineAuthenticationFilterInitializer.class.getName()); + TimelineAuthenticationFilterInitializer.class.getName()); driver.put(TimelineAuthenticationFilterInitializer.class.getName(), - TimelineAuthenticationFilterInitializer.class.getName()); + TimelineAuthenticationFilterInitializer.class.getName()); driver.put(AuthenticationFilterInitializer.class.getName() + "," + TimelineAuthenticationFilterInitializer.class.getName(), - TimelineAuthenticationFilterInitializer.class.getName()); + TimelineAuthenticationFilterInitializer.class.getName()); driver.put(AuthenticationFilterInitializer.class.getName() + ", " + TimelineAuthenticationFilterInitializer.class.getName(), - TimelineAuthenticationFilterInitializer.class.getName()); + TimelineAuthenticationFilterInitializer.class.getName()); for (Map.Entry entry : driver.entrySet()) { String filterInitializer = entry.getKey(); @@ -176,8 +183,9 @@ public void testFilterOverrides() throws Exception { } } - @Test(timeout = 240000) - public void testHostedUIs() throws Exception { + @Test + @Timeout(240000) + void testHostedUIs() throws Exception { ApplicationHistoryServer historyServer = new ApplicationHistoryServer(); Configuration config = new YarnConfiguration(); @@ -209,8 +217,8 @@ public void testHostedUIs() throws Exception { } finally { historyServer.stop(); } - assertEquals("Web file contents should be the same as on disk contents", - diskFileStr, connFileStr); + assertEquals(diskFileStr, connFileStr, + "Web file contents should be the same as on disk contents"); } private String readInputStream(InputStream input) throws Exception { ByteArrayOutputStream data = new ByteArrayOutputStream(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestFileSystemApplicationHistoryStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestFileSystemApplicationHistoryStore.java index 6b068c132d4c7..64b11eb4f1f1e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestFileSystemApplicationHistoryStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestFileSystemApplicationHistoryStore.java @@ -22,15 +22,12 @@ import java.net.URI; import java.net.URISyntaxException; -import org.junit.Assert; - -import static org.mockito.Mockito.any; -import static org.mockito.Mockito.doReturn; -import static org.mockito.Mockito.doThrow; -import static org.mockito.Mockito.never; -import static org.mockito.Mockito.spy; -import static org.mockito.Mockito.times; -import static org.mockito.Mockito.verify; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.mockito.Mockito; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileStatus; @@ -45,12 +42,19 @@ import org.apache.hadoop.yarn.server.applicationhistoryservice.records.ApplicationAttemptHistoryData; import org.apache.hadoop.yarn.server.applicationhistoryservice.records.ApplicationHistoryData; import org.apache.hadoop.yarn.server.applicationhistoryservice.records.ContainerHistoryData; -import org.junit.After; -import org.junit.Before; -import org.junit.Test; -import org.mockito.Mockito; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.fail; +import static org.mockito.Mockito.any; +import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; public class TestFileSystemApplicationHistoryStore extends ApplicationHistoryStoreTestUtils { @@ -61,7 +65,7 @@ public class TestFileSystemApplicationHistoryStore extends private FileSystem fs; private Path fsWorkingPath; - @Before + @BeforeEach public void setup() throws Exception { fs = new RawLocalFileSystem(); initAndStartStore(fs); @@ -75,8 +79,7 @@ private void initAndStartStore(final FileSystem fs) throws IOException, new Path("target", TestFileSystemApplicationHistoryStore.class.getSimpleName()); fs.delete(fsWorkingPath, true); - conf.set(YarnConfiguration.FS_APPLICATION_HISTORY_STORE_URI, - fsWorkingPath.toString()); + conf.set(YarnConfiguration.FS_APPLICATION_HISTORY_STORE_URI, fsWorkingPath.toString()); store = new FileSystemApplicationHistoryStore() { @Override protected FileSystem getFileSystem(Path path, Configuration conf) { @@ -87,7 +90,7 @@ protected FileSystem getFileSystem(Path path, Configuration conf) { store.start(); } - @After + @AfterEach public void tearDown() throws Exception { store.stop(); fs.delete(fsWorkingPath, true); @@ -95,7 +98,7 @@ public void tearDown() throws Exception { } @Test - public void testReadWriteHistoryData() throws IOException { + void testReadWriteHistoryData() throws IOException { LOG.info("Starting testReadWriteHistoryData"); testWriteHistoryData(5); testReadHistoryData(5); @@ -146,58 +149,56 @@ private void testReadHistoryData( int num, boolean missingContainer, boolean missingApplicationAttempt) throws IOException { // read application history data - Assert.assertEquals(num, store.getAllApplications().size()); + assertEquals(num, store.getAllApplications().size()); for (int i = 1; i <= num; ++i) { ApplicationId appId = ApplicationId.newInstance(0, i); ApplicationHistoryData appData = store.getApplication(appId); - Assert.assertNotNull(appData); - Assert.assertEquals(appId.toString(), appData.getApplicationName()); - Assert.assertEquals(appId.toString(), appData.getDiagnosticsInfo()); + assertNotNull(appData); + assertEquals(appId.toString(), appData.getApplicationName()); + assertEquals(appId.toString(), appData.getDiagnosticsInfo()); // read application attempt history data - Assert.assertEquals(num, store.getApplicationAttempts(appId).size()); + assertEquals(num, store.getApplicationAttempts(appId).size()); for (int j = 1; j <= num; ++j) { ApplicationAttemptId appAttemptId = ApplicationAttemptId.newInstance(appId, j); ApplicationAttemptHistoryData attemptData = store.getApplicationAttempt(appAttemptId); - Assert.assertNotNull(attemptData); - Assert.assertEquals(appAttemptId.toString(), attemptData.getHost()); + assertNotNull(attemptData); + assertEquals(appAttemptId.toString(), attemptData.getHost()); if (missingApplicationAttempt && j == num) { - Assert.assertNull(attemptData.getDiagnosticsInfo()); + assertNull(attemptData.getDiagnosticsInfo()); continue; } else { - Assert.assertEquals(appAttemptId.toString(), + assertEquals(appAttemptId.toString(), attemptData.getDiagnosticsInfo()); } // read container history data - Assert.assertEquals(num, store.getContainers(appAttemptId).size()); + assertEquals(num, store.getContainers(appAttemptId).size()); for (int k = 1; k <= num; ++k) { ContainerId containerId = ContainerId.newContainerId(appAttemptId, k); ContainerHistoryData containerData = store.getContainer(containerId); - Assert.assertNotNull(containerData); - Assert.assertEquals(Priority.newInstance(containerId.getId()), - containerData.getPriority()); + assertNotNull(containerData); + assertEquals(Priority.newInstance(containerId.getId()), containerData.getPriority()); if (missingContainer && k == num) { - Assert.assertNull(containerData.getDiagnosticsInfo()); + assertNull(containerData.getDiagnosticsInfo()); } else { - Assert.assertEquals(containerId.toString(), + assertEquals(containerId.toString(), containerData.getDiagnosticsInfo()); } } ContainerHistoryData masterContainer = store.getAMContainer(appAttemptId); - Assert.assertNotNull(masterContainer); - Assert.assertEquals(ContainerId.newContainerId(appAttemptId, 1), - masterContainer.getContainerId()); + assertNotNull(masterContainer); + assertEquals(ContainerId.newContainerId(appAttemptId, 1), masterContainer.getContainerId()); } } } @Test - public void testWriteAfterApplicationFinish() throws IOException { + void testWriteAfterApplicationFinish() throws IOException { LOG.info("Starting testWriteAfterApplicationFinish"); ApplicationId appId = ApplicationId.newInstance(0, 1); writeApplicationStartData(appId); @@ -207,34 +208,34 @@ public void testWriteAfterApplicationFinish() throws IOException { ApplicationAttemptId.newInstance(appId, 1); try { writeApplicationAttemptStartData(appAttemptId); - Assert.fail(); + fail(); } catch (IOException e) { - Assert.assertTrue(e.getMessage().contains("is not opened")); + assertTrue(e.getMessage().contains("is not opened")); } try { writeApplicationAttemptFinishData(appAttemptId); - Assert.fail(); + fail(); } catch (IOException e) { - Assert.assertTrue(e.getMessage().contains("is not opened")); + assertTrue(e.getMessage().contains("is not opened")); } // write container history data ContainerId containerId = ContainerId.newContainerId(appAttemptId, 1); try { writeContainerStartData(containerId); - Assert.fail(); + fail(); } catch (IOException e) { - Assert.assertTrue(e.getMessage().contains("is not opened")); + assertTrue(e.getMessage().contains("is not opened")); } try { writeContainerFinishData(containerId); - Assert.fail(); + fail(); } catch (IOException e) { - Assert.assertTrue(e.getMessage().contains("is not opened")); + assertTrue(e.getMessage().contains("is not opened")); } } @Test - public void testMassiveWriteContainerHistoryData() throws IOException { + void testMassiveWriteContainerHistoryData() throws IOException { LOG.info("Starting testMassiveWriteContainerHistoryData"); long mb = 1024 * 1024; long usedDiskBefore = fs.getContentSummary(fsWorkingPath).getLength() / mb; @@ -249,25 +250,25 @@ public void testMassiveWriteContainerHistoryData() throws IOException { } writeApplicationFinishData(appId); long usedDiskAfter = fs.getContentSummary(fsWorkingPath).getLength() / mb; - Assert.assertTrue((usedDiskAfter - usedDiskBefore) < 20); + assertTrue((usedDiskAfter - usedDiskBefore) < 20); } @Test - public void testMissingContainerHistoryData() throws IOException { + void testMissingContainerHistoryData() throws IOException { LOG.info("Starting testMissingContainerHistoryData"); testWriteHistoryData(3, true, false); testReadHistoryData(3, true, false); } - + @Test - public void testMissingApplicationAttemptHistoryData() throws IOException { + void testMissingApplicationAttemptHistoryData() throws IOException { LOG.info("Starting testMissingApplicationAttemptHistoryData"); testWriteHistoryData(3, false, true); testReadHistoryData(3, false, true); } @Test - public void testInitExistingWorkingDirectoryInSafeMode() throws Exception { + void testInitExistingWorkingDirectoryInSafeMode() throws Exception { LOG.info("Starting testInitExistingWorkingDirectoryInSafeMode"); tearDown(); @@ -280,7 +281,7 @@ public void testInitExistingWorkingDirectoryInSafeMode() throws Exception { try { initAndStartStore(fileSystem); } catch (Exception e) { - Assert.fail("Exception should not be thrown: " + e); + fail("Exception should not be thrown: " + e); } // Make sure that directory creation was not attempted @@ -289,7 +290,7 @@ public void testInitExistingWorkingDirectoryInSafeMode() throws Exception { } @Test - public void testInitNonExistingWorkingDirectoryInSafeMode() throws Exception { + void testInitNonExistingWorkingDirectoryInSafeMode() throws Exception { LOG.info("Starting testInitNonExistingWorkingDirectoryInSafeMode"); tearDown(); @@ -302,7 +303,7 @@ public void testInitNonExistingWorkingDirectoryInSafeMode() throws Exception { try { initAndStartStore(fileSystem); - Assert.fail("Exception should have been thrown"); + fail("Exception should have been thrown"); } catch (Exception e) { // Expected failure } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestMemoryApplicationHistoryStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestMemoryApplicationHistoryStore.java index 556db2beaf4b8..8a1b980680925 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestMemoryApplicationHistoryStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestMemoryApplicationHistoryStore.java @@ -20,7 +20,8 @@ import java.io.IOException; -import org.junit.Assert; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ApplicationId; @@ -29,27 +30,30 @@ import org.apache.hadoop.yarn.server.applicationhistoryservice.records.ApplicationAttemptHistoryData; import org.apache.hadoop.yarn.server.applicationhistoryservice.records.ApplicationHistoryData; import org.apache.hadoop.yarn.server.applicationhistoryservice.records.ContainerHistoryData; -import org.junit.Before; -import org.junit.Test; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.fail; public class TestMemoryApplicationHistoryStore extends ApplicationHistoryStoreTestUtils { - @Before + @BeforeEach public void setup() { store = new MemoryApplicationHistoryStore(); } @Test - public void testReadWriteApplicationHistory() throws Exception { + void testReadWriteApplicationHistory() throws Exception { // Out of order ApplicationId appId = ApplicationId.newInstance(0, 1); try { writeApplicationFinishData(appId); - Assert.fail(); + fail(); } catch (IOException e) { - Assert.assertTrue(e.getMessage().contains( - "is stored before the start information")); + assertTrue(e.getMessage().contains( + "is stored before the start information")); } // Normal int numApps = 5; @@ -58,42 +62,42 @@ public void testReadWriteApplicationHistory() throws Exception { writeApplicationStartData(appId); writeApplicationFinishData(appId); } - Assert.assertEquals(numApps, store.getAllApplications().size()); + assertEquals(numApps, store.getAllApplications().size()); for (int i = 1; i <= numApps; ++i) { appId = ApplicationId.newInstance(0, i); ApplicationHistoryData data = store.getApplication(appId); - Assert.assertNotNull(data); - Assert.assertEquals(appId.toString(), data.getApplicationName()); - Assert.assertEquals(appId.toString(), data.getDiagnosticsInfo()); + assertNotNull(data); + assertEquals(appId.toString(), data.getApplicationName()); + assertEquals(appId.toString(), data.getDiagnosticsInfo()); } // Write again appId = ApplicationId.newInstance(0, 1); try { writeApplicationStartData(appId); - Assert.fail(); + fail(); } catch (IOException e) { - Assert.assertTrue(e.getMessage().contains("is already stored")); + assertTrue(e.getMessage().contains("is already stored")); } try { writeApplicationFinishData(appId); - Assert.fail(); + fail(); } catch (IOException e) { - Assert.assertTrue(e.getMessage().contains("is already stored")); + assertTrue(e.getMessage().contains("is already stored")); } } @Test - public void testReadWriteApplicationAttemptHistory() throws Exception { + void testReadWriteApplicationAttemptHistory() throws Exception { // Out of order ApplicationId appId = ApplicationId.newInstance(0, 1); ApplicationAttemptId appAttemptId = ApplicationAttemptId.newInstance(appId, 1); try { writeApplicationAttemptFinishData(appAttemptId); - Assert.fail(); + fail(); } catch (IOException e) { - Assert.assertTrue(e.getMessage().contains( - "is stored before the start information")); + assertTrue(e.getMessage().contains( + "is stored before the start information")); } // Normal int numAppAttempts = 5; @@ -103,36 +107,36 @@ public void testReadWriteApplicationAttemptHistory() throws Exception { writeApplicationAttemptStartData(appAttemptId); writeApplicationAttemptFinishData(appAttemptId); } - Assert.assertEquals(numAppAttempts, store.getApplicationAttempts(appId) - .size()); + assertEquals(numAppAttempts, store.getApplicationAttempts(appId) + .size()); for (int i = 1; i <= numAppAttempts; ++i) { appAttemptId = ApplicationAttemptId.newInstance(appId, i); ApplicationAttemptHistoryData data = store.getApplicationAttempt(appAttemptId); - Assert.assertNotNull(data); - Assert.assertEquals(appAttemptId.toString(), data.getHost()); - Assert.assertEquals(appAttemptId.toString(), data.getDiagnosticsInfo()); + assertNotNull(data); + assertEquals(appAttemptId.toString(), data.getHost()); + assertEquals(appAttemptId.toString(), data.getDiagnosticsInfo()); } writeApplicationFinishData(appId); // Write again appAttemptId = ApplicationAttemptId.newInstance(appId, 1); try { writeApplicationAttemptStartData(appAttemptId); - Assert.fail(); + fail(); } catch (IOException e) { - Assert.assertTrue(e.getMessage().contains("is already stored")); + assertTrue(e.getMessage().contains("is already stored")); } try { writeApplicationAttemptFinishData(appAttemptId); - Assert.fail(); + fail(); } catch (IOException e) { - Assert.assertTrue(e.getMessage().contains("is already stored")); + assertTrue(e.getMessage().contains("is already stored")); } } @SuppressWarnings("deprecation") @Test - public void testReadWriteContainerHistory() throws Exception { + void testReadWriteContainerHistory() throws Exception { // Out of order ApplicationId appId = ApplicationId.newInstance(0, 1); ApplicationAttemptId appAttemptId = @@ -140,10 +144,10 @@ public void testReadWriteContainerHistory() throws Exception { ContainerId containerId = ContainerId.newContainerId(appAttemptId, 1); try { writeContainerFinishData(containerId); - Assert.fail(); + fail(); } catch (IOException e) { - Assert.assertTrue(e.getMessage().contains( - "is stored before the start information")); + assertTrue(e.getMessage().contains( + "is stored before the start information")); } // Normal writeApplicationAttemptStartData(appAttemptId); @@ -153,39 +157,38 @@ public void testReadWriteContainerHistory() throws Exception { writeContainerStartData(containerId); writeContainerFinishData(containerId); } - Assert - .assertEquals(numContainers, store.getContainers(appAttemptId).size()); + assertEquals(numContainers, store.getContainers(appAttemptId).size()); for (int i = 1; i <= numContainers; ++i) { containerId = ContainerId.newContainerId(appAttemptId, i); ContainerHistoryData data = store.getContainer(containerId); - Assert.assertNotNull(data); - Assert.assertEquals(Priority.newInstance(containerId.getId()), - data.getPriority()); - Assert.assertEquals(containerId.toString(), data.getDiagnosticsInfo()); + assertNotNull(data); + assertEquals(Priority.newInstance(containerId.getId()), + data.getPriority()); + assertEquals(containerId.toString(), data.getDiagnosticsInfo()); } ContainerHistoryData masterContainer = store.getAMContainer(appAttemptId); - Assert.assertNotNull(masterContainer); - Assert.assertEquals(ContainerId.newContainerId(appAttemptId, 1), - masterContainer.getContainerId()); + assertNotNull(masterContainer); + assertEquals(ContainerId.newContainerId(appAttemptId, 1), + masterContainer.getContainerId()); writeApplicationAttemptFinishData(appAttemptId); // Write again containerId = ContainerId.newContainerId(appAttemptId, 1); try { writeContainerStartData(containerId); - Assert.fail(); + fail(); } catch (IOException e) { - Assert.assertTrue(e.getMessage().contains("is already stored")); + assertTrue(e.getMessage().contains("is already stored")); } try { writeContainerFinishData(containerId); - Assert.fail(); + fail(); } catch (IOException e) { - Assert.assertTrue(e.getMessage().contains("is already stored")); + assertTrue(e.getMessage().contains("is already stored")); } } @Test - public void testMassiveWriteContainerHistory() throws IOException { + void testMassiveWriteContainerHistory() throws IOException { long mb = 1024 * 1024; Runtime runtime = Runtime.getRuntime(); long usedMemoryBefore = (runtime.totalMemory() - runtime.freeMemory()) / mb; @@ -199,7 +202,7 @@ public void testMassiveWriteContainerHistory() throws IOException { writeContainerFinishData(containerId); } long usedMemoryAfter = (runtime.totalMemory() - runtime.freeMemory()) / mb; - Assert.assertTrue((usedMemoryAfter - usedMemoryBefore) < 400); + assertTrue((usedMemoryAfter - usedMemoryBefore) < 400); } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/TestAHSWebApp.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/TestAHSWebApp.java index 49fcc58b2ae49..b5bd66937a309 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/TestAHSWebApp.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/TestAHSWebApp.java @@ -18,10 +18,12 @@ package org.apache.hadoop.yarn.server.applicationhistoryservice.webapp; -import static org.apache.hadoop.yarn.webapp.Params.TITLE; -import static org.mockito.Mockito.mock; - import java.util.Map; + +import com.google.inject.Injector; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.yarn.api.ApplicationBaseProtocol; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; @@ -38,11 +40,11 @@ import org.apache.hadoop.yarn.util.StringHelper; import org.apache.hadoop.yarn.webapp.YarnWebParams; import org.apache.hadoop.yarn.webapp.test.WebAppTests; -import org.junit.Assert; -import org.junit.Before; -import org.junit.Test; -import com.google.inject.Injector; +import static org.apache.hadoop.yarn.webapp.Params.TITLE; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.Mockito.mock; public class TestAHSWebApp extends ApplicationHistoryStoreTestUtils { @@ -50,46 +52,45 @@ public void setApplicationHistoryStore(ApplicationHistoryStore store) { this.store = store; } - @Before + @BeforeEach public void setup() { store = new MemoryApplicationHistoryStore(); } @Test - public void testAppControllerIndex() throws Exception { + void testAppControllerIndex() throws Exception { ApplicationHistoryManager ahManager = mock(ApplicationHistoryManager.class); Injector injector = WebAppTests.createMockInjector(ApplicationHistoryManager.class, - ahManager); + ahManager); AHSController controller = injector.getInstance(AHSController.class); controller.index(); - Assert - .assertEquals("Application History", controller.get(TITLE, "unknown")); + assertEquals("Application History", controller.get(TITLE, "unknown")); } @Test - public void testView() throws Exception { + void testView() throws Exception { Injector injector = WebAppTests.createMockInjector(ApplicationBaseProtocol.class, - mockApplicationHistoryClientService(5, 1, 1)); + mockApplicationHistoryClientService(5, 1, 1)); AHSView ahsViewInstance = injector.getInstance(AHSView.class); ahsViewInstance.render(); WebAppTests.flushOutput(injector); ahsViewInstance.set(YarnWebParams.APP_STATE, - YarnApplicationState.FAILED.toString()); + YarnApplicationState.FAILED.toString()); ahsViewInstance.render(); WebAppTests.flushOutput(injector); ahsViewInstance.set(YarnWebParams.APP_STATE, StringHelper.cjoin( - YarnApplicationState.FAILED.toString(), YarnApplicationState.KILLED)); + YarnApplicationState.FAILED.toString(), YarnApplicationState.KILLED)); ahsViewInstance.render(); WebAppTests.flushOutput(injector); } @Test - public void testAPPViewNaturalSortType() throws Exception { + void testAPPViewNaturalSortType() throws Exception { Injector injector = WebAppTests.createMockInjector(ApplicationBaseProtocol.class, mockApplicationHistoryClientService(5, 1, 1)); @@ -100,11 +101,11 @@ public void testAPPViewNaturalSortType() throws Exception { Map moreParams = ahsViewInstance.context().requestContext().moreParams(); String appTableColumnsMeta = moreParams.get("ui.dataTables.apps.init"); - Assert.assertTrue(appTableColumnsMeta.indexOf("natural") != -1); + assertTrue(appTableColumnsMeta.indexOf("natural") != -1); } @Test - public void testAboutPage() throws Exception { + void testAboutPage() throws Exception { Injector injector = WebAppTests.createMockInjector(ApplicationBaseProtocol.class, mockApplicationHistoryClientService(0, 0, 0)); @@ -118,23 +119,23 @@ public void testAboutPage() throws Exception { } @Test - public void testAppPage() throws Exception { + void testAppPage() throws Exception { Injector injector = WebAppTests.createMockInjector(ApplicationBaseProtocol.class, - mockApplicationHistoryClientService(1, 5, 1)); + mockApplicationHistoryClientService(1, 5, 1)); AppPage appPageInstance = injector.getInstance(AppPage.class); appPageInstance.render(); WebAppTests.flushOutput(injector); appPageInstance.set(YarnWebParams.APPLICATION_ID, ApplicationId - .newInstance(0, 1).toString()); + .newInstance(0, 1).toString()); appPageInstance.render(); WebAppTests.flushOutput(injector); } @Test - public void testAppPageNaturalSortType() throws Exception { + void testAppPageNaturalSortType() throws Exception { Injector injector = WebAppTests.createMockInjector(ApplicationBaseProtocol.class, mockApplicationHistoryClientService(1, 5, 1)); @@ -146,14 +147,14 @@ public void testAppPageNaturalSortType() throws Exception { appPageInstance.context().requestContext().moreParams(); String attemptsTableColumnsMeta = moreParams.get("ui.dataTables.attempts.init"); - Assert.assertTrue(attemptsTableColumnsMeta.indexOf("natural") != -1); + assertTrue(attemptsTableColumnsMeta.indexOf("natural") != -1); } @Test - public void testAppAttemptPage() throws Exception { + void testAppAttemptPage() throws Exception { Injector injector = WebAppTests.createMockInjector(ApplicationBaseProtocol.class, - mockApplicationHistoryClientService(1, 1, 5)); + mockApplicationHistoryClientService(1, 1, 5)); AppAttemptPage appAttemptPageInstance = injector.getInstance(AppAttemptPage.class); @@ -161,14 +162,14 @@ public void testAppAttemptPage() throws Exception { WebAppTests.flushOutput(injector); appAttemptPageInstance.set(YarnWebParams.APPLICATION_ATTEMPT_ID, - ApplicationAttemptId.newInstance(ApplicationId.newInstance(0, 1), 1) - .toString()); + ApplicationAttemptId.newInstance(ApplicationId.newInstance(0, 1), 1) + .toString()); appAttemptPageInstance.render(); WebAppTests.flushOutput(injector); } @Test - public void testAppAttemptPageNaturalSortType() throws Exception { + void testAppAttemptPageNaturalSortType() throws Exception { Injector injector = WebAppTests.createMockInjector(ApplicationBaseProtocol.class, mockApplicationHistoryClientService(1, 1, 5)); @@ -179,14 +180,14 @@ public void testAppAttemptPageNaturalSortType() throws Exception { Map moreParams = appAttemptPageInstance.context().requestContext().moreParams(); String tableColumnsMeta = moreParams.get("ui.dataTables.containers.init"); - Assert.assertTrue(tableColumnsMeta.indexOf("natural") != -1); + assertTrue(tableColumnsMeta.indexOf("natural") != -1); } @Test - public void testContainerPage() throws Exception { + void testContainerPage() throws Exception { Injector injector = WebAppTests.createMockInjector(ApplicationBaseProtocol.class, - mockApplicationHistoryClientService(1, 1, 1)); + mockApplicationHistoryClientService(1, 1, 1)); ContainerPage containerPageInstance = injector.getInstance(ContainerPage.class); @@ -194,11 +195,11 @@ public void testContainerPage() throws Exception { WebAppTests.flushOutput(injector); containerPageInstance.set( - YarnWebParams.CONTAINER_ID, - ContainerId - .newContainerId( - ApplicationAttemptId.newInstance(ApplicationId.newInstance(0, 1), 1), - 1).toString()); + YarnWebParams.CONTAINER_ID, + ContainerId + .newContainerId( + ApplicationAttemptId.newInstance(ApplicationId.newInstance(0, 1), 1), + 1).toString()); containerPageInstance.render(); WebAppTests.flushOutput(injector); } @@ -230,7 +231,7 @@ ApplicationHistoryClientService mockApplicationHistoryClientService(int numApps, class MockApplicationHistoryManagerImpl extends ApplicationHistoryManagerImpl { - public MockApplicationHistoryManagerImpl(ApplicationHistoryStore store) { + MockApplicationHistoryManagerImpl(ApplicationHistoryStore store) { super(); init(new YarnConfiguration()); start(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/TestAHSWebServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/TestAHSWebServices.java index 29890b2c7cd3f..fbcf9e9eb374e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/TestAHSWebServices.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/TestAHSWebServices.java @@ -18,16 +18,6 @@ package org.apache.hadoop.yarn.server.applicationhistoryservice.webapp; -import static org.assertj.core.api.Assertions.assertThat; -import static org.apache.hadoop.yarn.webapp.WebServicesTestUtils.assertResponseStatusCode; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.fail; -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.Mockito.doReturn; -import static org.mockito.Mockito.spy; - import java.net.HttpURLConnection; import java.net.URI; import java.net.URL; @@ -36,11 +26,31 @@ import java.util.Collections; import java.util.List; import java.util.Properties; - import javax.servlet.FilterConfig; import javax.servlet.ServletException; import javax.servlet.http.HttpServletResponse; import javax.ws.rs.core.MediaType; + +import com.google.inject.Guice; +import com.google.inject.Singleton; +import com.google.inject.servlet.ServletModule; +import com.sun.jersey.api.client.ClientResponse; +import com.sun.jersey.api.client.ClientResponse.Status; +import com.sun.jersey.api.client.GenericType; +import com.sun.jersey.api.client.UniformInterfaceException; +import com.sun.jersey.api.client.WebResource; +import com.sun.jersey.guice.spi.container.servlet.GuiceContainer; +import com.sun.jersey.test.framework.WebAppDescriptor; +import org.codehaus.jettison.json.JSONArray; +import org.codehaus.jettison.json.JSONException; +import org.codehaus.jettison.json.JSONObject; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Timeout; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.MethodSource; + import org.apache.commons.lang3.StringUtils; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; @@ -57,6 +67,7 @@ import org.apache.hadoop.yarn.api.records.NodeId; import org.apache.hadoop.yarn.api.records.YarnApplicationAttemptState; import org.apache.hadoop.yarn.api.records.YarnApplicationState; +import org.apache.hadoop.yarn.api.records.timeline.TimelineAbout; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.logaggregation.ContainerLogAggregationType; import org.apache.hadoop.yarn.logaggregation.ContainerLogFileInfo; @@ -72,41 +83,28 @@ import org.apache.hadoop.yarn.server.webapp.LogWebServiceUtils; import org.apache.hadoop.yarn.server.webapp.YarnWebServiceParams; import org.apache.hadoop.yarn.server.webapp.dao.ContainerLogsInfo; -import org.apache.hadoop.yarn.api.records.timeline.TimelineAbout; import org.apache.hadoop.yarn.util.timeline.TimelineUtils; import org.apache.hadoop.yarn.webapp.GenericExceptionHandler; import org.apache.hadoop.yarn.webapp.GuiceServletConfig; import org.apache.hadoop.yarn.webapp.JerseyTestBase; import org.apache.hadoop.yarn.webapp.WebServicesTestUtils; import org.apache.hadoop.yarn.webapp.util.WebAppUtils; -import org.codehaus.jettison.json.JSONArray; -import org.codehaus.jettison.json.JSONException; -import org.codehaus.jettison.json.JSONObject; -import org.junit.AfterClass; -import org.junit.Assert; -import org.junit.Before; -import org.junit.BeforeClass; -import org.junit.Test; -import org.junit.runner.RunWith; -import org.junit.runners.Parameterized; -import com.google.inject.Guice; -import com.google.inject.Singleton; -import com.google.inject.servlet.ServletModule; -import com.sun.jersey.api.client.ClientResponse; -import com.sun.jersey.api.client.ClientResponse.Status; -import com.sun.jersey.api.client.GenericType; -import com.sun.jersey.api.client.UniformInterfaceException; -import com.sun.jersey.api.client.WebResource; -import com.sun.jersey.guice.spi.container.servlet.GuiceContainer; -import com.sun.jersey.test.framework.WebAppDescriptor; +import static org.apache.hadoop.yarn.webapp.WebServicesTestUtils.assertResponseStatusCode; +import static org.assertj.core.api.Assertions.assertThat; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.fail; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.spy; -@RunWith(Parameterized.class) public class TestAHSWebServices extends JerseyTestBase { private static ApplicationHistoryClientService historyClientService; private static AHSWebServices ahsWebservice; - private static final String[] USERS = new String[] { "foo" , "bar" }; + private static final String[] USERS = new String[]{"foo", "bar"}; private static final int MAX_APPS = 6; private static Configuration conf; private static FileSystem fs; @@ -115,7 +113,7 @@ public class TestAHSWebServices extends JerseyTestBase { private static final String NM_WEBADDRESS = "test-nm-web-address:9999"; private static final String NM_ID = "test:1234"; - @BeforeClass + @BeforeAll public static void setupClass() throws Exception { conf = new YarnConfiguration(); TimelineStore store = @@ -153,7 +151,7 @@ protected void serviceStart() throws Exception { Guice.createInjector(new WebServletModule())); } - @AfterClass + @AfterAll public static void tearDownClass() throws Exception { if (historyClientService != null) { historyClientService.stop(); @@ -162,9 +160,8 @@ public static void tearDownClass() throws Exception { fs.delete(new Path(rootLogDir), true); } - @Parameterized.Parameters public static Collection rounds() { - return Arrays.asList(new Object[][] { { 0 }, { 1 } }); + return Arrays.asList(new Object[][]{{0}, {1}}); } private static class WebServletModule extends ServletModule { @@ -179,7 +176,7 @@ protected void configureServlets() { } } - @Before + @BeforeEach public void setUp() throws Exception { super.setUp(); GuiceServletConfig.setInjector( @@ -199,44 +196,43 @@ protected Properties getConfiguration(String configPrefix, } } - private int round; - - public TestAHSWebServices(int round) { + public TestAHSWebServices() { super(new WebAppDescriptor.Builder( - "org.apache.hadoop.yarn.server.applicationhistoryservice.webapp") - .contextListenerClass(GuiceServletConfig.class) - .filterClass(com.google.inject.servlet.GuiceFilter.class) - .contextPath("jersey-guice-filter").servletPath("/").build()); - this.round = round; + "org.apache.hadoop.yarn.server.applicationhistoryservice.webapp") + .contextListenerClass(GuiceServletConfig.class) + .filterClass(com.google.inject.servlet.GuiceFilter.class) + .contextPath("jersey-guice-filter").servletPath("/").build()); } - @Test - public void testInvalidApp() { + @MethodSource("rounds") + @ParameterizedTest + void testInvalidApp(int round) { ApplicationId appId = ApplicationId.newInstance(0, MAX_APPS + 1); WebResource r = resource(); ClientResponse response = r.path("ws").path("v1").path("applicationhistory").path("apps") - .path(appId.toString()) - .queryParam("user.name", USERS[round]) - .accept(MediaType.APPLICATION_JSON) - .get(ClientResponse.class); + .path(appId.toString()) + .queryParam("user.name", USERS[round]) + .accept(MediaType.APPLICATION_JSON) + .get(ClientResponse.class); assertResponseStatusCode("404 not found expected", Status.NOT_FOUND, response.getStatusInfo()); } - @Test - public void testInvalidAttempt() { + @MethodSource("rounds") + @ParameterizedTest + void testInvalidAttempt(int round) { ApplicationId appId = ApplicationId.newInstance(0, 1); ApplicationAttemptId appAttemptId = ApplicationAttemptId.newInstance(appId, MAX_APPS + 1); WebResource r = resource(); ClientResponse response = r.path("ws").path("v1").path("applicationhistory").path("apps") - .path(appId.toString()).path("appattempts") - .path(appAttemptId.toString()) - .queryParam("user.name", USERS[round]) - .accept(MediaType.APPLICATION_JSON) - .get(ClientResponse.class); + .path(appId.toString()).path("appattempts") + .path(appAttemptId.toString()) + .queryParam("user.name", USERS[round]) + .accept(MediaType.APPLICATION_JSON) + .get(ClientResponse.class); if (round == 1) { assertResponseStatusCode(Status.FORBIDDEN, response.getStatusInfo()); return; @@ -245,8 +241,9 @@ public void testInvalidAttempt() { Status.NOT_FOUND, response.getStatusInfo()); } - @Test - public void testInvalidContainer() throws Exception { + @MethodSource("rounds") + @ParameterizedTest + void testInvalidContainer(int round) throws Exception { ApplicationId appId = ApplicationId.newInstance(0, 1); ApplicationAttemptId appAttemptId = ApplicationAttemptId.newInstance(appId, 1); @@ -255,12 +252,12 @@ public void testInvalidContainer() throws Exception { WebResource r = resource(); ClientResponse response = r.path("ws").path("v1").path("applicationhistory").path("apps") - .path(appId.toString()).path("appattempts") - .path(appAttemptId.toString()).path("containers") - .path(containerId.toString()) - .queryParam("user.name", USERS[round]) - .accept(MediaType.APPLICATION_JSON) - .get(ClientResponse.class); + .path(appId.toString()).path("appattempts") + .path(appAttemptId.toString()).path("containers") + .path(containerId.toString()) + .queryParam("user.name", USERS[round]) + .accept(MediaType.APPLICATION_JSON) + .get(ClientResponse.class); if (round == 1) { assertResponseStatusCode(Status.FORBIDDEN, response.getStatusInfo()); return; @@ -269,27 +266,29 @@ public void testInvalidContainer() throws Exception { Status.NOT_FOUND, response.getStatusInfo()); } - @Test - public void testInvalidUri() throws JSONException, Exception { + @MethodSource("rounds") + @ParameterizedTest + void testInvalidUri(int round) throws JSONException, Exception { WebResource r = resource(); String responseStr = ""; try { responseStr = r.path("ws").path("v1").path("applicationhistory").path("bogus") - .queryParam("user.name", USERS[round]) - .accept(MediaType.APPLICATION_JSON).get(String.class); + .queryParam("user.name", USERS[round]) + .accept(MediaType.APPLICATION_JSON).get(String.class); fail("should have thrown exception on invalid uri"); } catch (UniformInterfaceException ue) { ClientResponse response = ue.getResponse(); assertResponseStatusCode(Status.NOT_FOUND, response.getStatusInfo()); WebServicesTestUtils.checkStringMatch( - "error string exists and shouldn't", "", responseStr); + "error string exists and shouldn't", "", responseStr); } } - @Test - public void testInvalidUri2() throws JSONException, Exception { + @MethodSource("rounds") + @ParameterizedTest + void testInvalidUri2(int round) throws JSONException, Exception { WebResource r = resource(); String responseStr = ""; try { @@ -300,31 +299,33 @@ public void testInvalidUri2() throws JSONException, Exception { ClientResponse response = ue.getResponse(); assertResponseStatusCode(Status.NOT_FOUND, response.getStatusInfo()); WebServicesTestUtils.checkStringMatch( - "error string exists and shouldn't", "", responseStr); + "error string exists and shouldn't", "", responseStr); } } - @Test - public void testInvalidAccept() throws JSONException, Exception { + @MethodSource("rounds") + @ParameterizedTest + void testInvalidAccept(int round) throws JSONException, Exception { WebResource r = resource(); String responseStr = ""; try { responseStr = r.path("ws").path("v1").path("applicationhistory") - .queryParam("user.name", USERS[round]) - .accept(MediaType.TEXT_PLAIN).get(String.class); + .queryParam("user.name", USERS[round]) + .accept(MediaType.TEXT_PLAIN).get(String.class); fail("should have thrown exception on invalid uri"); } catch (UniformInterfaceException ue) { ClientResponse response = ue.getResponse(); assertResponseStatusCode(Status.INTERNAL_SERVER_ERROR, response.getStatusInfo()); WebServicesTestUtils.checkStringMatch( - "error string exists and shouldn't", "", responseStr); + "error string exists and shouldn't", "", responseStr); } } - @Test - public void testAbout() throws Exception { + @MethodSource("rounds") + @ParameterizedTest + void testAbout(int round) throws Exception { WebResource r = resource(); ClientResponse response = r .path("ws").path("v1").path("applicationhistory").path("about") @@ -335,43 +336,45 @@ public void testAbout() throws Exception { TimelineAbout actualAbout = response.getEntity(TimelineAbout.class); TimelineAbout expectedAbout = TimelineUtils.createTimelineAbout("Generic History Service API"); - Assert.assertNotNull( - "Timeline service about response is null", actualAbout); - Assert.assertEquals(expectedAbout.getAbout(), actualAbout.getAbout()); - Assert.assertEquals(expectedAbout.getTimelineServiceVersion(), + assertNotNull( + actualAbout, "Timeline service about response is null"); + assertEquals(expectedAbout.getAbout(), actualAbout.getAbout()); + assertEquals(expectedAbout.getTimelineServiceVersion(), actualAbout.getTimelineServiceVersion()); - Assert.assertEquals(expectedAbout.getTimelineServiceBuildVersion(), + assertEquals(expectedAbout.getTimelineServiceBuildVersion(), actualAbout.getTimelineServiceBuildVersion()); - Assert.assertEquals(expectedAbout.getTimelineServiceVersionBuiltOn(), + assertEquals(expectedAbout.getTimelineServiceVersionBuiltOn(), actualAbout.getTimelineServiceVersionBuiltOn()); - Assert.assertEquals(expectedAbout.getHadoopVersion(), + assertEquals(expectedAbout.getHadoopVersion(), actualAbout.getHadoopVersion()); - Assert.assertEquals(expectedAbout.getHadoopBuildVersion(), + assertEquals(expectedAbout.getHadoopBuildVersion(), actualAbout.getHadoopBuildVersion()); - Assert.assertEquals(expectedAbout.getHadoopVersionBuiltOn(), + assertEquals(expectedAbout.getHadoopVersionBuiltOn(), actualAbout.getHadoopVersionBuiltOn()); } - @Test - public void testAppsQuery() throws Exception { + @MethodSource("rounds") + @ParameterizedTest + void testAppsQuery(int round) throws Exception { WebResource r = resource(); ClientResponse response = r.path("ws").path("v1").path("applicationhistory").path("apps") - .queryParam("state", YarnApplicationState.FINISHED.toString()) - .queryParam("user.name", USERS[round]) - .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class); + .queryParam("state", YarnApplicationState.FINISHED.toString()) + .queryParam("user.name", USERS[round]) + .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class); assertEquals(MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8, response.getType().toString()); JSONObject json = response.getEntity(JSONObject.class); - assertEquals("incorrect number of elements", 1, json.length()); + assertEquals(1, json.length(), "incorrect number of elements"); JSONObject apps = json.getJSONObject("apps"); - assertEquals("incorrect number of elements", 1, apps.length()); + assertEquals(1, apps.length(), "incorrect number of elements"); JSONArray array = apps.getJSONArray("app"); - assertEquals("incorrect number of elements", MAX_APPS, array.length()); + assertEquals(MAX_APPS, array.length(), "incorrect number of elements"); } - @Test - public void testQueueQuery() throws Exception { + @MethodSource("rounds") + @ParameterizedTest + void testQueueQuery(int round) throws Exception { WebResource r = resource(); ClientResponse response = r.path("ws").path("v1").path("applicationhistory").path("apps") @@ -382,28 +385,30 @@ public void testQueueQuery() throws Exception { assertEquals(MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8, response.getType().toString()); JSONObject json = response.getEntity(JSONObject.class); - assertEquals("incorrect number of elements", 1, json.length()); + assertEquals(1, json.length(), "incorrect number of elements"); JSONObject apps = json.getJSONObject("apps"); - assertEquals("incorrect number of elements", 1, apps.length()); + assertEquals(1, apps.length(), "incorrect number of elements"); JSONArray array = apps.getJSONArray("app"); - assertEquals("incorrect number of elements", MAX_APPS - 1, - array.length()); + assertEquals(MAX_APPS - 1, + array.length(), + "incorrect number of elements"); } - @Test - public void testSingleApp() throws Exception { + @MethodSource("rounds") + @ParameterizedTest + void testSingleApp(int round) throws Exception { ApplicationId appId = ApplicationId.newInstance(0, 1); WebResource r = resource(); ClientResponse response = r.path("ws").path("v1").path("applicationhistory").path("apps") - .path(appId.toString()) - .queryParam("user.name", USERS[round]) - .accept(MediaType.APPLICATION_JSON) - .get(ClientResponse.class); + .path(appId.toString()) + .queryParam("user.name", USERS[round]) + .accept(MediaType.APPLICATION_JSON) + .get(ClientResponse.class); assertEquals(MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8, response.getType().toString()); JSONObject json = response.getEntity(JSONObject.class); - assertEquals("incorrect number of elements", 1, json.length()); + assertEquals(1, json.length(), "incorrect number of elements"); JSONObject app = json.getJSONObject("app"); assertEquals(appId.toString(), app.getString("appId")); assertEquals("test app", app.get("name")); @@ -414,23 +419,24 @@ public void testSingleApp() throws Exception { assertEquals("user1", app.get("user")); assertEquals("test app type", app.get("type")); assertEquals(FinalApplicationStatus.UNDEFINED.toString(), - app.get("finalAppStatus")); + app.get("finalAppStatus")); assertEquals(YarnApplicationState.FINISHED.toString(), app.get("appState")); - assertNotNull("Aggregate resource allocation is null", - app.get("aggregateResourceAllocation")); - assertNotNull("Aggregate Preempted Resource Allocation is null", - app.get("aggregatePreemptedResourceAllocation")); + assertNotNull(app.get("aggregateResourceAllocation"), + "Aggregate resource allocation is null"); + assertNotNull(app.get("aggregatePreemptedResourceAllocation"), + "Aggregate Preempted Resource Allocation is null"); } - @Test - public void testMultipleAttempts() throws Exception { + @MethodSource("rounds") + @ParameterizedTest + void testMultipleAttempts(int round) throws Exception { ApplicationId appId = ApplicationId.newInstance(0, 1); WebResource r = resource(); ClientResponse response = r.path("ws").path("v1").path("applicationhistory").path("apps") - .path(appId.toString()).path("appattempts") - .queryParam("user.name", USERS[round]) - .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class); + .path(appId.toString()).path("appattempts") + .queryParam("user.name", USERS[round]) + .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class); if (round == 1) { assertResponseStatusCode(Status.FORBIDDEN, response.getStatusInfo()); return; @@ -438,26 +444,27 @@ public void testMultipleAttempts() throws Exception { assertEquals(MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8, response.getType().toString()); JSONObject json = response.getEntity(JSONObject.class); - assertEquals("incorrect number of elements", 1, json.length()); + assertEquals(1, json.length(), "incorrect number of elements"); JSONObject appAttempts = json.getJSONObject("appAttempts"); - assertEquals("incorrect number of elements", 1, appAttempts.length()); + assertEquals(1, appAttempts.length(), "incorrect number of elements"); JSONArray array = appAttempts.getJSONArray("appAttempt"); - assertEquals("incorrect number of elements", MAX_APPS, array.length()); + assertEquals(MAX_APPS, array.length(), "incorrect number of elements"); } - @Test - public void testSingleAttempt() throws Exception { + @MethodSource("rounds") + @ParameterizedTest + void testSingleAttempt(int round) throws Exception { ApplicationId appId = ApplicationId.newInstance(0, 1); ApplicationAttemptId appAttemptId = ApplicationAttemptId.newInstance(appId, 1); WebResource r = resource(); ClientResponse response = r.path("ws").path("v1").path("applicationhistory").path("apps") - .path(appId.toString()).path("appattempts") - .path(appAttemptId.toString()) - .queryParam("user.name", USERS[round]) - .accept(MediaType.APPLICATION_JSON) - .get(ClientResponse.class); + .path(appId.toString()).path("appattempts") + .path(appAttemptId.toString()) + .queryParam("user.name", USERS[round]) + .accept(MediaType.APPLICATION_JSON) + .get(ClientResponse.class); if (round == 1) { assertResponseStatusCode(Status.FORBIDDEN, response.getStatusInfo()); return; @@ -465,29 +472,30 @@ public void testSingleAttempt() throws Exception { assertEquals(MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8, response.getType().toString()); JSONObject json = response.getEntity(JSONObject.class); - assertEquals("incorrect number of elements", 1, json.length()); + assertEquals(1, json.length(), "incorrect number of elements"); JSONObject appAttempt = json.getJSONObject("appAttempt"); assertEquals(appAttemptId.toString(), appAttempt.getString("appAttemptId")); assertEquals("test host", appAttempt.getString("host")); assertEquals("test diagnostics info", - appAttempt.getString("diagnosticsInfo")); + appAttempt.getString("diagnosticsInfo")); assertEquals("test tracking url", appAttempt.getString("trackingUrl")); assertEquals(YarnApplicationAttemptState.FINISHED.toString(), - appAttempt.get("appAttemptState")); + appAttempt.get("appAttemptState")); } - @Test - public void testMultipleContainers() throws Exception { + @MethodSource("rounds") + @ParameterizedTest + void testMultipleContainers(int round) throws Exception { ApplicationId appId = ApplicationId.newInstance(0, 1); ApplicationAttemptId appAttemptId = ApplicationAttemptId.newInstance(appId, 1); WebResource r = resource(); ClientResponse response = r.path("ws").path("v1").path("applicationhistory").path("apps") - .path(appId.toString()).path("appattempts") - .path(appAttemptId.toString()).path("containers") - .queryParam("user.name", USERS[round]) - .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class); + .path(appId.toString()).path("appattempts") + .path(appAttemptId.toString()).path("containers") + .queryParam("user.name", USERS[round]) + .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class); if (round == 1) { assertResponseStatusCode(Status.FORBIDDEN, response.getStatusInfo()); return; @@ -495,15 +503,16 @@ public void testMultipleContainers() throws Exception { assertEquals(MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8, response.getType().toString()); JSONObject json = response.getEntity(JSONObject.class); - assertEquals("incorrect number of elements", 1, json.length()); + assertEquals(1, json.length(), "incorrect number of elements"); JSONObject containers = json.getJSONObject("containers"); - assertEquals("incorrect number of elements", 1, containers.length()); + assertEquals(1, containers.length(), "incorrect number of elements"); JSONArray array = containers.getJSONArray("container"); - assertEquals("incorrect number of elements", MAX_APPS, array.length()); + assertEquals(MAX_APPS, array.length(), "incorrect number of elements"); } - @Test - public void testSingleContainer() throws Exception { + @MethodSource("rounds") + @ParameterizedTest + void testSingleContainer(int round) throws Exception { ApplicationId appId = ApplicationId.newInstance(0, 1); ApplicationAttemptId appAttemptId = ApplicationAttemptId.newInstance(appId, 1); @@ -511,12 +520,12 @@ public void testSingleContainer() throws Exception { WebResource r = resource(); ClientResponse response = r.path("ws").path("v1").path("applicationhistory").path("apps") - .path(appId.toString()).path("appattempts") - .path(appAttemptId.toString()).path("containers") - .path(containerId.toString()) - .queryParam("user.name", USERS[round]) - .accept(MediaType.APPLICATION_JSON) - .get(ClientResponse.class); + .path(appId.toString()).path("appattempts") + .path(appAttemptId.toString()).path("containers") + .path(containerId.toString()) + .queryParam("user.name", USERS[round]) + .accept(MediaType.APPLICATION_JSON) + .get(ClientResponse.class); if (round == 1) { assertResponseStatusCode(Status.FORBIDDEN, response.getStatusInfo()); return; @@ -524,14 +533,14 @@ public void testSingleContainer() throws Exception { assertEquals(MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8, response.getType().toString()); JSONObject json = response.getEntity(JSONObject.class); - assertEquals("incorrect number of elements", 1, json.length()); + assertEquals(1, json.length(), "incorrect number of elements"); JSONObject container = json.getJSONObject("container"); assertEquals(containerId.toString(), container.getString("containerId")); assertEquals("test diagnostics info", container.getString("diagnosticsInfo")); assertEquals("-1", container.getString("allocatedMB")); assertEquals("-1", container.getString("allocatedVCores")); assertEquals(NodeId.newInstance("test host", 100).toString(), - container.getString("assignedNodeId")); + container.getString("assignedNodeId")); assertEquals("-1", container.getString("priority")); Configuration conf = new YarnConfiguration(); assertEquals(WebAppUtils.getHttpSchemePrefix(conf) + @@ -539,11 +548,13 @@ public void testSingleContainer() throws Exception { "/applicationhistory/logs/test host:100/container_0_0001_01_000001/" + "container_0_0001_01_000001/user1", container.getString("logUrl")); assertEquals(ContainerState.COMPLETE.toString(), - container.getString("containerState")); + container.getString("containerState")); } - @Test(timeout = 10000) - public void testContainerLogsForFinishedApps() throws Exception { + @MethodSource("rounds") + @ParameterizedTest + @Timeout(10000) + void testContainerLogsForFinishedApps(int round) throws Exception { String fileName = "syslog"; String user = "user1"; NodeId nodeId = NodeId.newInstance("test host", 100); @@ -553,7 +564,6 @@ public void testContainerLogsForFinishedApps() throws Exception { appId, 1); ContainerId containerId1 = ContainerId.newContainerId(appAttemptId, 1); ContainerId containerId100 = ContainerId.newContainerId(appAttemptId, 100); - TestContainerLogsUtils.createContainerLogFileInRemoteFS(conf, fs, rootLogDir, appId, Collections.singletonMap(containerId1, "Hello." + containerId1), @@ -573,7 +583,6 @@ public void testContainerLogsForFinishedApps() throws Exception { .get(ClientResponse.class); String responseText = response.getEntity(String.class); assertTrue(responseText.contains("Hello." + containerId1)); - // Do the same test with new API r = resource(); response = r.path("ws").path("v1") @@ -584,7 +593,6 @@ public void testContainerLogsForFinishedApps() throws Exception { .get(ClientResponse.class); responseText = response.getEntity(String.class); assertTrue(responseText.contains("Hello." + containerId1)); - // test whether we can find container log from remote diretory if // the containerInfo for this container could not be fetched from AHS. r = resource(); @@ -596,7 +604,6 @@ public void testContainerLogsForFinishedApps() throws Exception { .get(ClientResponse.class); responseText = response.getEntity(String.class); assertTrue(responseText.contains("Hello." + containerId100)); - // Do the same test with new API r = resource(); response = r.path("ws").path("v1") @@ -607,14 +614,12 @@ public void testContainerLogsForFinishedApps() throws Exception { .get(ClientResponse.class); responseText = response.getEntity(String.class); assertTrue(responseText.contains("Hello." + containerId100)); - // create an application which can not be found from AHS ApplicationId appId100 = ApplicationId.newInstance(0, 100); ApplicationAttemptId appAttemptId100 = ApplicationAttemptId.newInstance( appId100, 1); ContainerId containerId1ForApp100 = ContainerId.newContainerId( appAttemptId100, 1); - TestContainerLogsUtils.createContainerLogFileInRemoteFS(conf, fs, rootLogDir, appId100, Collections.singletonMap(containerId1ForApp100, @@ -634,7 +639,6 @@ public void testContainerLogsForFinishedApps() throws Exception { "End of LogType:syslog".length() + 50) + "\n\n"; int tailTextSize = "\nEnd of LogType:syslog\n".getBytes().length + tailEndSeparator.getBytes().length; - String logMessage = "Hello." + containerId1ForApp100; int fileContentSize = logMessage.getBytes().length; // specify how many bytes we should get from logs @@ -653,9 +657,8 @@ public void testContainerLogsForFinishedApps() throws Exception { (fullTextSize - fileContentSize) + 5); assertTrue(fullTextSize >= responseText.getBytes().length); assertEquals(new String(responseText.getBytes(), - (fullTextSize - fileContentSize - tailTextSize), 5), + (fullTextSize - fileContentSize - tailTextSize), 5), new String(logMessage.getBytes(), 0, 5)); - // specify how many bytes we should get from logs // if we specify a negative number, it would get the last n bytes from // container log @@ -672,9 +675,8 @@ public void testContainerLogsForFinishedApps() throws Exception { (fullTextSize - fileContentSize) + 5); assertTrue(fullTextSize >= responseText.getBytes().length); assertEquals(new String(responseText.getBytes(), - (fullTextSize - fileContentSize - tailTextSize), 5), + (fullTextSize - fileContentSize - tailTextSize), 5), new String(logMessage.getBytes(), fileContentSize - 5, 5)); - // specify the bytes which is larger than the actual file size, // we would get the full logs r = resource(); @@ -700,8 +702,10 @@ public void testContainerLogsForFinishedApps() throws Exception { assertThat(responseText.getBytes()).hasSize(fullTextSize); } - @Test(timeout = 10000) - public void testContainerLogsForRunningApps() throws Exception { + @MethodSource("rounds") + @ParameterizedTest + @Timeout(10000) + void testContainerLogsForRunningApps(int round) throws Exception { String fileName = "syslog"; String user = "user1"; ApplicationId appId = ApplicationId.newInstance( @@ -826,8 +830,10 @@ public void testContainerLogsForRunningApps() throws Exception { + ContainerLogAggregationType.AGGREGATED)); } - @Test(timeout = 10000) - public void testContainerLogsMetaForRunningApps() throws Exception { + @MethodSource("rounds") + @ParameterizedTest + @Timeout(10000) + void testContainerLogsMetaForRunningApps(int round) throws Exception { String user = "user1"; ApplicationId appId = ApplicationId.newInstance( 1234, 1); @@ -883,10 +889,11 @@ public void testContainerLogsMetaForRunningApps() throws Exception { .get(ClientResponse.class); List responseText = response.getEntity(new GenericType< - List>(){}); + List>(){ + }); assertTrue(responseText.size() == 2); for (ContainerLogsInfo logInfo : responseText) { - if(logInfo.getLogType().equals( + if (logInfo.getLogType().equals( ContainerLogAggregationType.AGGREGATED.toString())) { List logMeta = logInfo .getContainerLogsInfo(); @@ -911,10 +918,11 @@ public void testContainerLogsMetaForRunningApps() throws Exception { .accept(MediaType.APPLICATION_JSON) .get(ClientResponse.class); responseText = response.getEntity(new GenericType< - List>(){}); + List>(){ + }); assertTrue(responseText.size() == 2); for (ContainerLogsInfo logInfo : responseText) { - if(logInfo.getLogType().equals( + if (logInfo.getLogType().equals( ContainerLogAggregationType.AGGREGATED.toString())) { List logMeta = logInfo .getContainerLogsInfo(); @@ -929,8 +937,10 @@ public void testContainerLogsMetaForRunningApps() throws Exception { } } - @Test(timeout = 10000) - public void testContainerLogsMetaForFinishedApps() throws Exception { + @MethodSource("rounds") + @ParameterizedTest + @Timeout(10000) + void testContainerLogsMetaForFinishedApps(int round) throws Exception { ApplicationId appId = ApplicationId.newInstance(0, 1); ApplicationAttemptId appAttemptId = ApplicationAttemptId.newInstance(appId, 1); @@ -951,7 +961,8 @@ public void testContainerLogsMetaForFinishedApps() throws Exception { .accept(MediaType.APPLICATION_JSON) .get(ClientResponse.class); List responseText = response.getEntity(new GenericType< - List>(){}); + List>(){ + }); assertTrue(responseText.size() == 1); assertEquals(responseText.get(0).getLogType(), ContainerLogAggregationType.AGGREGATED.toString()); @@ -971,7 +982,7 @@ private static String getRedirectURL(String url) { // do not automatically follow the redirection // otherwise we get too many redirections exception conn.setInstanceFollowRedirects(false); - if(conn.getResponseCode() == HttpServletResponse.SC_TEMPORARY_REDIRECT) { + if (conn.getResponseCode() == HttpServletResponse.SC_TEMPORARY_REDIRECT) { redirectUrl = conn.getHeaderField("Location"); } } catch (Exception e) { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/TestGenericObjectMapper.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/TestGenericObjectMapper.java index ab0923173b937..6c671f7476339 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/TestGenericObjectMapper.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/TestGenericObjectMapper.java @@ -17,52 +17,56 @@ */ package org.apache.hadoop.yarn.server.timeline; -import org.apache.hadoop.classification.InterfaceAudience; -import org.apache.hadoop.classification.InterfaceStability; -import org.apache.hadoop.io.WritableComparator; -import org.apache.hadoop.yarn.server.timeline.GenericObjectMapper; -import org.junit.Test; - import java.io.IOException; import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; -import static org.junit.Assert.assertEquals; +import org.junit.jupiter.api.Test; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.io.WritableComparator; + +import static org.junit.jupiter.api.Assertions.assertEquals; @InterfaceAudience.Private @InterfaceStability.Unstable public class TestGenericObjectMapper { @Test - public void testEncoding() { + void testEncoding() { testEncoding(Long.MAX_VALUE); testEncoding(Long.MIN_VALUE); - testEncoding(0l); - testEncoding(128l); - testEncoding(256l); - testEncoding(512l); - testEncoding(-256l); + testEncoding(0L); + testEncoding(128L); + testEncoding(256L); + testEncoding(512L); + testEncoding(-256L); } private static void testEncoding(long l) { byte[] b = GenericObjectMapper.writeReverseOrderedLong(l); - assertEquals("error decoding", l, - GenericObjectMapper.readReverseOrderedLong(b, 0)); + assertEquals(l, + GenericObjectMapper.readReverseOrderedLong(b, 0), + "error decoding"); byte[] buf = new byte[16]; System.arraycopy(b, 0, buf, 5, 8); - assertEquals("error decoding at offset", l, - GenericObjectMapper.readReverseOrderedLong(buf, 5)); + assertEquals(l, + GenericObjectMapper.readReverseOrderedLong(buf, 5), + "error decoding at offset"); if (l > Long.MIN_VALUE) { byte[] a = GenericObjectMapper.writeReverseOrderedLong(l-1); - assertEquals("error preserving ordering", 1, - WritableComparator.compareBytes(a, 0, a.length, b, 0, b.length)); + assertEquals(1, + WritableComparator.compareBytes(a, 0, a.length, b, 0, b.length), + "error preserving ordering"); } if (l < Long.MAX_VALUE) { byte[] c = GenericObjectMapper.writeReverseOrderedLong(l+1); - assertEquals("error preserving ordering", 1, - WritableComparator.compareBytes(b, 0, b.length, c, 0, c.length)); + assertEquals(1, + WritableComparator.compareBytes(b, 0, b.length, c, 0, c.length), + "error preserving ordering"); } } @@ -71,20 +75,20 @@ private static void verify(Object o) throws IOException { } @Test - public void testValueTypes() throws IOException { + void testValueTypes() throws IOException { verify(Integer.MAX_VALUE); verify(Integer.MIN_VALUE); assertEquals(Integer.MAX_VALUE, GenericObjectMapper.read( GenericObjectMapper.write((long) Integer.MAX_VALUE))); assertEquals(Integer.MIN_VALUE, GenericObjectMapper.read( GenericObjectMapper.write((long) Integer.MIN_VALUE))); - verify((long)Integer.MAX_VALUE + 1l); - verify((long)Integer.MIN_VALUE - 1l); + verify((long) Integer.MAX_VALUE + 1L); + verify((long) Integer.MIN_VALUE - 1L); verify(Long.MAX_VALUE); verify(Long.MIN_VALUE); - assertEquals(42, GenericObjectMapper.read(GenericObjectMapper.write(42l))); + assertEquals(42, GenericObjectMapper.read(GenericObjectMapper.write(42L))); verify(42); verify(1.23); verify("abc"); @@ -93,9 +97,9 @@ public void testValueTypes() throws IOException { list.add("123"); list.add("abc"); verify(list); - Map map = new HashMap(); - map.put("k1","v1"); - map.put("k2","v2"); + Map map = new HashMap(); + map.put("k1", "v1"); + map.put("k2", "v2"); verify(map); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/TestLeveldbTimelineStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/TestLeveldbTimelineStore.java index f68a1c41c612c..c7166a1268c55 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/TestLeveldbTimelineStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/TestLeveldbTimelineStore.java @@ -17,10 +17,6 @@ */ package org.apache.hadoop.yarn.server.timeline; -import static org.apache.hadoop.yarn.server.timeline.GenericObjectMapper.writeReverseOrderedLong; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotNull; - import java.io.File; import java.io.FileFilter; import java.io.IOException; @@ -29,6 +25,14 @@ import java.util.Map; import java.util.Set; +import org.fusesource.leveldbjni.JniDBFactory; +import org.iq80.leveldb.DBException; +import org.iq80.leveldb.Options; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.mockito.Mockito; + import org.apache.commons.io.filefilter.WildcardFileFilter; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; @@ -46,14 +50,13 @@ import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.server.records.Version; import org.apache.hadoop.yarn.server.utils.LeveldbIterator; -import org.fusesource.leveldbjni.JniDBFactory; -import org.iq80.leveldb.DBException; -import org.iq80.leveldb.Options; -import org.junit.After; -import org.junit.Assert; -import org.junit.Before; -import org.junit.Test; -import org.mockito.Mockito; + +import static org.apache.hadoop.yarn.server.timeline.GenericObjectMapper.writeReverseOrderedLong; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.fail; @InterfaceAudience.Private @InterfaceStability.Unstable @@ -62,7 +65,7 @@ public class TestLeveldbTimelineStore extends TimelineStoreTestUtils { private File fsPath; private Configuration config = new YarnConfiguration(); - @Before + @BeforeEach public void setup() throws Exception { fsContext = FileContext.getLocalFSFileContext(); fsPath = new File("target", this.getClass().getSimpleName() + @@ -79,14 +82,14 @@ public void setup() throws Exception { loadTestDomainData(); } - @After + @AfterEach public void tearDown() throws Exception { store.stop(); fsContext.delete(new Path(fsPath.getAbsolutePath()), true); } @Test - public void testRootDirPermission() throws IOException { + void testRootDirPermission() throws IOException { FileSystem fs = FileSystem.getLocal(new YarnConfiguration()); FileStatus file = fs.getFileStatus( new Path(fsPath.getAbsolutePath(), LeveldbTimelineStore.FILENAME)); @@ -133,7 +136,7 @@ public void testGetEvents() throws IOException { } @Test - public void testCacheSizes() { + void testCacheSizes() { Configuration conf = new Configuration(); assertEquals(10000, LeveldbTimelineStore.getStartTimeReadCacheSize(conf)); assertEquals(10000, LeveldbTimelineStore.getStartTimeWriteCacheSize(conf)); @@ -165,8 +168,8 @@ private boolean deleteNextEntity(String entityType, byte[] ts) } @Test - public void testGetEntityTypes() throws IOException { - List entityTypes = ((LeveldbTimelineStore)store).getEntityTypes(); + void testGetEntityTypes() throws IOException { + List entityTypes = ((LeveldbTimelineStore) store).getEntityTypes(); assertEquals(7, entityTypes.size()); assertEquals("ACL_ENTITY_TYPE_1", entityTypes.get(0)); assertEquals("OLD_ENTITY_TYPE_1", entityTypes.get(1)); @@ -177,7 +180,7 @@ public void testGetEntityTypes() throws IOException { } @Test - public void testDeleteEntities() throws IOException, InterruptedException { + void testDeleteEntities() throws IOException, InterruptedException { assertEquals(3, getEntities("type_1").size()); assertEquals(1, getEntities("type_2").size()); @@ -191,7 +194,7 @@ public void testDeleteEntities() throws IOException, InterruptedException { List entities = getEntities("type_2"); assertEquals(1, entities.size()); verifyEntityInfo(entityId2, entityType2, events2, Collections.singletonMap( - entityType1, Collections.singleton(entityId1b)), EMPTY_PRIMARY_FILTERS, + entityType1, Collections.singleton(entityId1b)), EMPTY_PRIMARY_FILTERS, EMPTY_MAP, entities.get(0), domainId1); entities = getEntitiesWithPrimaryFilter("type_1", userFilter); assertEquals(2, entities.size()); @@ -201,20 +204,20 @@ public void testDeleteEntities() throws IOException, InterruptedException { verifyEntityInfo(entityId6, entityType1, EMPTY_EVENTS, EMPTY_REL_ENTITIES, primaryFilters, otherInfo, entities.get(1), domainId2); - ((LeveldbTimelineStore)store).discardOldEntities(0L); + ((LeveldbTimelineStore) store).discardOldEntities(0L); assertEquals(2, getEntities("type_1").size()); assertEquals(0, getEntities("type_2").size()); - assertEquals(6, ((LeveldbTimelineStore)store).getEntityTypes().size()); + assertEquals(6, ((LeveldbTimelineStore) store).getEntityTypes().size()); - ((LeveldbTimelineStore)store).discardOldEntities(123L); + ((LeveldbTimelineStore) store).discardOldEntities(123L); assertEquals(0, getEntities("type_1").size()); assertEquals(0, getEntities("type_2").size()); - assertEquals(0, ((LeveldbTimelineStore)store).getEntityTypes().size()); + assertEquals(0, ((LeveldbTimelineStore) store).getEntityTypes().size()); assertEquals(0, getEntitiesWithPrimaryFilter("type_1", userFilter).size()); } @Test - public void testDeleteEntitiesPrimaryFilters() + void testDeleteEntitiesPrimaryFilters() throws IOException, InterruptedException { Map> primaryFilter = Collections.singletonMap("user", Collections.singleton( @@ -243,28 +246,28 @@ public void testDeleteEntitiesPrimaryFilters() verifyEntityInfo(entityId6, entityType1, EMPTY_EVENTS, EMPTY_REL_ENTITIES, primaryFilters, otherInfo, entities.get(2), domainId2); - ((LeveldbTimelineStore)store).discardOldEntities(-123L); + ((LeveldbTimelineStore) store).discardOldEntities(-123L); assertEquals(1, getEntitiesWithPrimaryFilter("type_1", pfPair).size()); assertEquals(3, getEntitiesWithPrimaryFilter("type_1", userFilter).size()); - ((LeveldbTimelineStore)store).discardOldEntities(123L); + ((LeveldbTimelineStore) store).discardOldEntities(123L); assertEquals(0, getEntities("type_1").size()); assertEquals(0, getEntities("type_2").size()); - assertEquals(0, ((LeveldbTimelineStore)store).getEntityTypes().size()); + assertEquals(0, ((LeveldbTimelineStore) store).getEntityTypes().size()); assertEquals(0, getEntitiesWithPrimaryFilter("type_1", pfPair).size()); assertEquals(0, getEntitiesWithPrimaryFilter("type_1", userFilter).size()); } @Test - public void testFromTsWithDeletion() + void testFromTsWithDeletion() throws IOException, InterruptedException { long l = System.currentTimeMillis(); assertEquals(3, getEntitiesFromTs("type_1", l).size()); assertEquals(1, getEntitiesFromTs("type_2", l).size()); assertEquals(3, getEntitiesFromTsWithPrimaryFilter("type_1", userFilter, l).size()); - ((LeveldbTimelineStore)store).discardOldEntities(123L); + ((LeveldbTimelineStore) store).discardOldEntities(123L); assertEquals(0, getEntitiesFromTs("type_1", l).size()); assertEquals(0, getEntitiesFromTs("type_2", l).size()); assertEquals(0, getEntitiesFromTsWithPrimaryFilter("type_1", userFilter, @@ -284,22 +287,22 @@ public void testFromTsWithDeletion() } @Test - public void testCheckVersion() throws IOException { + void testCheckVersion() throws IOException { LeveldbTimelineStore dbStore = (LeveldbTimelineStore) store; // default version Version defaultVersion = dbStore.getCurrentVersion(); - Assert.assertEquals(defaultVersion, dbStore.loadVersion()); + assertEquals(defaultVersion, dbStore.loadVersion()); // compatible version Version compatibleVersion = Version.newInstance(defaultVersion.getMajorVersion(), - defaultVersion.getMinorVersion() + 2); + defaultVersion.getMinorVersion() + 2); dbStore.storeVersion(compatibleVersion); - Assert.assertEquals(compatibleVersion, dbStore.loadVersion()); + assertEquals(compatibleVersion, dbStore.loadVersion()); restartTimelineStore(); dbStore = (LeveldbTimelineStore) store; // overwrite the compatible version - Assert.assertEquals(defaultVersion, dbStore.loadVersion()); + assertEquals(defaultVersion, dbStore.loadVersion()); // incompatible version Version incompatibleVersion = Version.newInstance( @@ -307,24 +310,24 @@ public void testCheckVersion() throws IOException { dbStore.storeVersion(incompatibleVersion); try { restartTimelineStore(); - Assert.fail("Incompatible version, should expect fail here."); + fail("Incompatible version, should expect fail here."); } catch (ServiceStateException e) { - Assert.assertTrue("Exception message mismatch", - e.getMessage().contains("Incompatible version for timeline store")); + assertTrue(e.getMessage().contains("Incompatible version for timeline store"), + "Exception message mismatch"); } } @Test - public void testValidateConfig() throws IOException { + void testValidateConfig() throws IOException { Configuration copyConfig = new YarnConfiguration(config); try { Configuration newConfig = new YarnConfiguration(copyConfig); newConfig.setLong(YarnConfiguration.TIMELINE_SERVICE_TTL_MS, 0); config = newConfig; restartTimelineStore(); - Assert.fail(); + fail(); } catch (IllegalArgumentException e) { - Assert.assertTrue(e.getMessage().contains( + assertTrue(e.getMessage().contains( YarnConfiguration.TIMELINE_SERVICE_TTL_MS)); } try { @@ -333,9 +336,9 @@ public void testValidateConfig() throws IOException { YarnConfiguration.TIMELINE_SERVICE_LEVELDB_TTL_INTERVAL_MS, 0); config = newConfig; restartTimelineStore(); - Assert.fail(); + fail(); } catch (IllegalArgumentException e) { - Assert.assertTrue(e.getMessage().contains( + assertTrue(e.getMessage().contains( YarnConfiguration.TIMELINE_SERVICE_LEVELDB_TTL_INTERVAL_MS)); } try { @@ -344,9 +347,9 @@ public void testValidateConfig() throws IOException { YarnConfiguration.TIMELINE_SERVICE_LEVELDB_READ_CACHE_SIZE, -1); config = newConfig; restartTimelineStore(); - Assert.fail(); + fail(); } catch (IllegalArgumentException e) { - Assert.assertTrue(e.getMessage().contains( + assertTrue(e.getMessage().contains( YarnConfiguration.TIMELINE_SERVICE_LEVELDB_READ_CACHE_SIZE)); } try { @@ -357,12 +360,11 @@ public void testValidateConfig() throws IOException { 0); config = newConfig; restartTimelineStore(); - Assert.fail(); + fail(); } catch (IllegalArgumentException e) { - Assert - .assertTrue(e + assertTrue(e .getMessage().contains( - YarnConfiguration.TIMELINE_SERVICE_LEVELDB_START_TIME_READ_CACHE_SIZE)); + YarnConfiguration.TIMELINE_SERVICE_LEVELDB_START_TIME_READ_CACHE_SIZE)); } try { Configuration newConfig = new YarnConfiguration(copyConfig); @@ -372,10 +374,9 @@ public void testValidateConfig() throws IOException { 0); config = newConfig; restartTimelineStore(); - Assert.fail(); + fail(); } catch (IllegalArgumentException e) { - Assert - .assertTrue(e + assertTrue(e .getMessage() .contains( YarnConfiguration.TIMELINE_SERVICE_LEVELDB_START_TIME_WRITE_CACHE_SIZE)); @@ -405,7 +406,7 @@ public void testGetDomains() throws IOException { } @Test - public void testRelatingToNonExistingEntity() throws IOException { + void testRelatingToNonExistingEntity() throws IOException { TimelineEntity entityToStore = new TimelineEntity(); entityToStore.setEntityType("TEST_ENTITY_TYPE_1"); entityToStore.setEntityId("TEST_ENTITY_ID_1"); @@ -416,17 +417,17 @@ public void testRelatingToNonExistingEntity() throws IOException { store.put(entities); TimelineEntity entityToGet = store.getEntity("TEST_ENTITY_ID_2", "TEST_ENTITY_TYPE_2", null); - Assert.assertNotNull(entityToGet); - Assert.assertEquals("DEFAULT", entityToGet.getDomainId()); - Assert.assertEquals("TEST_ENTITY_TYPE_1", + assertNotNull(entityToGet); + assertEquals("DEFAULT", entityToGet.getDomainId()); + assertEquals("TEST_ENTITY_TYPE_1", entityToGet.getRelatedEntities().keySet().iterator().next()); - Assert.assertEquals("TEST_ENTITY_ID_1", + assertEquals("TEST_ENTITY_ID_1", entityToGet.getRelatedEntities().values().iterator().next() .iterator().next()); } @Test - public void testRelatingToOldEntityWithoutDomainId() throws IOException { + void testRelatingToOldEntityWithoutDomainId() throws IOException { // New entity is put in the default domain TimelineEntity entityToStore = new TimelineEntity(); entityToStore.setEntityType("NEW_ENTITY_TYPE_1"); @@ -439,11 +440,11 @@ public void testRelatingToOldEntityWithoutDomainId() throws IOException { TimelineEntity entityToGet = store.getEntity("OLD_ENTITY_ID_1", "OLD_ENTITY_TYPE_1", null); - Assert.assertNotNull(entityToGet); - Assert.assertNull(entityToGet.getDomainId()); - Assert.assertEquals("NEW_ENTITY_TYPE_1", + assertNotNull(entityToGet); + assertNull(entityToGet.getDomainId()); + assertEquals("NEW_ENTITY_TYPE_1", entityToGet.getRelatedEntities().keySet().iterator().next()); - Assert.assertEquals("NEW_ENTITY_ID_1", + assertEquals("NEW_ENTITY_ID_1", entityToGet.getRelatedEntities().values().iterator().next() .iterator().next()); @@ -456,26 +457,26 @@ public void testRelatingToOldEntityWithoutDomainId() throws IOException { entities = new TimelineEntities(); entities.addEntity(entityToStore); TimelinePutResponse response = store.put(entities); - Assert.assertEquals(1, response.getErrors().size()); - Assert.assertEquals(TimelinePutError.FORBIDDEN_RELATION, + assertEquals(1, response.getErrors().size()); + assertEquals(TimelinePutError.FORBIDDEN_RELATION, response.getErrors().get(0).getErrorCode()); entityToGet = store.getEntity("OLD_ENTITY_ID_1", "OLD_ENTITY_TYPE_1", null); - Assert.assertNotNull(entityToGet); - Assert.assertNull(entityToGet.getDomainId()); + assertNotNull(entityToGet); + assertNull(entityToGet.getDomainId()); // Still have one related entity - Assert.assertEquals(1, entityToGet.getRelatedEntities().keySet().size()); - Assert.assertEquals(1, entityToGet.getRelatedEntities().values() + assertEquals(1, entityToGet.getRelatedEntities().keySet().size()); + assertEquals(1, entityToGet.getRelatedEntities().values() .iterator().next().size()); } - @Test /** * Test that LevelDb repair is attempted at least once during * serviceInit for LeveldbTimelineStore in case open fails the * first time. */ - public void testLevelDbRepair() throws IOException { + @Test + void testLevelDbRepair() throws IOException { LeveldbTimelineStore store = new LeveldbTimelineStore(); JniDBFactory factory = Mockito.mock(JniDBFactory.class); @@ -496,8 +497,8 @@ public void testLevelDbRepair() throws IOException { Mockito.verify(factory, Mockito.times(1)) .repair(Mockito.any(File.class), Mockito.any(Options.class)); FileFilter fileFilter = new WildcardFileFilter( - "*" + LeveldbTimelineStore.BACKUP_EXT +"*"); - Assert.assertTrue(path.listFiles(fileFilter).length > 0); + "*" + LeveldbTimelineStore.BACKUP_EXT + "*"); + assertTrue(path.listFiles(fileFilter).length > 0); } finally { store.close(); fsContext.delete(new Path(path.getAbsolutePath()), true); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/TestMemoryTimelineStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/TestMemoryTimelineStore.java index 6ba1bfeca7e33..9a018dbad71a9 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/TestMemoryTimelineStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/TestMemoryTimelineStore.java @@ -18,18 +18,17 @@ package org.apache.hadoop.yarn.server.timeline; -import org.apache.hadoop.yarn.conf.YarnConfiguration; -import org.apache.hadoop.yarn.server.timeline.MemoryTimelineStore; -import org.apache.hadoop.yarn.server.timeline.TimelineStore; -import org.junit.After; -import org.junit.Before; -import org.junit.Test; - import java.io.IOException; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import org.apache.hadoop.yarn.conf.YarnConfiguration; + public class TestMemoryTimelineStore extends TimelineStoreTestUtils { - @Before + @BeforeEach public void setup() throws Exception { store = new MemoryTimelineStore(); store.init(new YarnConfiguration()); @@ -39,7 +38,7 @@ public void setup() throws Exception { loadTestDomainData(); } - @After + @AfterEach public void tearDown() throws Exception { store.stop(); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/TestRollingLevelDB.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/TestRollingLevelDB.java index d2d0860386cc8..44effe33a0d79 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/TestRollingLevelDB.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/TestRollingLevelDB.java @@ -19,14 +19,16 @@ import java.io.File; +import org.iq80.leveldb.DB; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.yarn.conf.YarnConfiguration; -import org.iq80.leveldb.DB; -import org.junit.Assert; -import org.junit.Before; -import org.junit.Test; + +import static org.junit.jupiter.api.Assertions.assertEquals; /** Test class for verification of RollingLevelDB. */ public class TestRollingLevelDB { @@ -53,7 +55,7 @@ public void setCurrentTimeMillis(long time) { } }; - @Before + @BeforeEach public void setup() throws Exception { lfs = FileSystem.getLocal(conf); File fsPath = new File("target", this.getClass().getSimpleName() + @@ -65,26 +67,26 @@ public void setup() throws Exception { } @Test - public void testInsertAfterRollPeriodRollsDB() throws Exception { + void testInsertAfterRollPeriodRollsDB() throws Exception { rollingLevelDB.init(conf); long now = rollingLevelDB.currentTimeMillis(); DB db = rollingLevelDB.getDBForStartTime(now); long startTime = rollingLevelDB.getStartTimeFor(db); - Assert.assertEquals("Received level db for incorrect start time", - rollingLevelDB.computeCurrentCheckMillis(now), - startTime); + assertEquals(rollingLevelDB.computeCurrentCheckMillis(now), + startTime, + "Received level db for incorrect start time"); now = rollingLevelDB.getNextRollingTimeMillis(); rollingLevelDB.setCurrentTimeMillis(now); db = rollingLevelDB.getDBForStartTime(now); startTime = rollingLevelDB.getStartTimeFor(db); - Assert.assertEquals("Received level db for incorrect start time", - rollingLevelDB.computeCurrentCheckMillis(now), - startTime); + assertEquals(rollingLevelDB.computeCurrentCheckMillis(now), + startTime, + "Received level db for incorrect start time"); } @Test - public void testInsertForPreviousPeriodAfterRollPeriodRollsDB() + void testInsertForPreviousPeriodAfterRollPeriodRollsDB() throws Exception { rollingLevelDB.init(conf); @@ -93,8 +95,8 @@ public void testInsertForPreviousPeriodAfterRollPeriodRollsDB() rollingLevelDB.setCurrentTimeMillis(now); DB db = rollingLevelDB.getDBForStartTime(now - 1); long startTime = rollingLevelDB.getStartTimeFor(db); - Assert.assertEquals("Received level db for incorrect start time", - rollingLevelDB.computeCurrentCheckMillis(now - 1), - startTime); + assertEquals(rollingLevelDB.computeCurrentCheckMillis(now - 1), + startTime, + "Received level db for incorrect start time"); } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/TestRollingLevelDBTimelineStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/TestRollingLevelDBTimelineStore.java index ff5682c43e436..b20497728bab1 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/TestRollingLevelDBTimelineStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/TestRollingLevelDBTimelineStore.java @@ -17,13 +17,18 @@ */ package org.apache.hadoop.yarn.server.timeline; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotNull; - import java.io.File; import java.io.FilenameFilter; import java.io.IOException; +import org.eclipse.jetty.util.log.Log; +import org.fusesource.leveldbjni.JniDBFactory; +import org.iq80.leveldb.Options; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.mockito.Mockito; + import org.apache.commons.io.filefilter.WildcardFileFilter; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; @@ -41,14 +46,10 @@ import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.server.records.Version; -import org.fusesource.leveldbjni.JniDBFactory; -import org.iq80.leveldb.Options; -import org.junit.After; -import org.junit.Assert; -import org.junit.Before; -import org.junit.Test; -import org.eclipse.jetty.util.log.Log; -import org.mockito.Mockito; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.fail; /** Test class to verify RollingLevelDBTimelineStore. */ @InterfaceAudience.Private @@ -58,7 +59,7 @@ public class TestRollingLevelDBTimelineStore extends TimelineStoreTestUtils { private File fsPath; private Configuration config = new YarnConfiguration(); - @Before + @BeforeEach public void setup() throws Exception { fsContext = FileContext.getLocalFSFileContext(); fsPath = new File("target", this.getClass().getSimpleName() + @@ -75,14 +76,14 @@ public void setup() throws Exception { loadTestDomainData(); } - @After + @AfterEach public void tearDown() throws Exception { store.stop(); fsContext.delete(new Path(fsPath.getAbsolutePath()), true); } @Test - public void testRootDirPermission() throws IOException { + void testRootDirPermission() throws IOException { FileSystem fs = FileSystem.getLocal(new YarnConfiguration()); FileStatus file = fs.getFileStatus(new Path(fsPath.getAbsolutePath(), RollingLevelDBTimelineStore.FILENAME)); @@ -130,7 +131,7 @@ public void testGetEvents() throws IOException { } @Test - public void testCacheSizes() { + void testCacheSizes() { Configuration conf = new Configuration(); assertEquals(10000, RollingLevelDBTimelineStore.getStartTimeReadCacheSize(conf)); @@ -150,48 +151,48 @@ public void testCacheSizes() { } @Test - public void testCheckVersion() throws IOException { + void testCheckVersion() throws IOException { RollingLevelDBTimelineStore dbStore = (RollingLevelDBTimelineStore) store; // default version Version defaultVersion = dbStore.getCurrentVersion(); - Assert.assertEquals(defaultVersion, dbStore.loadVersion()); + assertEquals(defaultVersion, dbStore.loadVersion()); // compatible version Version compatibleVersion = Version.newInstance(defaultVersion.getMajorVersion(), - defaultVersion.getMinorVersion() + 2); + defaultVersion.getMinorVersion() + 2); dbStore.storeVersion(compatibleVersion); - Assert.assertEquals(compatibleVersion, dbStore.loadVersion()); + assertEquals(compatibleVersion, dbStore.loadVersion()); restartTimelineStore(); dbStore = (RollingLevelDBTimelineStore) store; // overwrite the compatible version - Assert.assertEquals(defaultVersion, dbStore.loadVersion()); + assertEquals(defaultVersion, dbStore.loadVersion()); // incompatible version Version incompatibleVersion = Version.newInstance(defaultVersion.getMajorVersion() + 1, - defaultVersion.getMinorVersion()); + defaultVersion.getMinorVersion()); dbStore.storeVersion(incompatibleVersion); try { restartTimelineStore(); - Assert.fail("Incompatible version, should expect fail here."); + fail("Incompatible version, should expect fail here."); } catch (ServiceStateException e) { - Assert.assertTrue("Exception message mismatch", - e.getMessage().contains("Incompatible version for timeline store")); + assertTrue(e.getMessage().contains("Incompatible version for timeline store"), + "Exception message mismatch"); } } @Test - public void testValidateConfig() throws IOException { + void testValidateConfig() throws IOException { Configuration copyConfig = new YarnConfiguration(config); try { Configuration newConfig = new YarnConfiguration(copyConfig); newConfig.setLong(YarnConfiguration.TIMELINE_SERVICE_TTL_MS, 0); config = newConfig; restartTimelineStore(); - Assert.fail(); + fail(); } catch (IllegalArgumentException e) { - Assert.assertTrue(e.getMessage().contains( + assertTrue(e.getMessage().contains( YarnConfiguration.TIMELINE_SERVICE_TTL_MS)); } try { @@ -200,9 +201,9 @@ public void testValidateConfig() throws IOException { YarnConfiguration.TIMELINE_SERVICE_LEVELDB_TTL_INTERVAL_MS, 0); config = newConfig; restartTimelineStore(); - Assert.fail(); + fail(); } catch (IllegalArgumentException e) { - Assert.assertTrue(e.getMessage().contains( + assertTrue(e.getMessage().contains( YarnConfiguration.TIMELINE_SERVICE_LEVELDB_TTL_INTERVAL_MS)); } try { @@ -211,9 +212,9 @@ public void testValidateConfig() throws IOException { YarnConfiguration.TIMELINE_SERVICE_LEVELDB_READ_CACHE_SIZE, -1); config = newConfig; restartTimelineStore(); - Assert.fail(); + fail(); } catch (IllegalArgumentException e) { - Assert.assertTrue(e.getMessage().contains( + assertTrue(e.getMessage().contains( YarnConfiguration.TIMELINE_SERVICE_LEVELDB_READ_CACHE_SIZE)); } try { @@ -223,25 +224,25 @@ public void testValidateConfig() throws IOException { 0); config = newConfig; restartTimelineStore(); - Assert.fail(); + fail(); } catch (IllegalArgumentException e) { - Assert.assertTrue(e.getMessage().contains( + assertTrue(e.getMessage().contains( YarnConfiguration - .TIMELINE_SERVICE_LEVELDB_START_TIME_READ_CACHE_SIZE)); + .TIMELINE_SERVICE_LEVELDB_START_TIME_READ_CACHE_SIZE)); } try { Configuration newConfig = new YarnConfiguration(copyConfig); newConfig.setLong( YarnConfiguration - .TIMELINE_SERVICE_LEVELDB_START_TIME_WRITE_CACHE_SIZE, + .TIMELINE_SERVICE_LEVELDB_START_TIME_WRITE_CACHE_SIZE, 0); config = newConfig; restartTimelineStore(); - Assert.fail(); + fail(); } catch (IllegalArgumentException e) { - Assert.assertTrue(e.getMessage().contains( + assertTrue(e.getMessage().contains( YarnConfiguration - .TIMELINE_SERVICE_LEVELDB_START_TIME_WRITE_CACHE_SIZE)); + .TIMELINE_SERVICE_LEVELDB_START_TIME_WRITE_CACHE_SIZE)); } config = copyConfig; restartTimelineStore(); @@ -268,7 +269,7 @@ public void testGetDomains() throws IOException { } @Test - public void testRelatingToNonExistingEntity() throws IOException { + void testRelatingToNonExistingEntity() throws IOException { TimelineEntity entityToStore = new TimelineEntity(); entityToStore.setEntityType("TEST_ENTITY_TYPE_1"); entityToStore.setEntityId("TEST_ENTITY_ID_1"); @@ -279,17 +280,17 @@ public void testRelatingToNonExistingEntity() throws IOException { store.put(entities); TimelineEntity entityToGet = store.getEntity("TEST_ENTITY_ID_2", "TEST_ENTITY_TYPE_2", null); - Assert.assertNotNull(entityToGet); - Assert.assertEquals("DEFAULT", entityToGet.getDomainId()); - Assert.assertEquals("TEST_ENTITY_TYPE_1", + assertNotNull(entityToGet); + assertEquals("DEFAULT", entityToGet.getDomainId()); + assertEquals("TEST_ENTITY_TYPE_1", entityToGet.getRelatedEntities().keySet().iterator().next()); - Assert.assertEquals("TEST_ENTITY_ID_1", + assertEquals("TEST_ENTITY_ID_1", entityToGet.getRelatedEntities().values().iterator().next() .iterator().next()); } @Test - public void testRelatingToEntityInSamePut() throws IOException { + void testRelatingToEntityInSamePut() throws IOException { TimelineEntity entityToRelate = new TimelineEntity(); entityToRelate.setEntityType("TEST_ENTITY_TYPE_2"); entityToRelate.setEntityId("TEST_ENTITY_ID_2"); @@ -305,17 +306,17 @@ public void testRelatingToEntityInSamePut() throws IOException { store.put(entities); TimelineEntity entityToGet = store.getEntity("TEST_ENTITY_ID_2", "TEST_ENTITY_TYPE_2", null); - Assert.assertNotNull(entityToGet); - Assert.assertEquals("TEST_DOMAIN", entityToGet.getDomainId()); - Assert.assertEquals("TEST_ENTITY_TYPE_1", + assertNotNull(entityToGet); + assertEquals("TEST_DOMAIN", entityToGet.getDomainId()); + assertEquals("TEST_ENTITY_TYPE_1", entityToGet.getRelatedEntities().keySet().iterator().next()); - Assert.assertEquals("TEST_ENTITY_ID_1", + assertEquals("TEST_ENTITY_ID_1", entityToGet.getRelatedEntities().values().iterator().next() .iterator().next()); } @Test - public void testRelatingToOldEntityWithoutDomainId() throws IOException { + void testRelatingToOldEntityWithoutDomainId() throws IOException { // New entity is put in the default domain TimelineEntity entityToStore = new TimelineEntity(); entityToStore.setEntityType("NEW_ENTITY_TYPE_1"); @@ -328,11 +329,11 @@ public void testRelatingToOldEntityWithoutDomainId() throws IOException { TimelineEntity entityToGet = store.getEntity("OLD_ENTITY_ID_1", "OLD_ENTITY_TYPE_1", null); - Assert.assertNotNull(entityToGet); - Assert.assertEquals("DEFAULT", entityToGet.getDomainId()); - Assert.assertEquals("NEW_ENTITY_TYPE_1", + assertNotNull(entityToGet); + assertEquals("DEFAULT", entityToGet.getDomainId()); + assertEquals("NEW_ENTITY_TYPE_1", entityToGet.getRelatedEntities().keySet().iterator().next()); - Assert.assertEquals("NEW_ENTITY_ID_1", + assertEquals("NEW_ENTITY_ID_1", entityToGet.getRelatedEntities().values().iterator().next() .iterator().next()); @@ -345,16 +346,16 @@ public void testRelatingToOldEntityWithoutDomainId() throws IOException { entities = new TimelineEntities(); entities.addEntity(entityToStore); TimelinePutResponse response = store.put(entities); - Assert.assertEquals(1, response.getErrors().size()); - Assert.assertEquals(TimelinePutError.FORBIDDEN_RELATION, + assertEquals(1, response.getErrors().size()); + assertEquals(TimelinePutError.FORBIDDEN_RELATION, response.getErrors().get(0).getErrorCode()); entityToGet = store.getEntity("OLD_ENTITY_ID_1", "OLD_ENTITY_TYPE_1", null); - Assert.assertNotNull(entityToGet); - Assert.assertEquals("DEFAULT", entityToGet.getDomainId()); + assertNotNull(entityToGet); + assertEquals("DEFAULT", entityToGet.getDomainId()); // Still have one related entity - Assert.assertEquals(1, entityToGet.getRelatedEntities().keySet().size()); - Assert.assertEquals(1, entityToGet.getRelatedEntities().values() + assertEquals(1, entityToGet.getRelatedEntities().keySet().size()); + assertEquals(1, entityToGet.getRelatedEntities().values() .iterator().next().size()); } @@ -423,12 +424,12 @@ public void testStorePerformance() throws IOException { Log.getLog().info("Duration for " + num + ": " + duration); } - @Test /** * Test that RollingLevelDb repair is attempted at least once during * serviceInit for RollingLeveldbTimelineStore in case open fails the * first time. - */ public void testLevelDbRepair() throws IOException { + */ @Test + void testLevelDbRepair() throws IOException { RollingLevelDBTimelineStore store = new RollingLevelDBTimelineStore(); JniDBFactory factory = Mockito.mock(JniDBFactory.class); Mockito.when(factory.open(Mockito.any(File.class), Mockito.any(Options.class))) @@ -445,7 +446,7 @@ public void testStorePerformance() throws IOException { .repair(Mockito.any(File.class), Mockito.any(Options.class)); FilenameFilter fileFilter = new WildcardFileFilter("*" + RollingLevelDBTimelineStore.BACKUP_EXT + "*"); - Assert.assertTrue(new File(path.getAbsolutePath(), RollingLevelDBTimelineStore.FILENAME) + assertTrue(new File(path.getAbsolutePath(), RollingLevelDBTimelineStore.FILENAME) .list(fileFilter).length > 0); } finally { store.close(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/TestTimelineDataManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/TestTimelineDataManager.java index 8fba54c9e8055..be57c84cd8c7d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/TestTimelineDataManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/TestTimelineDataManager.java @@ -20,6 +20,10 @@ import java.io.File; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileContext; import org.apache.hadoop.fs.Path; @@ -30,10 +34,10 @@ import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.security.AdminACLsManager; import org.apache.hadoop.yarn.server.timeline.security.TimelineACLsManager; -import org.junit.After; -import org.junit.Assert; -import org.junit.Before; -import org.junit.Test; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; public class TestTimelineDataManager extends TimelineStoreTestUtils { @@ -43,7 +47,7 @@ public class TestTimelineDataManager extends TimelineStoreTestUtils { private TimelineDataManager dataManaer; private static TimelineACLsManager aclsManager; private static AdminACLsManager adminACLsManager; - @Before + @BeforeEach public void setup() throws Exception { fsPath = new File("target", this.getClass().getSimpleName() + "-tmpDir").getAbsoluteFile(); @@ -70,7 +74,7 @@ public void setup() throws Exception { adminACLsManager = new AdminACLsManager(conf); } - @After + @AfterEach public void tearDown() throws Exception { if (store != null) { store.stop(); @@ -81,55 +85,55 @@ public void tearDown() throws Exception { } @Test - public void testGetOldEntityWithOutDomainId() throws Exception { + void testGetOldEntityWithOutDomainId() throws Exception { TimelineEntity entity = dataManaer.getEntity( "OLD_ENTITY_TYPE_1", "OLD_ENTITY_ID_1", null, UserGroupInformation.getCurrentUser()); - Assert.assertNotNull(entity); - Assert.assertEquals("OLD_ENTITY_ID_1", entity.getEntityId()); - Assert.assertEquals("OLD_ENTITY_TYPE_1", entity.getEntityType()); - Assert.assertEquals( + assertNotNull(entity); + assertEquals("OLD_ENTITY_ID_1", entity.getEntityId()); + assertEquals("OLD_ENTITY_TYPE_1", entity.getEntityType()); + assertEquals( TimelineDataManager.DEFAULT_DOMAIN_ID, entity.getDomainId()); } @Test - public void testGetEntitiesAclEnabled() throws Exception { + void testGetEntitiesAclEnabled() throws Exception { AdminACLsManager oldAdminACLsManager = - aclsManager.setAdminACLsManager(adminACLsManager); + aclsManager.setAdminACLsManager(adminACLsManager); try { TimelineEntities entities = dataManaer.getEntities( - "ACL_ENTITY_TYPE_1", null, null, null, null, null, null, 1l, null, - UserGroupInformation.createUserForTesting("owner_1", new String[] {"group1"})); - Assert.assertEquals(1, entities.getEntities().size()); - Assert.assertEquals("ACL_ENTITY_ID_11", - entities.getEntities().get(0).getEntityId()); + "ACL_ENTITY_TYPE_1", null, null, null, null, null, null, 1L, null, + UserGroupInformation.createUserForTesting("owner_1", new String[]{"group1"})); + assertEquals(1, entities.getEntities().size()); + assertEquals("ACL_ENTITY_ID_11", + entities.getEntities().get(0).getEntityId()); } finally { aclsManager.setAdminACLsManager(oldAdminACLsManager); } } @Test - public void testGetOldEntitiesWithOutDomainId() throws Exception { + void testGetOldEntitiesWithOutDomainId() throws Exception { TimelineEntities entities = dataManaer.getEntities( "OLD_ENTITY_TYPE_1", null, null, null, null, null, null, null, null, UserGroupInformation.getCurrentUser()); - Assert.assertEquals(2, entities.getEntities().size()); - Assert.assertEquals("OLD_ENTITY_ID_2", + assertEquals(2, entities.getEntities().size()); + assertEquals("OLD_ENTITY_ID_2", entities.getEntities().get(0).getEntityId()); - Assert.assertEquals("OLD_ENTITY_TYPE_1", + assertEquals("OLD_ENTITY_TYPE_1", entities.getEntities().get(0).getEntityType()); - Assert.assertEquals(TimelineDataManager.DEFAULT_DOMAIN_ID, + assertEquals(TimelineDataManager.DEFAULT_DOMAIN_ID, entities.getEntities().get(0).getDomainId()); - Assert.assertEquals("OLD_ENTITY_ID_1", + assertEquals("OLD_ENTITY_ID_1", entities.getEntities().get(1).getEntityId()); - Assert.assertEquals("OLD_ENTITY_TYPE_1", + assertEquals("OLD_ENTITY_TYPE_1", entities.getEntities().get(1).getEntityType()); - Assert.assertEquals(TimelineDataManager.DEFAULT_DOMAIN_ID, + assertEquals(TimelineDataManager.DEFAULT_DOMAIN_ID, entities.getEntities().get(1).getDomainId()); } @Test - public void testUpdatingOldEntityWithoutDomainId() throws Exception { + void testUpdatingOldEntityWithoutDomainId() throws Exception { // Set the domain to the default domain when updating TimelineEntity entity = new TimelineEntity(); entity.setEntityType("OLD_ENTITY_TYPE_1"); @@ -140,18 +144,18 @@ public void testUpdatingOldEntityWithoutDomainId() throws Exception { entities.addEntity(entity); TimelinePutResponse response = dataManaer.postEntities( entities, UserGroupInformation.getCurrentUser()); - Assert.assertEquals(0, response.getErrors().size()); + assertEquals(0, response.getErrors().size()); entity = store.getEntity("OLD_ENTITY_ID_1", "OLD_ENTITY_TYPE_1", null); - Assert.assertNotNull(entity); + assertNotNull(entity); // Even in leveldb, the domain is updated to the default domain Id - Assert.assertEquals( + assertEquals( TimelineDataManager.DEFAULT_DOMAIN_ID, entity.getDomainId()); - Assert.assertEquals(1, entity.getOtherInfo().size()); - Assert.assertEquals("NEW_OTHER_INFO_KEY", + assertEquals(1, entity.getOtherInfo().size()); + assertEquals("NEW_OTHER_INFO_KEY", entity.getOtherInfo().keySet().iterator().next()); - Assert.assertEquals("NEW_OTHER_INFO_VALUE", + assertEquals("NEW_OTHER_INFO_VALUE", entity.getOtherInfo().values().iterator().next()); - + // Set the domain to the non-default domain when updating entity = new TimelineEntity(); entity.setEntityType("OLD_ENTITY_TYPE_1"); @@ -162,15 +166,15 @@ public void testUpdatingOldEntityWithoutDomainId() throws Exception { entities.addEntity(entity); response = dataManaer.postEntities( entities, UserGroupInformation.getCurrentUser()); - Assert.assertEquals(1, response.getErrors().size()); - Assert.assertEquals(TimelinePutResponse.TimelinePutError.ACCESS_DENIED, + assertEquals(1, response.getErrors().size()); + assertEquals(TimelinePutResponse.TimelinePutError.ACCESS_DENIED, response.getErrors().get(0).getErrorCode()); entity = store.getEntity("OLD_ENTITY_ID_2", "OLD_ENTITY_TYPE_1", null); - Assert.assertNotNull(entity); + assertNotNull(entity); // In leveldb, the domain Id is still null - Assert.assertNull(entity.getDomainId()); + assertNull(entity.getDomainId()); // Updating is not executed - Assert.assertEquals(0, entity.getOtherInfo().size()); + assertEquals(0, entity.getOtherInfo().size()); } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/TimelineStoreTestUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/TimelineStoreTestUtils.java index 39b22b1afd5f0..f925b1104f503 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/TimelineStoreTestUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/TimelineStoreTestUtils.java @@ -17,10 +17,6 @@ */ package org.apache.hadoop.yarn.server.timeline; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNull; -import static org.junit.Assert.assertTrue; - import java.io.IOException; import java.util.ArrayList; import java.util.Collection; @@ -35,16 +31,20 @@ import java.util.SortedSet; import java.util.TreeSet; +import org.apache.hadoop.yarn.api.records.timeline.TimelineDomain; +import org.apache.hadoop.yarn.api.records.timeline.TimelineDomains; import org.apache.hadoop.yarn.api.records.timeline.TimelineEntities; import org.apache.hadoop.yarn.api.records.timeline.TimelineEntity; import org.apache.hadoop.yarn.api.records.timeline.TimelineEvent; import org.apache.hadoop.yarn.api.records.timeline.TimelineEvents.EventsOfOneEntity; -import org.apache.hadoop.yarn.api.records.timeline.TimelineDomain; -import org.apache.hadoop.yarn.api.records.timeline.TimelineDomains; import org.apache.hadoop.yarn.api.records.timeline.TimelinePutResponse; import org.apache.hadoop.yarn.api.records.timeline.TimelinePutResponse.TimelinePutError; import org.apache.hadoop.yarn.server.timeline.TimelineReader.Field; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + public class TimelineStoreTestUtils { protected static final List EMPTY_EVENTS = @@ -503,18 +503,24 @@ protected List getEntities(String entityType, Long limit, public void testGetEntities() throws IOException { // test getting entities - assertEquals("nonzero entities size for nonexistent type", 0, - getEntities("type_0").size()); - assertEquals("nonzero entities size for nonexistent type", 0, - getEntities("type_3").size()); - assertEquals("nonzero entities size for nonexistent type", 0, - getEntities("type_6").size()); - assertEquals("nonzero entities size for nonexistent type", 0, - getEntitiesWithPrimaryFilter("type_0", userFilter).size()); - assertEquals("nonzero entities size for nonexistent type", 0, - getEntitiesWithPrimaryFilter("type_3", userFilter).size()); - assertEquals("nonzero entities size for nonexistent type", 0, - getEntitiesWithPrimaryFilter("type_6", userFilter).size()); + assertEquals(0, + getEntities("type_0").size(), + "nonzero entities size for nonexistent type"); + assertEquals(0, + getEntities("type_3").size(), + "nonzero entities size for nonexistent type"); + assertEquals(0, + getEntities("type_6").size(), + "nonzero entities size for nonexistent type"); + assertEquals(0, + getEntitiesWithPrimaryFilter("type_0", userFilter).size(), + "nonzero entities size for nonexistent type"); + assertEquals(0, + getEntitiesWithPrimaryFilter("type_3", userFilter).size(), + "nonzero entities size for nonexistent type"); + assertEquals(0, + getEntitiesWithPrimaryFilter("type_6", userFilter).size(), + "nonzero entities size for nonexistent type"); List entities = getEntities("type_1"); assertEquals(3, entities.size()); @@ -681,15 +687,18 @@ public void testGetEntitiesWithFromTs() throws IOException { public void testGetEntitiesWithPrimaryFilters() throws IOException { // test using primary filter - assertEquals("nonzero entities size for primary filter", 0, + assertEquals(0, getEntitiesWithPrimaryFilter("type_1", - new NameValuePair("none", "none")).size()); - assertEquals("nonzero entities size for primary filter", 0, + new NameValuePair("none", "none")).size(), + "nonzero entities size for primary filter"); + assertEquals(0, getEntitiesWithPrimaryFilter("type_2", - new NameValuePair("none", "none")).size()); - assertEquals("nonzero entities size for primary filter", 0, + new NameValuePair("none", "none")).size(), + "nonzero entities size for primary filter"); + assertEquals(0, getEntitiesWithPrimaryFilter("type_3", - new NameValuePair("none", "none")).size()); + new NameValuePair("none", "none")).size(), + "nonzero entities size for primary filter"); List entities = getEntitiesWithPrimaryFilter("type_1", userFilter); @@ -927,13 +936,12 @@ protected static void verifyEntityInfo(String entityId, String entityType, if (primaryFilters == null) { assertNull(retrievedEntityInfo.getPrimaryFilters()); } else { - assertTrue(primaryFilters.equals( - retrievedEntityInfo.getPrimaryFilters())); + assertEquals(primaryFilters, retrievedEntityInfo.getPrimaryFilters()); } if (otherInfo == null) { assertNull(retrievedEntityInfo.getOtherInfo()); } else { - assertTrue(otherInfo.equals(retrievedEntityInfo.getOtherInfo())); + assertEquals(otherInfo, retrievedEntityInfo.getOtherInfo()); } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/recovery/TestLeveldbTimelineStateStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/recovery/TestLeveldbTimelineStateStore.java index a35477dc70b99..3baf41425cb97 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/recovery/TestLeveldbTimelineStateStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/recovery/TestLeveldbTimelineStateStore.java @@ -18,14 +18,13 @@ package org.apache.hadoop.yarn.server.timeline.recovery; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; -import static org.junit.Assert.fail; - import java.io.File; import java.io.IOException; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileContext; import org.apache.hadoop.fs.Path; @@ -36,10 +35,11 @@ import org.apache.hadoop.yarn.security.client.TimelineDelegationTokenIdentifier; import org.apache.hadoop.yarn.server.records.Version; import org.apache.hadoop.yarn.server.timeline.recovery.TimelineStateStore.TimelineServiceState; -import org.junit.After; -import org.junit.Assert; -import org.junit.Before; -import org.junit.Test; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.fail; public class TestLeveldbTimelineStateStore { @@ -48,7 +48,7 @@ public class TestLeveldbTimelineStateStore { private Configuration conf; private TimelineStateStore store; - @Before + @BeforeEach public void setup() throws Exception { fsPath = new File("target", getClass().getSimpleName() + "-tmpDir").getAbsoluteFile(); @@ -63,7 +63,7 @@ public void setup() throws Exception { fsPath.getAbsolutePath()); } - @After + @AfterEach public void tearDown() throws Exception { if (store != null) { store.stop(); @@ -81,11 +81,11 @@ private LeveldbTimelineStateStore initAndStartTimelineServiceStateStoreService() } @Test - public void testTokenStore() throws Exception { + void testTokenStore() throws Exception { initAndStartTimelineServiceStateStoreService(); TimelineServiceState state = store.loadState(); - assertTrue("token state not empty", state.tokenState.isEmpty()); - assertTrue("key state not empty", state.tokenMasterKeyState.isEmpty()); + assertTrue(state.tokenState.isEmpty(), "token state not empty"); + assertTrue(state.tokenMasterKeyState.isEmpty(), "key state not empty"); final DelegationKey key1 = new DelegationKey(1, 2, "keyData1".getBytes()); final TimelineDelegationTokenIdentifier token1 = @@ -120,19 +120,23 @@ public void testTokenStore() throws Exception { initAndStartTimelineServiceStateStoreService(); state = store.loadState(); - assertEquals("incorrect loaded token count", 2, state.tokenState.size()); - assertTrue("missing token 1", state.tokenState.containsKey(token1)); - assertEquals("incorrect token 1 date", tokenDate1, - state.tokenState.get(token1)); - assertTrue("missing token 2", state.tokenState.containsKey(token2)); - assertEquals("incorrect token 2 date", tokenDate2, - state.tokenState.get(token2)); - assertEquals("incorrect master key count", 1, - state.tokenMasterKeyState.size()); - assertTrue("missing master key 1", - state.tokenMasterKeyState.contains(key1)); - assertEquals("incorrect latest sequence number", 12345678, - state.getLatestSequenceNumber()); + assertEquals(2, state.tokenState.size(), "incorrect loaded token count"); + assertTrue(state.tokenState.containsKey(token1), "missing token 1"); + assertEquals(tokenDate1, + state.tokenState.get(token1), + "incorrect token 1 date"); + assertTrue(state.tokenState.containsKey(token2), "missing token 2"); + assertEquals(tokenDate2, + state.tokenState.get(token2), + "incorrect token 2 date"); + assertEquals(1, + state.tokenMasterKeyState.size(), + "incorrect master key count"); + assertTrue(state.tokenMasterKeyState.contains(key1), + "missing master key 1"); + assertEquals(12345678, + state.getLatestSequenceNumber(), + "incorrect latest sequence number"); final DelegationKey key2 = new DelegationKey(3, 4, "keyData2".getBytes()); final DelegationKey key3 = new DelegationKey(5, 6, "keyData3".getBytes()); @@ -154,46 +158,50 @@ public void testTokenStore() throws Exception { initAndStartTimelineServiceStateStoreService(); state = store.loadState(); - assertEquals("incorrect loaded token count", 2, state.tokenState.size()); - assertFalse("token 1 not removed", state.tokenState.containsKey(token1)); - assertTrue("missing token 2", state.tokenState.containsKey(token2)); - assertEquals("incorrect token 2 date", newTokenDate2, - state.tokenState.get(token2)); - assertTrue("missing token 3", state.tokenState.containsKey(token3)); - assertEquals("incorrect token 3 date", tokenDate3, - state.tokenState.get(token3)); - assertEquals("incorrect master key count", 2, - state.tokenMasterKeyState.size()); - assertFalse("master key 1 not removed", - state.tokenMasterKeyState.contains(key1)); - assertTrue("missing master key 2", - state.tokenMasterKeyState.contains(key2)); - assertTrue("missing master key 3", - state.tokenMasterKeyState.contains(key3)); - assertEquals("incorrect latest sequence number", 12345679, - state.getLatestSequenceNumber()); + assertEquals(2, state.tokenState.size(), "incorrect loaded token count"); + assertFalse(state.tokenState.containsKey(token1), "token 1 not removed"); + assertTrue(state.tokenState.containsKey(token2), "missing token 2"); + assertEquals(newTokenDate2, + state.tokenState.get(token2), + "incorrect token 2 date"); + assertTrue(state.tokenState.containsKey(token3), "missing token 3"); + assertEquals(tokenDate3, + state.tokenState.get(token3), + "incorrect token 3 date"); + assertEquals(2, + state.tokenMasterKeyState.size(), + "incorrect master key count"); + assertFalse(state.tokenMasterKeyState.contains(key1), + "master key 1 not removed"); + assertTrue(state.tokenMasterKeyState.contains(key2), + "missing master key 2"); + assertTrue(state.tokenMasterKeyState.contains(key3), + "missing master key 3"); + assertEquals(12345679, + state.getLatestSequenceNumber(), + "incorrect latest sequence number"); store.close(); } @Test - public void testCheckVersion() throws IOException { + void testCheckVersion() throws IOException { LeveldbTimelineStateStore store = initAndStartTimelineServiceStateStoreService(); // default version Version defaultVersion = store.getCurrentVersion(); - Assert.assertEquals(defaultVersion, store.loadVersion()); + assertEquals(defaultVersion, store.loadVersion()); // compatible version Version compatibleVersion = Version.newInstance(defaultVersion.getMajorVersion(), defaultVersion.getMinorVersion() + 2); store.storeVersion(compatibleVersion); - Assert.assertEquals(compatibleVersion, store.loadVersion()); + assertEquals(compatibleVersion, store.loadVersion()); store.stop(); // overwrite the compatible version store = initAndStartTimelineServiceStateStoreService(); - Assert.assertEquals(defaultVersion, store.loadVersion()); + assertEquals(defaultVersion, store.loadVersion()); // incompatible version Version incompatibleVersion = @@ -204,10 +212,10 @@ public void testCheckVersion() throws IOException { try { initAndStartTimelineServiceStateStoreService(); - Assert.fail("Incompatible version, should expect fail here."); + fail("Incompatible version, should expect fail here."); } catch (ServiceStateException e) { - Assert.assertTrue("Exception message mismatch", - e.getMessage().contains("Incompatible version for timeline state store")); + assertTrue(e.getMessage().contains("Incompatible version for timeline state store"), + "Exception message mismatch"); } } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/security/TestTimelineACLsManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/security/TestTimelineACLsManager.java index 3bfb4b09576e4..da3a9e226bfd3 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/security/TestTimelineACLsManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/security/TestTimelineACLsManager.java @@ -20,6 +20,8 @@ import java.io.IOException; +import org.junit.jupiter.api.Test; + import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.yarn.api.records.ApplicationAccessType; @@ -29,8 +31,10 @@ import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.server.timeline.MemoryTimelineStore; import org.apache.hadoop.yarn.server.timeline.TimelineStore; -import org.junit.Assert; -import org.junit.Test; + +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.fail; public class TestTimelineACLsManager { @@ -45,7 +49,7 @@ public class TestTimelineACLsManager { } @Test - public void testYarnACLsNotEnabledForEntity() throws Exception { + void testYarnACLsNotEnabledForEntity() throws Exception { Configuration conf = new YarnConfiguration(); conf.setBoolean(YarnConfiguration.YARN_ACL_ENABLE, false); TimelineACLsManager timelineACLsManager = @@ -56,20 +60,20 @@ public void testYarnACLsNotEnabledForEntity() throws Exception { TimelineStore.SystemFilter.ENTITY_OWNER .toString(), "owner"); entity.setDomainId("domain_id_1"); - Assert.assertTrue( - "Always true when ACLs are not enabled", + assertTrue( timelineACLsManager.checkAccess( UserGroupInformation.createRemoteUser("user"), - ApplicationAccessType.VIEW_APP, entity)); - Assert.assertTrue( - "Always true when ACLs are not enabled", + ApplicationAccessType.VIEW_APP, entity), + "Always true when ACLs are not enabled"); + assertTrue( timelineACLsManager.checkAccess( UserGroupInformation.createRemoteUser("user"), - ApplicationAccessType.MODIFY_APP, entity)); + ApplicationAccessType.MODIFY_APP, entity), + "Always true when ACLs are not enabled"); } @Test - public void testYarnACLsEnabledForEntity() throws Exception { + void testYarnACLsEnabledForEntity() throws Exception { Configuration conf = new YarnConfiguration(); conf.setBoolean(YarnConfiguration.YARN_ACL_ENABLE, true); conf.set(YarnConfiguration.YARN_ADMIN_ACL, "admin"); @@ -81,51 +85,51 @@ public void testYarnACLsEnabledForEntity() throws Exception { TimelineStore.SystemFilter.ENTITY_OWNER .toString(), "owner"); entity.setDomainId("domain_id_1"); - Assert.assertTrue( - "Owner should be allowed to view", + assertTrue( timelineACLsManager.checkAccess( UserGroupInformation.createRemoteUser("owner"), - ApplicationAccessType.VIEW_APP, entity)); - Assert.assertTrue( - "Reader should be allowed to view", + ApplicationAccessType.VIEW_APP, entity), + "Owner should be allowed to view"); + assertTrue( timelineACLsManager.checkAccess( UserGroupInformation.createRemoteUser("reader"), - ApplicationAccessType.VIEW_APP, entity)); - Assert.assertFalse( - "Other shouldn't be allowed to view", + ApplicationAccessType.VIEW_APP, entity), + "Reader should be allowed to view"); + assertFalse( timelineACLsManager.checkAccess( UserGroupInformation.createRemoteUser("other"), - ApplicationAccessType.VIEW_APP, entity)); - Assert.assertTrue( - "Admin should be allowed to view", + ApplicationAccessType.VIEW_APP, entity), + "Other shouldn't be allowed to view"); + assertTrue( timelineACLsManager.checkAccess( UserGroupInformation.createRemoteUser("admin"), - ApplicationAccessType.VIEW_APP, entity)); + ApplicationAccessType.VIEW_APP, entity), + "Admin should be allowed to view"); - Assert.assertTrue( - "Owner should be allowed to modify", + assertTrue( timelineACLsManager.checkAccess( UserGroupInformation.createRemoteUser("owner"), - ApplicationAccessType.MODIFY_APP, entity)); - Assert.assertTrue( - "Writer should be allowed to modify", + ApplicationAccessType.MODIFY_APP, entity), + "Owner should be allowed to modify"); + assertTrue( timelineACLsManager.checkAccess( UserGroupInformation.createRemoteUser("writer"), - ApplicationAccessType.MODIFY_APP, entity)); - Assert.assertFalse( - "Other shouldn't be allowed to modify", + ApplicationAccessType.MODIFY_APP, entity), + "Writer should be allowed to modify"); + assertFalse( timelineACLsManager.checkAccess( UserGroupInformation.createRemoteUser("other"), - ApplicationAccessType.MODIFY_APP, entity)); - Assert.assertTrue( - "Admin should be allowed to modify", + ApplicationAccessType.MODIFY_APP, entity), + "Other shouldn't be allowed to modify"); + assertTrue( timelineACLsManager.checkAccess( UserGroupInformation.createRemoteUser("admin"), - ApplicationAccessType.MODIFY_APP, entity)); + ApplicationAccessType.MODIFY_APP, entity), + "Admin should be allowed to modify"); } @Test - public void testCorruptedOwnerInfoForEntity() throws Exception { + void testCorruptedOwnerInfoForEntity() throws Exception { Configuration conf = new YarnConfiguration(); conf.setBoolean(YarnConfiguration.YARN_ACL_ENABLE, true); conf.set(YarnConfiguration.YARN_ADMIN_ACL, "owner"); @@ -137,29 +141,29 @@ public void testCorruptedOwnerInfoForEntity() throws Exception { timelineACLsManager.checkAccess( UserGroupInformation.createRemoteUser("owner"), ApplicationAccessType.VIEW_APP, entity); - Assert.fail("Exception is expected"); + fail("Exception is expected"); } catch (YarnException e) { - Assert.assertTrue("It's not the exact expected exception", e.getMessage() - .contains("doesn't exist.")); + assertTrue(e.getMessage() + .contains("doesn't exist."), "It's not the exact expected exception"); } } @Test - public void testYarnACLsNotEnabledForDomain() throws Exception { + void testYarnACLsNotEnabledForDomain() throws Exception { Configuration conf = new YarnConfiguration(); conf.setBoolean(YarnConfiguration.YARN_ACL_ENABLE, false); TimelineACLsManager timelineACLsManager = new TimelineACLsManager(conf); TimelineDomain domain = new TimelineDomain(); domain.setOwner("owner"); - Assert.assertTrue( - "Always true when ACLs are not enabled", + assertTrue( timelineACLsManager.checkAccess( - UserGroupInformation.createRemoteUser("user"), domain)); + UserGroupInformation.createRemoteUser("user"), domain), + "Always true when ACLs are not enabled"); } @Test - public void testYarnACLsEnabledForDomain() throws Exception { + void testYarnACLsEnabledForDomain() throws Exception { Configuration conf = new YarnConfiguration(); conf.setBoolean(YarnConfiguration.YARN_ACL_ENABLE, true); conf.set(YarnConfiguration.YARN_ADMIN_ACL, "admin"); @@ -167,22 +171,22 @@ public void testYarnACLsEnabledForDomain() throws Exception { new TimelineACLsManager(conf); TimelineDomain domain = new TimelineDomain(); domain.setOwner("owner"); - Assert.assertTrue( - "Owner should be allowed to access", + assertTrue( timelineACLsManager.checkAccess( - UserGroupInformation.createRemoteUser("owner"), domain)); - Assert.assertFalse( - "Other shouldn't be allowed to access", + UserGroupInformation.createRemoteUser("owner"), domain), + "Owner should be allowed to access"); + assertFalse( timelineACLsManager.checkAccess( - UserGroupInformation.createRemoteUser("other"), domain)); - Assert.assertTrue( - "Admin should be allowed to access", + UserGroupInformation.createRemoteUser("other"), domain), + "Other shouldn't be allowed to access"); + assertTrue( timelineACLsManager.checkAccess( - UserGroupInformation.createRemoteUser("admin"), domain)); + UserGroupInformation.createRemoteUser("admin"), domain), + "Admin should be allowed to access"); } @Test - public void testCorruptedOwnerInfoForDomain() throws Exception { + void testCorruptedOwnerInfoForDomain() throws Exception { Configuration conf = new YarnConfiguration(); conf.setBoolean(YarnConfiguration.YARN_ACL_ENABLE, true); conf.set(YarnConfiguration.YARN_ADMIN_ACL, "owner"); @@ -192,10 +196,10 @@ public void testCorruptedOwnerInfoForDomain() throws Exception { try { timelineACLsManager.checkAccess( UserGroupInformation.createRemoteUser("owner"), domain); - Assert.fail("Exception is expected"); + fail("Exception is expected"); } catch (YarnException e) { - Assert.assertTrue("It's not the exact expected exception", e.getMessage() - .contains("is corrupted.")); + assertTrue(e.getMessage() + .contains("is corrupted."), "It's not the exact expected exception"); } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/security/TestTimelineAuthenticationFilterForV1.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/security/TestTimelineAuthenticationFilterForV1.java index cf2db2f8c6fb4..01e787af605f8 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/security/TestTimelineAuthenticationFilterForV1.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/security/TestTimelineAuthenticationFilterForV1.java @@ -24,6 +24,13 @@ import java.util.Collection; import java.util.concurrent.Callable; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.MethodSource; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.CommonConfigurationKeysPublic; import org.apache.hadoop.fs.FileUtil; @@ -46,23 +53,18 @@ import org.apache.hadoop.yarn.server.applicationhistoryservice.ApplicationHistoryServer; import org.apache.hadoop.yarn.server.timeline.MemoryTimelineStore; import org.apache.hadoop.yarn.server.timeline.TimelineStore; -import static org.apache.hadoop.yarn.conf.YarnConfiguration.TIMELINE_HTTP_AUTH_PREFIX; -import static org.junit.Assert.fail; -import org.junit.AfterClass; -import org.junit.Assert; -import org.junit.BeforeClass; -import org.junit.Test; -import org.junit.runner.RunWith; -import org.junit.runners.Parameterized; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; +import static org.apache.hadoop.yarn.conf.YarnConfiguration.TIMELINE_HTTP_AUTH_PREFIX; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.fail; /** * Test cases for authentication via TimelineAuthenticationFilter while * publishing entities for ATSv1. */ -@RunWith(Parameterized.class) public class TestTimelineAuthenticationFilterForV1 { private static final Logger LOG = LoggerFactory.getLogger(TestTimelineAuthenticationFilterForV1.class); @@ -74,7 +76,7 @@ public class TestTimelineAuthenticationFilterForV1 { private static final File TEST_ROOT_DIR = new File( System.getProperty("test.build.dir", "target/test-dir"), - TestTimelineAuthenticationFilterForV1.class.getName() + "-root"); + TestTimelineAuthenticationFilterForV1.class.getName() + "-root"); private static final File httpSpnegoKeytabFile = new File( KerberosTestUtils.getKeytabFile()); private static final String httpSpnegoPrincipal = @@ -83,9 +85,8 @@ public class TestTimelineAuthenticationFilterForV1 { System.getProperty("test.build.dir", "target/test-dir") + "/" + TestTimelineAuthenticationFilterForV1.class.getSimpleName(); - @Parameterized.Parameters public static Collection withSsl() { - return Arrays.asList(new Object[][] {{false}, {true}}); + return Arrays.asList(new Object[][]{{false}, {true}}); } private static MiniKdc testMiniKDC; @@ -95,11 +96,11 @@ public static Collection withSsl() { private static Configuration conf; private static boolean withSsl; - public TestTimelineAuthenticationFilterForV1(boolean withSsl) { - TestTimelineAuthenticationFilterForV1.withSsl = withSsl; + public void initTestTimelineAuthenticationFilterForV1(boolean isSslEnabled) { + TestTimelineAuthenticationFilterForV1.withSsl = isSslEnabled; } - @BeforeClass + @BeforeAll public static void setup() { try { testMiniKDC = new MiniKdc(MiniKdc.createConf(), TEST_ROOT_DIR); @@ -167,7 +168,7 @@ private TimelineClient createTimelineClientForUGI() { return client; } - @AfterClass + @AfterAll public static void tearDown() throws Exception { if (testMiniKDC != null) { testMiniKDC.stop(); @@ -184,8 +185,10 @@ public static void tearDown() throws Exception { } } - @Test - public void testPutTimelineEntities() throws Exception { + @MethodSource("withSsl") + @ParameterizedTest + void testPutTimelineEntities(boolean isSslEnabled) throws Exception { + initTestTimelineAuthenticationFilterForV1(isSslEnabled); KerberosTestUtils.doAs(PRINCIPAL, new Callable() { @Override public Void call() throws Exception { @@ -199,20 +202,22 @@ public Void call() throws Exception { if (putResponse.getErrors().size() > 0) { LOG.error("putResponse errors: {}", putResponse.getErrors()); } - Assert.assertTrue("There were some errors in the putResponse", - putResponse.getErrors().isEmpty()); + assertTrue(putResponse.getErrors().isEmpty(), + "There were some errors in the putResponse"); TimelineEntity entityToRead = testTimelineServer.getTimelineStore().getEntity("entity1", TestTimelineAuthenticationFilterForV1.class.getName(), null); - Assert.assertNotNull("Timeline entity should not be null", - entityToRead); + assertNotNull(entityToRead, + "Timeline entity should not be null"); return null; } }); } - @Test - public void testPutDomains() throws Exception { + @MethodSource("withSsl") + @ParameterizedTest + void testPutDomains(boolean isSslEnabled) throws Exception { + initTestTimelineAuthenticationFilterForV1(isSslEnabled); KerberosTestUtils.doAs(PRINCIPAL, new Callable() { @Override public Void call() throws Exception { @@ -226,71 +231,73 @@ public Void call() throws Exception { TimelineDomain domainToRead = testTimelineServer.getTimelineStore().getDomain( TestTimelineAuthenticationFilterForV1.class.getName()); - Assert.assertNotNull("Timeline domain should not be null", - domainToRead); + assertNotNull(domainToRead, + "Timeline domain should not be null"); return null; } }); } - @Test - public void testDelegationTokenOperations() throws Exception { + @MethodSource("withSsl") + @ParameterizedTest + void testDelegationTokenOperations(boolean isSslEnabled) throws Exception { + initTestTimelineAuthenticationFilterForV1(isSslEnabled); TimelineClient httpUserClient = KerberosTestUtils.doAs(PRINCIPAL, new Callable() { - @Override - public TimelineClient call() throws Exception { - return createTimelineClientForUGI(); - } - }); + @Override + public TimelineClient call() throws Exception { + return createTimelineClientForUGI(); + } + }); UserGroupInformation httpUser = KerberosTestUtils.doAs(PRINCIPAL, new Callable() { - @Override - public UserGroupInformation call() throws Exception { - return UserGroupInformation.getCurrentUser(); - } - }); + @Override + public UserGroupInformation call() throws Exception { + return UserGroupInformation.getCurrentUser(); + } + }); // Let HTTP user to get the delegation for itself Token token = httpUserClient.getDelegationToken(httpUser.getShortUserName()); - Assert.assertNotNull("Delegation token should not be null", token); + assertNotNull(token, "Delegation token should not be null"); TimelineDelegationTokenIdentifier tDT = token.decodeIdentifier(); - Assert.assertNotNull("Delegation token identifier should not be null", - tDT); - Assert.assertEquals("Owner of delegation token identifier does not match", - new Text(HTTP_USER), tDT.getOwner()); + assertNotNull(tDT, + "Delegation token identifier should not be null"); + assertEquals(new Text(HTTP_USER), tDT.getOwner(), + "Owner of delegation token identifier does not match"); // Renew token - Assert.assertFalse("Service field of token should not be empty", - token.getService().toString().isEmpty()); + assertFalse(token.getService().toString().isEmpty(), + "Service field of token should not be empty"); // Renew the token from the token service address long renewTime1 = httpUserClient.renewDelegationToken(token); Thread.sleep(100); token.setService(new Text()); - Assert.assertTrue("Service field of token should be empty", - token.getService().toString().isEmpty()); + assertTrue(token.getService().toString().isEmpty(), + "Service field of token should be empty"); // If the token service address is not available, it still can be renewed // from the configured address long renewTime2 = httpUserClient.renewDelegationToken(token); - Assert.assertTrue("renewTime2 should be later than renewTime1", - renewTime1 < renewTime2); + assertTrue(renewTime1 < renewTime2, + "renewTime2 should be later than renewTime1"); // Cancel token - Assert.assertTrue("Service field of token should be empty", - token.getService().toString().isEmpty()); + assertTrue(token.getService().toString().isEmpty(), + "Service field of token should be empty"); // If the token service address is not available, it still can be canceled // from the configured address httpUserClient.cancelDelegationToken(token); // Renew should not be successful because the token is canceled try { httpUserClient.renewDelegationToken(token); - Assert.fail("Renew of delegation token should not be successful"); + fail("Renew of delegation token should not be successful"); } catch (Exception e) { LOG.info("Exception while renewing delegation token", e); - Assert.assertTrue(e.getMessage().contains( - "Renewal request for unknown token")); + assertTrue(e.getMessage().contains( + "Renewal request for unknown token")); } // Let HTTP user to get the delegation token for FOO user @@ -304,35 +311,35 @@ public TimelineClient run() { } }); token = fooUserClient.getDelegationToken(httpUser.getShortUserName()); - Assert.assertNotNull("Delegation token should not be null", token); + assertNotNull(token, "Delegation token should not be null"); tDT = token.decodeIdentifier(); - Assert.assertNotNull("Delegation token identifier should not be null", - tDT); - Assert.assertEquals("Owner of delegation token is not the expected", - new Text(FOO_USER), tDT.getOwner()); - Assert.assertEquals("Real user of delegation token is not the expected", - new Text(HTTP_USER), tDT.getRealUser()); + assertNotNull(tDT, + "Delegation token identifier should not be null"); + assertEquals(new Text(FOO_USER), tDT.getOwner(), + "Owner of delegation token is not the expected"); + assertEquals(new Text(HTTP_USER), tDT.getRealUser(), + "Real user of delegation token is not the expected"); // Renew token as the renewer final Token tokenToRenew = token; renewTime1 = httpUserClient.renewDelegationToken(tokenToRenew); renewTime2 = httpUserClient.renewDelegationToken(tokenToRenew); - Assert.assertTrue("renewTime2 should be later than renewTime1", - renewTime1 < renewTime2); + assertTrue(renewTime1 < renewTime2, + "renewTime2 should be later than renewTime1"); // Cancel token - Assert.assertFalse("Service field of token should not be empty", - tokenToRenew.getService().toString().isEmpty()); + assertFalse(tokenToRenew.getService().toString().isEmpty(), + "Service field of token should not be empty"); // Cancel the token from the token service address fooUserClient.cancelDelegationToken(tokenToRenew); // Renew should not be successful because the token is canceled try { httpUserClient.renewDelegationToken(tokenToRenew); - Assert.fail("Renew of delegation token should not be successful"); + fail("Renew of delegation token should not be successful"); } catch (Exception e) { LOG.info("Exception while renewing delegation token", e); - Assert.assertTrue( + assertTrue( e.getMessage().contains("Renewal request for unknown token")); } @@ -349,10 +356,10 @@ public TimelineClient run() { try { barUserClient.getDelegationToken(httpUser.getShortUserName()); - Assert.fail("Retrieval of delegation token should not be successful"); + fail("Retrieval of delegation token should not be successful"); } catch (Exception e) { LOG.info("Exception while retrieving delegation token", e); - Assert.assertTrue(e.getCause() instanceof AuthorizationException || + assertTrue(e.getCause() instanceof AuthorizationException || e.getCause() instanceof AuthenticationException); } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/webapp/TestTimelineWebServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/webapp/TestTimelineWebServices.java index c8f9eb2d522cb..fd2d34f39cc8c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/webapp/TestTimelineWebServices.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/webapp/TestTimelineWebServices.java @@ -18,22 +18,12 @@ package org.apache.hadoop.yarn.server.timeline.webapp; -import static org.apache.hadoop.yarn.webapp.WebServicesTestUtils.assertResponseStatusCode; -import static org.assertj.core.api.Assertions.assertThat; -import static org.junit.Assert.assertEquals; -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.Mockito.doNothing; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.spy; -import static org.mockito.Mockito.when; - import java.util.Collections; import java.util.Enumeration; import java.util.HashMap; import java.util.HashSet; import java.util.Map; import java.util.Set; - import javax.servlet.FilterConfig; import javax.servlet.ServletContext; import javax.servlet.ServletException; @@ -42,17 +32,28 @@ import javax.xml.bind.JAXBContext; import javax.xml.bind.JAXBException; +import com.google.inject.Guice; +import com.google.inject.servlet.ServletModule; +import com.sun.jersey.api.client.ClientResponse; +import com.sun.jersey.api.client.WebResource; +import com.sun.jersey.api.client.config.DefaultClientConfig; +import com.sun.jersey.guice.spi.container.servlet.GuiceContainer; +import com.sun.jersey.test.framework.WebAppDescriptor; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.http.JettyUtils; import org.apache.hadoop.security.authentication.server.AuthenticationFilter; import org.apache.hadoop.security.authentication.server.PseudoAuthenticationHandler; import org.apache.hadoop.security.token.delegation.web.DelegationTokenAuthenticationHandler; +import org.apache.hadoop.yarn.api.records.timeline.TimelineAbout; +import org.apache.hadoop.yarn.api.records.timeline.TimelineDomain; +import org.apache.hadoop.yarn.api.records.timeline.TimelineDomains; import org.apache.hadoop.yarn.api.records.timeline.TimelineEntities; import org.apache.hadoop.yarn.api.records.timeline.TimelineEntity; import org.apache.hadoop.yarn.api.records.timeline.TimelineEvent; import org.apache.hadoop.yarn.api.records.timeline.TimelineEvents; -import org.apache.hadoop.yarn.api.records.timeline.TimelineDomain; -import org.apache.hadoop.yarn.api.records.timeline.TimelineDomains; import org.apache.hadoop.yarn.api.records.timeline.TimelinePutResponse; import org.apache.hadoop.yarn.api.records.timeline.TimelinePutResponse.TimelinePutError; import org.apache.hadoop.yarn.conf.YarnConfiguration; @@ -64,23 +65,23 @@ import org.apache.hadoop.yarn.server.timeline.TimelineStore; import org.apache.hadoop.yarn.server.timeline.security.TimelineACLsManager; import org.apache.hadoop.yarn.server.timeline.security.TimelineAuthenticationFilter; -import org.apache.hadoop.yarn.api.records.timeline.TimelineAbout; import org.apache.hadoop.yarn.util.timeline.TimelineUtils; import org.apache.hadoop.yarn.webapp.GenericExceptionHandler; import org.apache.hadoop.yarn.webapp.GuiceServletConfig; import org.apache.hadoop.yarn.webapp.JerseyTestBase; import org.apache.hadoop.yarn.webapp.YarnJacksonJaxbJsonProvider; -import org.junit.Assert; -import org.junit.Before; -import org.junit.Test; -import com.google.inject.Guice; -import com.google.inject.servlet.ServletModule; -import com.sun.jersey.api.client.ClientResponse; -import com.sun.jersey.api.client.WebResource; -import com.sun.jersey.api.client.config.DefaultClientConfig; -import com.sun.jersey.guice.spi.container.servlet.GuiceContainer; -import com.sun.jersey.test.framework.WebAppDescriptor; +import static org.apache.hadoop.yarn.webapp.WebServicesTestUtils.assertResponseStatusCode; +import static org.assertj.core.api.Assertions.assertThat; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.fail; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.doNothing; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.when; public class TestTimelineWebServices extends JerseyTestBase { @@ -99,7 +100,7 @@ protected void configureServlets() { try { store = mockTimelineStore(); } catch (Exception e) { - Assert.fail(); + fail(); } Configuration conf = new YarnConfiguration(); conf.setBoolean(YarnConfiguration.YARN_ACL_ENABLE, false); @@ -138,7 +139,7 @@ protected void configureServlets() { try { taFilter.init(filterConfig); } catch (ServletException e) { - Assert.fail("Unable to initialize TimelineAuthenticationFilter: " + + fail("Unable to initialize TimelineAuthenticationFilter: " + e.getMessage()); } @@ -146,7 +147,7 @@ protected void configureServlets() { try { doNothing().when(taFilter).init(any(FilterConfig.class)); } catch (ServletException e) { - Assert.fail("Unable to initialize TimelineAuthenticationFilter: " + + fail("Unable to initialize TimelineAuthenticationFilter: " + e.getMessage()); } filter("/*").through(taFilter); @@ -158,7 +159,7 @@ protected void configureServlets() { Guice.createInjector(new WebServletModule())); } - @Before + @BeforeEach public void setUp() throws Exception { super.setUp(); GuiceServletConfig.setInjector( @@ -187,7 +188,7 @@ public TestTimelineWebServices() { } @Test - public void testAbout() throws Exception { + void testAbout() throws Exception { WebResource r = resource(); ClientResponse response = r.path("ws").path("v1").path("timeline") .accept(MediaType.APPLICATION_JSON) @@ -197,54 +198,54 @@ public void testAbout() throws Exception { TimelineAbout actualAbout = response.getEntity(TimelineAbout.class); TimelineAbout expectedAbout = TimelineUtils.createTimelineAbout("Timeline API"); - Assert.assertNotNull( - "Timeline service about response is null", actualAbout); - Assert.assertEquals(expectedAbout.getAbout(), actualAbout.getAbout()); - Assert.assertEquals(expectedAbout.getTimelineServiceVersion(), + assertNotNull( + actualAbout, "Timeline service about response is null"); + assertEquals(expectedAbout.getAbout(), actualAbout.getAbout()); + assertEquals(expectedAbout.getTimelineServiceVersion(), actualAbout.getTimelineServiceVersion()); - Assert.assertEquals(expectedAbout.getTimelineServiceBuildVersion(), + assertEquals(expectedAbout.getTimelineServiceBuildVersion(), actualAbout.getTimelineServiceBuildVersion()); - Assert.assertEquals(expectedAbout.getTimelineServiceVersionBuiltOn(), + assertEquals(expectedAbout.getTimelineServiceVersionBuiltOn(), actualAbout.getTimelineServiceVersionBuiltOn()); - Assert.assertEquals(expectedAbout.getHadoopVersion(), + assertEquals(expectedAbout.getHadoopVersion(), actualAbout.getHadoopVersion()); - Assert.assertEquals(expectedAbout.getHadoopBuildVersion(), + assertEquals(expectedAbout.getHadoopBuildVersion(), actualAbout.getHadoopBuildVersion()); - Assert.assertEquals(expectedAbout.getHadoopVersionBuiltOn(), + assertEquals(expectedAbout.getHadoopVersionBuiltOn(), actualAbout.getHadoopVersionBuiltOn()); } private static void verifyEntities(TimelineEntities entities) { - Assert.assertNotNull(entities); - Assert.assertEquals(3, entities.getEntities().size()); + assertNotNull(entities); + assertEquals(3, entities.getEntities().size()); TimelineEntity entity1 = entities.getEntities().get(0); - Assert.assertNotNull(entity1); - Assert.assertEquals("id_1", entity1.getEntityId()); - Assert.assertEquals("type_1", entity1.getEntityType()); - Assert.assertEquals(123l, entity1.getStartTime().longValue()); - Assert.assertEquals(2, entity1.getEvents().size()); - Assert.assertEquals(4, entity1.getPrimaryFilters().size()); - Assert.assertEquals(4, entity1.getOtherInfo().size()); + assertNotNull(entity1); + assertEquals("id_1", entity1.getEntityId()); + assertEquals("type_1", entity1.getEntityType()); + assertEquals(123L, entity1.getStartTime().longValue()); + assertEquals(2, entity1.getEvents().size()); + assertEquals(4, entity1.getPrimaryFilters().size()); + assertEquals(4, entity1.getOtherInfo().size()); TimelineEntity entity2 = entities.getEntities().get(1); - Assert.assertNotNull(entity2); - Assert.assertEquals("id_2", entity2.getEntityId()); - Assert.assertEquals("type_1", entity2.getEntityType()); - Assert.assertEquals(123l, entity2.getStartTime().longValue()); - Assert.assertEquals(2, entity2.getEvents().size()); - Assert.assertEquals(4, entity2.getPrimaryFilters().size()); - Assert.assertEquals(4, entity2.getOtherInfo().size()); + assertNotNull(entity2); + assertEquals("id_2", entity2.getEntityId()); + assertEquals("type_1", entity2.getEntityType()); + assertEquals(123L, entity2.getStartTime().longValue()); + assertEquals(2, entity2.getEvents().size()); + assertEquals(4, entity2.getPrimaryFilters().size()); + assertEquals(4, entity2.getOtherInfo().size()); TimelineEntity entity3 = entities.getEntities().get(2); - Assert.assertNotNull(entity2); - Assert.assertEquals("id_6", entity3.getEntityId()); - Assert.assertEquals("type_1", entity3.getEntityType()); - Assert.assertEquals(61l, entity3.getStartTime().longValue()); - Assert.assertEquals(0, entity3.getEvents().size()); - Assert.assertEquals(4, entity3.getPrimaryFilters().size()); - Assert.assertEquals(4, entity3.getOtherInfo().size()); + assertNotNull(entity2); + assertEquals("id_6", entity3.getEntityId()); + assertEquals("type_1", entity3.getEntityType()); + assertEquals(61L, entity3.getStartTime().longValue()); + assertEquals(0, entity3.getEvents().size()); + assertEquals(4, entity3.getPrimaryFilters().size()); + assertEquals(4, entity3.getOtherInfo().size()); } @Test - public void testGetEntities() throws Exception { + void testGetEntities() throws Exception { WebResource r = resource(); ClientResponse response = r.path("ws").path("v1").path("timeline") .path("type_1") @@ -256,7 +257,7 @@ public void testGetEntities() throws Exception { } @Test - public void testFromId() throws Exception { + void testFromId() throws Exception { WebResource r = resource(); ClientResponse response = r.path("ws").path("v1").path("timeline") .path("type_1").queryParam("fromId", "id_2") @@ -278,7 +279,7 @@ public void testFromId() throws Exception { } @Test - public void testFromTs() throws Exception { + void testFromTs() throws Exception { WebResource r = resource(); ClientResponse response = r.path("ws").path("v1").path("timeline") .path("type_1").queryParam("fromTs", Long.toString(beforeTime)) @@ -291,7 +292,7 @@ public void testFromTs() throws Exception { response = r.path("ws").path("v1").path("timeline") .path("type_1").queryParam("fromTs", Long.toString( - System.currentTimeMillis())) + System.currentTimeMillis())) .accept(MediaType.APPLICATION_JSON) .get(ClientResponse.class); assertEquals(MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8, @@ -301,7 +302,7 @@ public void testFromTs() throws Exception { } @Test - public void testPrimaryFilterString() { + void testPrimaryFilterString() { WebResource r = resource(); ClientResponse response = r.path("ws").path("v1").path("timeline") .path("type_1").queryParam("primaryFilter", "user:username") @@ -313,11 +314,11 @@ public void testPrimaryFilterString() { } @Test - public void testPrimaryFilterInteger() { + void testPrimaryFilterInteger() { WebResource r = resource(); ClientResponse response = r.path("ws").path("v1").path("timeline") .path("type_1").queryParam("primaryFilter", - "appname:" + Integer.toString(Integer.MAX_VALUE)) + "appname:" + Integer.toString(Integer.MAX_VALUE)) .accept(MediaType.APPLICATION_JSON) .get(ClientResponse.class); assertEquals(MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8, @@ -326,11 +327,11 @@ public void testPrimaryFilterInteger() { } @Test - public void testPrimaryFilterLong() { + void testPrimaryFilterLong() { WebResource r = resource(); ClientResponse response = r.path("ws").path("v1").path("timeline") .path("type_1").queryParam("primaryFilter", - "long:" + Long.toString((long) Integer.MAX_VALUE + 1l)) + "long:" + Long.toString((long) Integer.MAX_VALUE + 1L)) .accept(MediaType.APPLICATION_JSON) .get(ClientResponse.class); assertEquals(MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8, @@ -339,7 +340,7 @@ public void testPrimaryFilterLong() { } @Test - public void testSecondaryFilters() { + void testSecondaryFilters() { WebResource r = resource(); ClientResponse response = r.path("ws").path("v1").path("timeline") .path("type_1") @@ -353,7 +354,7 @@ public void testSecondaryFilters() { } @Test - public void testGetEntity() throws Exception { + void testGetEntity() throws Exception { WebResource r = resource(); ClientResponse response = r.path("ws").path("v1").path("timeline") .path("type_1").path("id_1") @@ -362,17 +363,17 @@ public void testGetEntity() throws Exception { assertEquals(MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8, response.getType().toString()); TimelineEntity entity = response.getEntity(TimelineEntity.class); - Assert.assertNotNull(entity); - Assert.assertEquals("id_1", entity.getEntityId()); - Assert.assertEquals("type_1", entity.getEntityType()); - Assert.assertEquals(123l, entity.getStartTime().longValue()); - Assert.assertEquals(2, entity.getEvents().size()); - Assert.assertEquals(4, entity.getPrimaryFilters().size()); - Assert.assertEquals(4, entity.getOtherInfo().size()); + assertNotNull(entity); + assertEquals("id_1", entity.getEntityId()); + assertEquals("type_1", entity.getEntityType()); + assertEquals(123L, entity.getStartTime().longValue()); + assertEquals(2, entity.getEvents().size()); + assertEquals(4, entity.getPrimaryFilters().size()); + assertEquals(4, entity.getOtherInfo().size()); } @Test - public void testGetEntityFields1() throws Exception { + void testGetEntityFields1() throws Exception { WebResource r = resource(); ClientResponse response = r.path("ws").path("v1").path("timeline") .path("type_1").path("id_1").queryParam("fields", "events,otherinfo") @@ -381,37 +382,37 @@ public void testGetEntityFields1() throws Exception { assertEquals(MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8, response.getType().toString()); TimelineEntity entity = response.getEntity(TimelineEntity.class); - Assert.assertNotNull(entity); - Assert.assertEquals("id_1", entity.getEntityId()); - Assert.assertEquals("type_1", entity.getEntityType()); - Assert.assertEquals(123l, entity.getStartTime().longValue()); - Assert.assertEquals(2, entity.getEvents().size()); - Assert.assertEquals(0, entity.getPrimaryFilters().size()); - Assert.assertEquals(4, entity.getOtherInfo().size()); + assertNotNull(entity); + assertEquals("id_1", entity.getEntityId()); + assertEquals("type_1", entity.getEntityType()); + assertEquals(123L, entity.getStartTime().longValue()); + assertEquals(2, entity.getEvents().size()); + assertEquals(0, entity.getPrimaryFilters().size()); + assertEquals(4, entity.getOtherInfo().size()); } @Test - public void testGetEntityFields2() throws Exception { + void testGetEntityFields2() throws Exception { WebResource r = resource(); ClientResponse response = r.path("ws").path("v1").path("timeline") .path("type_1").path("id_1").queryParam("fields", "lasteventonly," + - "primaryfilters,relatedentities") + "primaryfilters,relatedentities") .accept(MediaType.APPLICATION_JSON) .get(ClientResponse.class); assertEquals(MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8, response.getType().toString()); TimelineEntity entity = response.getEntity(TimelineEntity.class); - Assert.assertNotNull(entity); - Assert.assertEquals("id_1", entity.getEntityId()); - Assert.assertEquals("type_1", entity.getEntityType()); - Assert.assertEquals(123l, entity.getStartTime().longValue()); - Assert.assertEquals(1, entity.getEvents().size()); - Assert.assertEquals(4, entity.getPrimaryFilters().size()); - Assert.assertEquals(0, entity.getOtherInfo().size()); + assertNotNull(entity); + assertEquals("id_1", entity.getEntityId()); + assertEquals("type_1", entity.getEntityType()); + assertEquals(123L, entity.getStartTime().longValue()); + assertEquals(1, entity.getEvents().size()); + assertEquals(4, entity.getPrimaryFilters().size()); + assertEquals(0, entity.getOtherInfo().size()); } @Test - public void testGetEvents() throws Exception { + void testGetEvents() throws Exception { WebResource r = resource(); ClientResponse response = r.path("ws").path("v1").path("timeline") .path("type_1").path("events") @@ -421,22 +422,22 @@ public void testGetEvents() throws Exception { assertEquals(MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8, response.getType().toString()); TimelineEvents events = response.getEntity(TimelineEvents.class); - Assert.assertNotNull(events); - Assert.assertEquals(1, events.getAllEvents().size()); + assertNotNull(events); + assertEquals(1, events.getAllEvents().size()); TimelineEvents.EventsOfOneEntity partEvents = events.getAllEvents().get(0); - Assert.assertEquals(2, partEvents.getEvents().size()); + assertEquals(2, partEvents.getEvents().size()); TimelineEvent event1 = partEvents.getEvents().get(0); - Assert.assertEquals(456l, event1.getTimestamp()); - Assert.assertEquals("end_event", event1.getEventType()); - Assert.assertEquals(1, event1.getEventInfo().size()); + assertEquals(456L, event1.getTimestamp()); + assertEquals("end_event", event1.getEventType()); + assertEquals(1, event1.getEventInfo().size()); TimelineEvent event2 = partEvents.getEvents().get(1); - Assert.assertEquals(123l, event2.getTimestamp()); - Assert.assertEquals("start_event", event2.getEventType()); - Assert.assertEquals(0, event2.getEventInfo().size()); + assertEquals(123L, event2.getTimestamp()); + assertEquals("start_event", event2.getEventType()); + assertEquals(0, event2.getEventInfo().size()); } @Test - public void testPostEntitiesWithPrimaryFilter() throws Exception { + void testPostEntitiesWithPrimaryFilter() throws Exception { TimelineEntities entities = new TimelineEntities(); TimelineEntity entity = new TimelineEntity(); Map> filters = new HashMap>(); @@ -455,11 +456,11 @@ public void testPostEntitiesWithPrimaryFilter() throws Exception { .post(ClientResponse.class, entities); TimelinePutResponse putResposne = response.getEntity(TimelinePutResponse.class); - Assert.assertEquals(0, putResposne.getErrors().size()); + assertEquals(0, putResposne.getErrors().size()); } @Test - public void testPostEntities() throws Exception { + void testPostEntities() throws Exception { TimelineEntities entities = new TimelineEntities(); TimelineEntity entity = new TimelineEntity(); entity.setEntityId("test id 1"); @@ -486,8 +487,8 @@ public void testPostEntities() throws Exception { response.getType().toString()); TimelinePutResponse putResposne = response.getEntity(TimelinePutResponse.class); - Assert.assertNotNull(putResposne); - Assert.assertEquals(0, putResposne.getErrors().size()); + assertNotNull(putResposne); + assertEquals(0, putResposne.getErrors().size()); // verify the entity exists in the store response = r.path("ws").path("v1").path("timeline") .path("test type 1").path("test id 1") @@ -496,13 +497,13 @@ public void testPostEntities() throws Exception { assertEquals(MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8, response.getType().toString()); entity = response.getEntity(TimelineEntity.class); - Assert.assertNotNull(entity); - Assert.assertEquals("test id 1", entity.getEntityId()); - Assert.assertEquals("test type 1", entity.getEntityType()); + assertNotNull(entity); + assertEquals("test id 1", entity.getEntityId()); + assertEquals("test type 1", entity.getEntityType()); } @Test - public void testPostIncompleteEntities() throws Exception { + void testPostIncompleteEntities() throws Exception { TimelineEntities entities = new TimelineEntities(); TimelineEntity entity1 = new TimelineEntity(); entity1.setEntityId("test id 1"); @@ -516,14 +517,14 @@ public void testPostIncompleteEntities() throws Exception { // One of the entities has no id or type. HTTP 400 will be returned ClientResponse response = r.path("ws").path("v1").path("timeline") .queryParam("user.name", "tester").accept(MediaType.APPLICATION_JSON) - .type(MediaType.APPLICATION_JSON).post(ClientResponse.class, entities); + .type(MediaType.APPLICATION_JSON).post(ClientResponse.class, entities); assertEquals(MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8, response.getType().toString()); assertResponseStatusCode(Status.BAD_REQUEST, response.getStatusInfo()); } @Test - public void testPostEntitiesWithYarnACLsEnabled() throws Exception { + void testPostEntitiesWithYarnACLsEnabled() throws Exception { AdminACLsManager oldAdminACLsManager = timelineACLsManager.setAdminACLsManager(adminACLsManager); try { @@ -544,8 +545,8 @@ public void testPostEntitiesWithYarnACLsEnabled() throws Exception { response.getType().toString()); TimelinePutResponse putResponse = response.getEntity(TimelinePutResponse.class); - Assert.assertNotNull(putResponse); - Assert.assertEquals(0, putResponse.getErrors().size()); + assertNotNull(putResponse); + assertEquals(0, putResponse.getErrors().size()); // override/append timeline data in the same entity with different user response = r.path("ws").path("v1").path("timeline") @@ -556,9 +557,9 @@ public void testPostEntitiesWithYarnACLsEnabled() throws Exception { assertEquals(MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8, response.getType().toString()); putResponse = response.getEntity(TimelinePutResponse.class); - Assert.assertNotNull(putResponse); - Assert.assertEquals(1, putResponse.getErrors().size()); - Assert.assertEquals(TimelinePutResponse.TimelinePutError.ACCESS_DENIED, + assertNotNull(putResponse); + assertEquals(1, putResponse.getErrors().size()); + assertEquals(TimelinePutResponse.TimelinePutError.ACCESS_DENIED, putResponse.getErrors().get(0).getErrorCode()); // Cross domain relationship will be rejected @@ -580,9 +581,9 @@ public void testPostEntitiesWithYarnACLsEnabled() throws Exception { assertEquals(MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8, response.getType().toString()); putResponse = response.getEntity(TimelinePutResponse.class); - Assert.assertNotNull(putResponse); - Assert.assertEquals(1, putResponse.getErrors().size()); - Assert.assertEquals(TimelinePutError.FORBIDDEN_RELATION, + assertNotNull(putResponse); + assertEquals(1, putResponse.getErrors().size()); + assertEquals(TimelinePutError.FORBIDDEN_RELATION, putResponse.getErrors().get(0).getErrorCode()); // Make sure the entity has been added anyway even though the @@ -595,16 +596,16 @@ public void testPostEntitiesWithYarnACLsEnabled() throws Exception { assertEquals(MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8, response.getType().toString()); entity = response.getEntity(TimelineEntity.class); - Assert.assertNotNull(entity); - Assert.assertEquals("test id 3", entity.getEntityId()); - Assert.assertEquals("test type 2", entity.getEntityType()); + assertNotNull(entity); + assertEquals("test id 3", entity.getEntityId()); + assertEquals("test type 2", entity.getEntityType()); } finally { timelineACLsManager.setAdminACLsManager(oldAdminACLsManager); } } @Test - public void testPostEntitiesToDefaultDomain() throws Exception { + void testPostEntitiesToDefaultDomain() throws Exception { AdminACLsManager oldAdminACLsManager = timelineACLsManager.setAdminACLsManager(adminACLsManager); try { @@ -624,8 +625,8 @@ public void testPostEntitiesToDefaultDomain() throws Exception { response.getType().toString()); TimelinePutResponse putResposne = response.getEntity(TimelinePutResponse.class); - Assert.assertNotNull(putResposne); - Assert.assertEquals(0, putResposne.getErrors().size()); + assertNotNull(putResposne); + assertEquals(0, putResposne.getErrors().size()); // verify the entity exists in the store response = r.path("ws").path("v1").path("timeline") .path("test type 7").path("test id 7") @@ -635,10 +636,10 @@ public void testPostEntitiesToDefaultDomain() throws Exception { assertEquals(MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8, response.getType().toString()); entity = response.getEntity(TimelineEntity.class); - Assert.assertNotNull(entity); - Assert.assertEquals("test id 7", entity.getEntityId()); - Assert.assertEquals("test type 7", entity.getEntityType()); - Assert.assertEquals(TimelineDataManager.DEFAULT_DOMAIN_ID, + assertNotNull(entity); + assertEquals("test id 7", entity.getEntityId()); + assertEquals("test type 7", entity.getEntityType()); + assertEquals(TimelineDataManager.DEFAULT_DOMAIN_ID, entity.getDomainId()); } finally { timelineACLsManager.setAdminACLsManager(oldAdminACLsManager); @@ -646,7 +647,7 @@ public void testPostEntitiesToDefaultDomain() throws Exception { } @Test - public void testGetEntityWithYarnACLsEnabled() throws Exception { + void testGetEntityWithYarnACLsEnabled() throws Exception { AdminACLsManager oldAdminACLsManager = timelineACLsManager.setAdminACLsManager(adminACLsManager); try { @@ -667,7 +668,7 @@ public void testGetEntityWithYarnACLsEnabled() throws Exception { response.getType().toString()); TimelinePutResponse putResponse = response.getEntity(TimelinePutResponse.class); - Assert.assertEquals(0, putResponse.getErrors().size()); + assertEquals(0, putResponse.getErrors().size()); // verify the system data will not be exposed // 1. No field specification response = r.path("ws").path("v1").path("timeline") @@ -678,7 +679,7 @@ public void testGetEntityWithYarnACLsEnabled() throws Exception { assertEquals(MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8, response.getType().toString()); entity = response.getEntity(TimelineEntity.class); - Assert.assertNull(entity.getPrimaryFilters().get( + assertNull(entity.getPrimaryFilters().get( TimelineStore.SystemFilter.ENTITY_OWNER.toString())); // 2. other field response = r.path("ws").path("v1").path("timeline") @@ -690,7 +691,7 @@ public void testGetEntityWithYarnACLsEnabled() throws Exception { assertEquals(MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8, response.getType().toString()); entity = response.getEntity(TimelineEntity.class); - Assert.assertNull(entity.getPrimaryFilters().get( + assertNull(entity.getPrimaryFilters().get( TimelineStore.SystemFilter.ENTITY_OWNER.toString())); // 3. primaryfilters field response = r.path("ws").path("v1").path("timeline") @@ -702,7 +703,7 @@ public void testGetEntityWithYarnACLsEnabled() throws Exception { assertEquals(MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8, response.getType().toString()); entity = response.getEntity(TimelineEntity.class); - Assert.assertNull(entity.getPrimaryFilters().get( + assertNull(entity.getPrimaryFilters().get( TimelineStore.SystemFilter.ENTITY_OWNER.toString())); // get entity with other user @@ -720,7 +721,7 @@ public void testGetEntityWithYarnACLsEnabled() throws Exception { } @Test - public void testGetEntitiesWithYarnACLsEnabled() { + void testGetEntitiesWithYarnACLsEnabled() { AdminACLsManager oldAdminACLsManager = timelineACLsManager.setAdminACLsManager(adminACLsManager); try { @@ -742,7 +743,7 @@ public void testGetEntitiesWithYarnACLsEnabled() { response.getType().toString()); TimelinePutResponse putResponse = response.getEntity(TimelinePutResponse.class); - Assert.assertEquals(0, putResponse.getErrors().size()); + assertEquals(0, putResponse.getErrors().size()); // Put entity [4, 5] in domain 2 entities = new TimelineEntities(); @@ -761,7 +762,7 @@ public void testGetEntitiesWithYarnACLsEnabled() { assertEquals(MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8, response.getType().toString()); putResponse = response.getEntity(TimelinePutResponse.class); - Assert.assertEquals(0, putResponse.getErrors().size()); + assertEquals(0, putResponse.getErrors().size()); // Query entities of type 4 response = r.path("ws").path("v1").path("timeline") @@ -782,7 +783,7 @@ public void testGetEntitiesWithYarnACLsEnabled() { } @Test - public void testGetEventsWithYarnACLsEnabled() { + void testGetEventsWithYarnACLsEnabled() { AdminACLsManager oldAdminACLsManager = timelineACLsManager.setAdminACLsManager(adminACLsManager); try { @@ -808,7 +809,7 @@ public void testGetEventsWithYarnACLsEnabled() { response.getType().toString()); TimelinePutResponse putResponse = response.getEntity(TimelinePutResponse.class); - Assert.assertEquals(0, putResponse.getErrors().size()); + assertEquals(0, putResponse.getErrors().size()); // Put entity [5, 6] in domain 2 entities = new TimelineEntities(); @@ -831,7 +832,7 @@ public void testGetEventsWithYarnACLsEnabled() { assertEquals(MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8, response.getType().toString()); putResponse = response.getEntity(TimelinePutResponse.class); - Assert.assertEquals(0, putResponse.getErrors().size()); + assertEquals(0, putResponse.getErrors().size()); // Query events belonging to the entities of type 4 response = r.path("ws").path("v1").path("timeline") @@ -852,7 +853,7 @@ public void testGetEventsWithYarnACLsEnabled() { } @Test - public void testGetDomain() throws Exception { + void testGetDomain() throws Exception { WebResource r = resource(); ClientResponse response = r.path("ws").path("v1").path("timeline") .path("domain").path("domain_id_1") @@ -865,7 +866,7 @@ public void testGetDomain() throws Exception { } @Test - public void testGetDomainYarnACLsEnabled() { + void testGetDomainYarnACLsEnabled() { AdminACLsManager oldAdminACLsManager = timelineACLsManager.setAdminACLsManager(adminACLsManager); try { @@ -894,7 +895,7 @@ public void testGetDomainYarnACLsEnabled() { } @Test - public void testGetDomains() throws Exception { + void testGetDomains() throws Exception { WebResource r = resource(); ClientResponse response = r.path("ws").path("v1").path("timeline") .path("domain") @@ -904,7 +905,7 @@ public void testGetDomains() throws Exception { assertEquals(MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8, response.getType().toString()); TimelineDomains domains = response.getEntity(TimelineDomains.class); - Assert.assertEquals(2, domains.getDomains().size()); + assertEquals(2, domains.getDomains().size()); for (int i = 0; i < domains.getDomains().size(); ++i) { verifyDomain(domains.getDomains().get(i), i == 0 ? "domain_id_4" : "domain_id_1"); @@ -912,7 +913,7 @@ public void testGetDomains() throws Exception { } @Test - public void testGetDomainsYarnACLsEnabled() throws Exception { + void testGetDomainsYarnACLsEnabled() throws Exception { AdminACLsManager oldAdminACLsManager = timelineACLsManager.setAdminACLsManager(adminACLsManager); try { @@ -925,7 +926,7 @@ public void testGetDomainsYarnACLsEnabled() throws Exception { assertEquals(MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8, response.getType().toString()); TimelineDomains domains = response.getEntity(TimelineDomains.class); - Assert.assertEquals(2, domains.getDomains().size()); + assertEquals(2, domains.getDomains().size()); for (int i = 0; i < domains.getDomains().size(); ++i) { verifyDomain(domains.getDomains().get(i), i == 0 ? "domain_id_4" : "domain_id_1"); @@ -940,14 +941,14 @@ public void testGetDomainsYarnACLsEnabled() throws Exception { assertEquals(MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8, response.getType().toString()); domains = response.getEntity(TimelineDomains.class); - Assert.assertEquals(0, domains.getDomains().size()); + assertEquals(0, domains.getDomains().size()); } finally { timelineACLsManager.setAdminACLsManager(oldAdminACLsManager); } } @Test - public void testPutDomain() throws Exception { + void testPutDomain() throws Exception { TimelineDomain domain = new TimelineDomain(); domain.setId("test_domain_id"); WebResource r = resource(); @@ -977,10 +978,10 @@ public void testPutDomain() throws Exception { assertEquals(MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8, response.getType().toString()); domain = response.getEntity(TimelineDomain.class); - Assert.assertNotNull(domain); - Assert.assertEquals("test_domain_id", domain.getId()); - Assert.assertEquals("tester", domain.getOwner()); - Assert.assertEquals(null, domain.getDescription()); + assertNotNull(domain); + assertEquals("test_domain_id", domain.getId()); + assertEquals("tester", domain.getOwner()); + assertNull(domain.getDescription()); // Update the domain domain.setDescription("test_description"); @@ -1000,13 +1001,13 @@ public void testPutDomain() throws Exception { assertEquals(MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8, response.getType().toString()); domain = response.getEntity(TimelineDomain.class); - Assert.assertNotNull(domain); - Assert.assertEquals("test_domain_id", domain.getId()); - Assert.assertEquals("test_description", domain.getDescription()); + assertNotNull(domain); + assertEquals("test_domain_id", domain.getId()); + assertEquals("test_description", domain.getDescription()); } @Test - public void testPutDomainYarnACLsEnabled() throws Exception { + void testPutDomainYarnACLsEnabled() throws Exception { AdminACLsManager oldAdminACLsManager = timelineACLsManager.setAdminACLsManager(adminACLsManager); try { @@ -1035,7 +1036,7 @@ public void testPutDomainYarnACLsEnabled() throws Exception { } @Test - public void testContextFactory() throws Exception { + void testContextFactory() throws Exception { JAXBContext jaxbContext1 = ContextFactory.createContext( new Class[]{TimelineDomain.class}, Collections.EMPTY_MAP); JAXBContext jaxbContext2 = ContextFactory.createContext( @@ -1045,21 +1046,21 @@ public void testContextFactory() throws Exception { try { ContextFactory.createContext(new Class[]{TimelineEntity.class}, Collections.EMPTY_MAP); - Assert.fail("Expected JAXBException"); - } catch(Exception e) { + fail("Expected JAXBException"); + } catch (Exception e) { assertThat(e).isExactlyInstanceOf(JAXBException.class); } } private static void verifyDomain(TimelineDomain domain, String domainId) { - Assert.assertNotNull(domain); - Assert.assertEquals(domainId, domain.getId()); + assertNotNull(domain); + assertEquals(domainId, domain.getId()); // The specific values have been verified in TestMemoryTimelineStore - Assert.assertNotNull(domain.getDescription()); - Assert.assertNotNull(domain.getOwner()); - Assert.assertNotNull(domain.getReaders()); - Assert.assertNotNull(domain.getWriters()); - Assert.assertNotNull(domain.getCreatedTime()); - Assert.assertNotNull(domain.getModifiedTime()); + assertNotNull(domain.getDescription()); + assertNotNull(domain.getOwner()); + assertNotNull(domain.getReaders()); + assertNotNull(domain.getWriters()); + assertNotNull(domain.getCreatedTime()); + assertNotNull(domain.getModifiedTime()); } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/webapp/TestTimelineWebServicesWithSSL.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/webapp/TestTimelineWebServicesWithSSL.java index 36209a9be4bde..70b58aaca01f4 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/webapp/TestTimelineWebServicesWithSSL.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/webapp/TestTimelineWebServicesWithSSL.java @@ -23,6 +23,12 @@ import java.net.URI; import java.util.EnumSet; +import com.sun.jersey.api.client.Client; +import com.sun.jersey.api.client.ClientResponse; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; + import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileUtil; import org.apache.hadoop.security.UserGroupInformation; @@ -38,13 +44,10 @@ import org.apache.hadoop.yarn.server.timeline.MemoryTimelineStore; import org.apache.hadoop.yarn.server.timeline.TimelineReader.Field; import org.apache.hadoop.yarn.server.timeline.TimelineStore; -import org.junit.AfterClass; -import org.junit.Assert; -import org.junit.BeforeClass; -import org.junit.Test; -import com.sun.jersey.api.client.Client; -import com.sun.jersey.api.client.ClientResponse; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; public class TestTimelineWebServicesWithSSL { @@ -58,7 +61,7 @@ public class TestTimelineWebServicesWithSSL { private static TimelineStore store; private static Configuration conf; - @BeforeClass + @BeforeAll public static void setupServer() throws Exception { conf = new YarnConfiguration(); conf.setBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, true); @@ -84,7 +87,7 @@ public static void setupServer() throws Exception { store = timelineServer.getTimelineStore(); } - @AfterClass + @AfterAll public static void tearDownServer() throws Exception { if (timelineServer != null) { timelineServer.stop(); @@ -92,7 +95,7 @@ public static void tearDownServer() throws Exception { } @Test - public void testPutEntities() throws Exception { + void testPutEntities() throws Exception { TestTimelineClient client = new TestTimelineClient(); try { client.init(conf); @@ -107,16 +110,16 @@ public void testPutEntities() throws Exception { expectedEntity.addEvent(event); TimelinePutResponse response = client.putEntities(expectedEntity); - Assert.assertEquals(0, response.getErrors().size()); - Assert.assertTrue(client.resp.toString().contains("https")); + assertEquals(0, response.getErrors().size()); + assertTrue(client.resp.toString().contains("https")); TimelineEntity actualEntity = store.getEntity( expectedEntity.getEntityId(), expectedEntity.getEntityType(), EnumSet.allOf(Field.class)); - Assert.assertNotNull(actualEntity); - Assert.assertEquals( + assertNotNull(actualEntity); + assertEquals( expectedEntity.getEntityId(), actualEntity.getEntityId()); - Assert.assertEquals( + assertEquals( expectedEntity.getEntityType(), actualEntity.getEntityType()); } finally { client.stop(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/FederationPolicyUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/FederationPolicyUtils.java index 9b795b0507bd2..fc8cc5bd464b7 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/FederationPolicyUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/FederationPolicyUtils.java @@ -20,7 +20,7 @@ import java.nio.ByteBuffer; import java.nio.charset.StandardCharsets; import java.util.ArrayList; -import java.util.List; +import java.util.Collection; import java.util.Random; import org.apache.hadoop.classification.InterfaceAudience.Private; @@ -188,8 +188,8 @@ public static FederationAMRMProxyPolicy loadAMRMPolicy(String queue, * @throws FederationPolicyException if there are no usable subclusters. */ public static void validateSubClusterAvailability( - List activeSubClusters, - List blackListSubClusters) + Collection activeSubClusters, + Collection blackListSubClusters) throws FederationPolicyException { if (activeSubClusters != null && !activeSubClusters.isEmpty()) { if (blackListSubClusters == null) { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/RouterPolicyFacade.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/RouterPolicyFacade.java index 0cbda314f287e..c4fc7322a1eca 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/RouterPolicyFacade.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/RouterPolicyFacade.java @@ -25,6 +25,7 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext; +import org.apache.hadoop.yarn.api.protocolrecords.ReservationSubmissionRequest; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.server.federation.policies.exceptions.FederationPolicyException; @@ -136,7 +137,7 @@ public SubClusterId getHomeSubcluster( if (appSubmissionContext == null) { throw new FederationPolicyException( - "The ApplicationSubmissionContext " + "cannot be null."); + "The ApplicationSubmissionContext cannot be null."); } String queue = appSubmissionContext.getQueue(); @@ -148,51 +149,7 @@ public SubClusterId getHomeSubcluster( queue = YarnConfiguration.DEFAULT_QUEUE_NAME; } - // the facade might cache this request, based on its parameterization - SubClusterPolicyConfiguration configuration = null; - - try { - configuration = federationFacade.getPolicyConfiguration(queue); - } catch (YarnException e) { - String errMsg = "There is no policy configured for the queue: " + queue - + ", falling back to defaults."; - LOG.warn(errMsg, e); - } - - // If there is no policy configured for this queue, fallback to the baseline - // policy that is configured either in the store or via XML config (and - // cached) - if (configuration == null) { - LOG.warn("There is no policies configured for queue: " + queue + " we" - + " fallback to default policy for: " - + YarnConfiguration.DEFAULT_FEDERATION_POLICY_KEY); - - queue = YarnConfiguration.DEFAULT_FEDERATION_POLICY_KEY; - try { - configuration = federationFacade.getPolicyConfiguration(queue); - } catch (YarnException e) { - String errMsg = "Cannot retrieve policy configured for the queue: " - + queue + ", falling back to defaults."; - LOG.warn(errMsg, e); - - } - } - - // the fallback is not configure via store, but via XML, using - // previously loaded configuration. - if (configuration == null) { - configuration = - cachedConfs.get(YarnConfiguration.DEFAULT_FEDERATION_POLICY_KEY); - } - - // if the configuration has changed since last loaded, reinit the policy - // based on current configuration - if (!cachedConfs.containsKey(queue) - || !cachedConfs.get(queue).equals(configuration)) { - singlePolicyReinit(policyMap, cachedConfs, queue, configuration); - } - - FederationRouterPolicy policy = policyMap.get(queue); + FederationRouterPolicy policy = getFederationRouterPolicy(cachedConfs, policyMap, queue); if (policy == null) { // this should never happen, as the to maps are updated together throw new FederationPolicyException("No FederationRouterPolicy found " @@ -262,4 +219,92 @@ public synchronized void reset() { } + /** + * This method provides a wrapper of all policy functionalities for routing a + * reservation. Internally it manages configuration changes, and policy + * init/reinit. + * + * @param request the reservation to route. + * + * @return the id of the subcluster that will be the "home" for this + * reservation. + * + * @throws YarnException if there are issues initializing policies, or no + * valid sub-cluster id could be found for this reservation. + */ + public SubClusterId getReservationHomeSubCluster( + ReservationSubmissionRequest request) throws YarnException { + + // the maps are concurrent, but we need to protect from reset() + // reinitialization mid-execution by creating a new reference local to this + // method. + Map cachedConfs = globalConfMap; + Map policyMap = globalPolicyMap; + + if (request == null) { + throw new FederationPolicyException( + "The ReservationSubmissionRequest cannot be null."); + } + + String queue = request.getQueue(); + FederationRouterPolicy policy = getFederationRouterPolicy(cachedConfs, policyMap, queue); + + if (policy == null) { + // this should never happen, as the to maps are updated together + throw new FederationPolicyException("No FederationRouterPolicy found " + + "for queue: " + request.getQueue() + " (while routing " + + "reservation: " + request.getReservationId() + ") " + + "and no default specified."); + } + + return policy.getReservationHomeSubcluster(request); + } + + private FederationRouterPolicy getFederationRouterPolicy( + Map cachedConfiguration, + Map policyMap, String queue) + throws FederationPolicyInitializationException { + + // the facade might cache this request, based on its parameterization + SubClusterPolicyConfiguration configuration = null; + String copyQueue = queue; + + try { + configuration = federationFacade.getPolicyConfiguration(copyQueue); + } catch (YarnException e) { + LOG.warn("There is no policy configured for the queue: {}, falling back to defaults.", + copyQueue, e); + } + + // If there is no policy configured for this queue, fallback to the baseline + // policy that is configured either in the store or via XML config (and cached) + if (configuration == null) { + final String policyKey = YarnConfiguration.DEFAULT_FEDERATION_POLICY_KEY; + LOG.warn("There is no policies configured for queue: {} " + + "we fallback to default policy for: {}. ", copyQueue, policyKey); + copyQueue = YarnConfiguration.DEFAULT_FEDERATION_POLICY_KEY; + try { + configuration = federationFacade.getPolicyConfiguration(copyQueue); + } catch (YarnException e) { + LOG.warn("Cannot retrieve policy configured for the queue: {}, falling back to defaults.", + copyQueue, e); + } + } + + // the fallback is not configure via store, but via XML, using + // previously loaded configuration. + if (configuration == null) { + configuration = cachedConfiguration.get(YarnConfiguration.DEFAULT_FEDERATION_POLICY_KEY); + } + + // if the configuration has changed since last loaded, reinit the policy + // based on current configuration + SubClusterPolicyConfiguration policyConfiguration = + cachedConfiguration.getOrDefault(copyQueue, null); + if (policyConfiguration == null || !policyConfiguration.equals(configuration)) { + singlePolicyReinit(policyMap, cachedConfiguration, copyQueue, configuration); + } + + return policyMap.get(copyQueue); + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/router/AbstractRouterPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/router/AbstractRouterPolicy.java index 730fb417f883d..3bab20cef1661 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/router/AbstractRouterPolicy.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/router/AbstractRouterPolicy.java @@ -18,15 +18,24 @@ package org.apache.hadoop.yarn.server.federation.policies.router; +import java.util.List; import java.util.Map; +import java.util.Collections; +import org.apache.hadoop.yarn.api.protocolrecords.ReservationSubmissionRequest; import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext; +import org.apache.hadoop.yarn.api.records.ReservationId; import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.server.federation.policies.AbstractConfigurableFederationPolicy; +import org.apache.hadoop.yarn.server.federation.policies.FederationPolicyUtils; import org.apache.hadoop.yarn.server.federation.policies.dao.WeightedPolicyInfo; import org.apache.hadoop.yarn.server.federation.policies.exceptions.FederationPolicyException; import org.apache.hadoop.yarn.server.federation.policies.exceptions.FederationPolicyInitializationException; +import org.apache.hadoop.yarn.server.federation.store.records.SubClusterId; import org.apache.hadoop.yarn.server.federation.store.records.SubClusterIdInfo; +import org.apache.hadoop.yarn.server.federation.store.records.SubClusterInfo; +import org.apache.hadoop.yarn.server.federation.utils.FederationStateStoreFacade; /** * Base abstract class for {@link FederationRouterPolicy} implementations, that @@ -63,4 +72,116 @@ public void validate(ApplicationSubmissionContext appSubmissionContext) } } + /** + * This method is implemented by the specific policy, and it is used to route + * both reservations, and applications among a given set of + * sub-clusters. + * + * @param queue the queue for this application/reservation + * @param preSelectSubClusters a pre-filter set of sub-clusters + * @return the chosen sub-cluster + * + * @throws YarnException if the policy fails to choose a sub-cluster + */ + protected abstract SubClusterId chooseSubCluster(String queue, + Map preSelectSubClusters) throws YarnException; + + /** + * Filter chosen SubCluster based on reservationId. + * + * @param reservationId the globally unique identifier for a reservation. + * @param activeSubClusters the map of ids to info for all active subclusters. + * @return the chosen sub-cluster + * @throws YarnException if the policy fails to choose a sub-cluster + */ + protected Map prefilterSubClusters( + ReservationId reservationId, Map activeSubClusters) + throws YarnException { + + // if a reservation exists limit scope to the sub-cluster this + // reservation is mapped to + if (reservationId != null) { + // note this might throw YarnException if the reservation is + // unknown. This is to be expected, and should be handled by + // policy invoker. + FederationStateStoreFacade stateStoreFacade = + getPolicyContext().getFederationStateStoreFacade(); + SubClusterId resSubCluster = stateStoreFacade.getReservationHomeSubCluster(reservationId); + SubClusterInfo subClusterInfo = activeSubClusters.get(resSubCluster); + return Collections.singletonMap(resSubCluster, subClusterInfo); + } + + return activeSubClusters; + } + + /** + * Simply picks from alphabetically-sorted active subclusters based on the + * hash of quey name. Jobs of the same queue will all be routed to the same + * sub-cluster, as far as the number of active sub-cluster and their names + * remain the same. + * + * @param appContext the {@link ApplicationSubmissionContext} that + * has to be routed to an appropriate subCluster for execution. + * + * @param blackLists the list of subClusters as identified by + * {@link SubClusterId} to blackList from the selection of the home + * subCluster. + * + * @return a hash-based chosen {@link SubClusterId} that will be the "home" + * for this application. + * + * @throws YarnException if there are no active subclusters. + */ + @Override + public SubClusterId getHomeSubcluster(ApplicationSubmissionContext appContext, + List blackLists) throws YarnException { + + // null checks and default-queue behavior + validate(appContext); + + // apply filtering based on reservation location and active sub-clusters + Map filteredSubClusters = prefilterSubClusters( + appContext.getReservationID(), getActiveSubclusters()); + + FederationPolicyUtils.validateSubClusterAvailability(filteredSubClusters.keySet(), blackLists); + + // remove black SubCluster + if (blackLists != null) { + blackLists.forEach(filteredSubClusters::remove); + } + + // pick the chosen subCluster from the active ones + return chooseSubCluster(appContext.getQueue(), filteredSubClusters); + } + + /** + * This method provides a wrapper of all policy functionalities for routing a + * reservation. Internally it manages configuration changes, and policy + * init/reinit. + * + * @param request the reservation to route. + * + * @return the id of the subcluster that will be the "home" for this + * reservation. + * + * @throws YarnException if there are issues initializing policies, or no + * valid sub-cluster id could be found for this reservation. + */ + @Override + public SubClusterId getReservationHomeSubcluster(ReservationSubmissionRequest request) + throws YarnException { + if (request == null) { + throw new FederationPolicyException("The ReservationSubmissionRequest cannot be null."); + } + + if (request.getQueue() == null) { + request.setQueue(YarnConfiguration.DEFAULT_QUEUE_NAME); + } + + // apply filtering based on reservation location and active sub-clusters + Map filteredSubClusters = getActiveSubclusters(); + + // pick the chosen subCluster from the active ones + return chooseSubCluster(request.getQueue(), filteredSubClusters); + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/router/FederationRouterPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/router/FederationRouterPolicy.java index 9325bd8ca2a15..af5810665913c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/router/FederationRouterPolicy.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/router/FederationRouterPolicy.java @@ -19,6 +19,7 @@ import java.util.List; +import org.apache.hadoop.yarn.api.protocolrecords.ReservationSubmissionRequest; import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext; import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.server.federation.policies.ConfigurableFederationPolicy; @@ -49,4 +50,16 @@ public interface FederationRouterPolicy extends ConfigurableFederationPolicy { SubClusterId getHomeSubcluster( ApplicationSubmissionContext appSubmissionContext, List blackListSubClusters) throws YarnException; + + /** + * Determines the sub-cluster where a ReservationSubmissionRequest should be + * sent to. + * + * @param request the original request + * @return a mapping of sub-clusters and the requests + * + * @throws YarnException if the policy fails to choose a sub-cluster + */ + SubClusterId getReservationHomeSubcluster( + ReservationSubmissionRequest request) throws YarnException; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/router/HashBasedRouterPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/router/HashBasedRouterPolicy.java index cc11880665335..5ac2d1cce0720 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/router/HashBasedRouterPolicy.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/router/HashBasedRouterPolicy.java @@ -22,11 +22,9 @@ import java.util.List; import java.util.Map; -import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext; import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.server.federation.policies.FederationPolicyInitializationContext; import org.apache.hadoop.yarn.server.federation.policies.FederationPolicyInitializationContextValidator; -import org.apache.hadoop.yarn.server.federation.policies.FederationPolicyUtils; import org.apache.hadoop.yarn.server.federation.policies.exceptions.FederationPolicyInitializationException; import org.apache.hadoop.yarn.server.federation.store.records.SubClusterId; import org.apache.hadoop.yarn.server.federation.store.records.SubClusterInfo; @@ -50,53 +48,12 @@ public void reinitialize( setPolicyContext(federationPolicyContext); } - /** - * Simply picks from alphabetically-sorted active subclusters based on the - * hash of quey name. Jobs of the same queue will all be routed to the same - * sub-cluster, as far as the number of active sub-cluster and their names - * remain the same. - * - * @param appSubmissionContext the {@link ApplicationSubmissionContext} that - * has to be routed to an appropriate subCluster for execution. - * - * @param blackListSubClusters the list of subClusters as identified by - * {@link SubClusterId} to blackList from the selection of the home - * subCluster. - * - * @return a hash-based chosen {@link SubClusterId} that will be the "home" - * for this application. - * - * @throws YarnException if there are no active subclusters. - */ @Override - public SubClusterId getHomeSubcluster( - ApplicationSubmissionContext appSubmissionContext, - List blackListSubClusters) throws YarnException { - - // throws if no active subclusters available - Map activeSubclusters = - getActiveSubclusters(); - - FederationPolicyUtils.validateSubClusterAvailability( - new ArrayList(activeSubclusters.keySet()), - blackListSubClusters); - - if (blackListSubClusters != null) { - - // Remove from the active SubClusters from StateStore the blacklisted ones - for (SubClusterId scId : blackListSubClusters) { - activeSubclusters.remove(scId); - } - } - - validate(appSubmissionContext); - - int chosenPosition = Math.abs( - appSubmissionContext.getQueue().hashCode() % activeSubclusters.size()); - - List list = new ArrayList<>(activeSubclusters.keySet()); + protected SubClusterId chooseSubCluster(String queue, + Map preSelectSubclusters) throws YarnException { + int chosenPosition = Math.abs(queue.hashCode() % preSelectSubclusters.size()); + List list = new ArrayList<>(preSelectSubclusters.keySet()); Collections.sort(list); return list.get(chosenPosition); } - } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/router/LoadBasedRouterPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/router/LoadBasedRouterPolicy.java index fa5eb4be2cfd5..a86a43a213de0 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/router/LoadBasedRouterPolicy.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/router/LoadBasedRouterPolicy.java @@ -17,14 +17,10 @@ package org.apache.hadoop.yarn.server.federation.policies.router; -import java.util.ArrayList; -import java.util.List; import java.util.Map; -import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext; import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.server.federation.policies.FederationPolicyInitializationContext; -import org.apache.hadoop.yarn.server.federation.policies.FederationPolicyUtils; import org.apache.hadoop.yarn.server.federation.policies.dao.WeightedPolicyInfo; import org.apache.hadoop.yarn.server.federation.policies.exceptions.FederationPolicyException; import org.apache.hadoop.yarn.server.federation.policies.exceptions.FederationPolicyInitializationException; @@ -65,28 +61,12 @@ public void reinitialize(FederationPolicyInitializationContext policyContext) } @Override - public SubClusterId getHomeSubcluster( - ApplicationSubmissionContext appSubmissionContext, - List blacklist) throws YarnException { - - // null checks and default-queue behavior - validate(appSubmissionContext); - - Map activeSubclusters = - getActiveSubclusters(); - - FederationPolicyUtils.validateSubClusterAvailability( - new ArrayList(activeSubclusters.keySet()), blacklist); - - Map weights = - getPolicyInfo().getRouterPolicyWeights(); + protected SubClusterId chooseSubCluster( + String queue, Map preSelectSubclusters) throws YarnException { + Map weights = getPolicyInfo().getRouterPolicyWeights(); SubClusterIdInfo chosen = null; long currBestMem = -1; - for (Map.Entry entry : activeSubclusters - .entrySet()) { - if (blacklist != null && blacklist.contains(entry.getKey())) { - continue; - } + for (Map.Entry entry : preSelectSubclusters.entrySet()) { SubClusterIdInfo id = new SubClusterIdInfo(entry.getKey()); if (weights.containsKey(id) && weights.get(id) > 0) { long availableMemory = getAvailableMemory(entry.getValue()); @@ -110,7 +90,7 @@ private long getAvailableMemory(SubClusterInfo value) throws YarnException { mem = obj.getJSONObject("clusterMetrics").getLong("availableMB"); return mem; } catch (JSONException j) { - throw new YarnException("FederationSubCluserInfo cannot be parsed", j); + throw new YarnException("FederationSubClusterInfo cannot be parsed", j); } } } \ No newline at end of file diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/router/LocalityRouterPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/router/LocalityRouterPolicy.java index 469240af518d9..3abcf6fa378e4 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/router/LocalityRouterPolicy.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/router/LocalityRouterPolicy.java @@ -21,6 +21,7 @@ import java.util.ArrayList; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.Collections; import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext; @@ -78,7 +79,7 @@ public void reinitialize(FederationPolicyInitializationContext policyContext) resolver = policyContext.getFederationSubclusterResolver(); Map weights = getPolicyInfo().getRouterPolicyWeights(); - enabledSCs = new ArrayList(); + enabledSCs = new ArrayList<>(); for (Map.Entry entry : weights.entrySet()) { if (entry != null && entry.getValue() > 0) { enabledSCs.add(entry.getKey().toId()); @@ -100,8 +101,7 @@ public SubClusterId getHomeSubcluster( // Fast path for FailForward to WeightedRandomRouterPolicy if (rrList == null || rrList.isEmpty() || (rrList.size() == 1 && ResourceRequest.isAnyLocation(rrList.get(0).getResourceName()))) { - return super - .getHomeSubcluster(appSubmissionContext, blackListSubClusters); + return super.getHomeSubcluster(appSubmissionContext, blackListSubClusters); } if (rrList.size() != 3) { @@ -109,12 +109,11 @@ public SubClusterId getHomeSubcluster( "Invalid number of resource requests: " + rrList.size()); } - Map activeSubClusters = - getActiveSubclusters(); - List validSubClusters = - new ArrayList<>(activeSubClusters.keySet()); - FederationPolicyUtils - .validateSubClusterAvailability(validSubClusters, blackListSubClusters); + Map activeSubClusters = getActiveSubclusters(); + Set validSubClusters = activeSubClusters.keySet(); + FederationPolicyUtils.validateSubClusterAvailability(activeSubClusters.keySet(), + blackListSubClusters); + if (blackListSubClusters != null) { // Remove from the active SubClusters from StateStore the blacklisted ones validSubClusters.removeAll(blackListSubClusters); @@ -128,20 +127,21 @@ public SubClusterId getHomeSubcluster( ResourceRequest nodeRequest = null; ResourceRequest rackRequest = null; ResourceRequest anyRequest = null; + for (ResourceRequest rr : rrList) { // Handle "node" requests try { targetId = resolver.getSubClusterForNode(rr.getResourceName()); nodeRequest = rr; } catch (YarnException e) { - LOG.error("Cannot resolve node : {}", e.getLocalizedMessage()); + LOG.error("Cannot resolve node : {}.", e.getMessage()); } // Handle "rack" requests try { resolver.getSubClustersForRack(rr.getResourceName()); rackRequest = rr; } catch (YarnException e) { - LOG.error("Cannot resolve rack : {}", e.getLocalizedMessage()); + LOG.error("Cannot resolve rack : {}.", e.getMessage()); } // Handle "ANY" requests if (ResourceRequest.isAnyLocation(rr.getResourceName())) { @@ -149,32 +149,33 @@ public SubClusterId getHomeSubcluster( continue; } } + if (nodeRequest == null) { - throw new YarnException("Missing node request"); + throw new YarnException("Missing node request."); } if (rackRequest == null) { - throw new YarnException("Missing rack request"); + throw new YarnException("Missing rack request."); } if (anyRequest == null) { - throw new YarnException("Missing any request"); + throw new YarnException("Missing any request."); } - LOG.info( - "Node request: " + nodeRequest.getResourceName() + ", Rack request: " - + rackRequest.getResourceName() + ", Any request: " + anyRequest - .getResourceName()); + + LOG.info("Node request: {} , Rack request: {} , Any request: {}.", + nodeRequest.getResourceName(), rackRequest.getResourceName(), + anyRequest.getResourceName()); + // Handle "node" requests if (validSubClusters.contains(targetId) && enabledSCs .contains(targetId)) { - LOG.info("Node {} is in SubCluster: {}", nodeRequest.getResourceName(), - targetId); + LOG.info("Node {} is in SubCluster: {}.", nodeRequest.getResourceName(), targetId); return targetId; } else { throw new YarnException("The node " + nodeRequest.getResourceName() + " is in a blacklist SubCluster or not active. "); } } catch (YarnException e) { - LOG.error("Validating resource requests failed, Falling back to " - + "WeightedRandomRouterPolicy placement: " + e.getMessage()); + LOG.error("Validating resource requests failed, " + + "Falling back to WeightedRandomRouterPolicy placement : {}.", e.getMessage()); // FailForward to WeightedRandomRouterPolicy // Overwrite request to use a default ANY ResourceRequest amReq = Records.newRecord(ResourceRequest.class); @@ -183,14 +184,10 @@ public SubClusterId getHomeSubcluster( amReq.setCapability(appSubmissionContext.getResource()); amReq.setNumContainers(1); amReq.setRelaxLocality(true); - amReq.setNodeLabelExpression( - appSubmissionContext.getNodeLabelExpression()); - amReq.setExecutionTypeRequest( - ExecutionTypeRequest.newInstance(ExecutionType.GUARANTEED)); - appSubmissionContext - .setAMContainerResourceRequests(Collections.singletonList(amReq)); - return super - .getHomeSubcluster(appSubmissionContext, blackListSubClusters); + amReq.setNodeLabelExpression(appSubmissionContext.getNodeLabelExpression()); + amReq.setExecutionTypeRequest(ExecutionTypeRequest.newInstance(ExecutionType.GUARANTEED)); + appSubmissionContext.setAMContainerResourceRequests(Collections.singletonList(amReq)); + return super.getHomeSubcluster(appSubmissionContext, blackListSubClusters); } } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/router/PriorityRouterPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/router/PriorityRouterPolicy.java index b81ca07b42ad8..7d50d3814a0dd 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/router/PriorityRouterPolicy.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/router/PriorityRouterPolicy.java @@ -17,13 +17,9 @@ package org.apache.hadoop.yarn.server.federation.policies.router; -import java.util.ArrayList; -import java.util.List; import java.util.Map; -import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext; import org.apache.hadoop.yarn.exceptions.YarnException; -import org.apache.hadoop.yarn.server.federation.policies.FederationPolicyUtils; import org.apache.hadoop.yarn.server.federation.policies.exceptions.FederationPolicyException; import org.apache.hadoop.yarn.server.federation.store.records.SubClusterId; import org.apache.hadoop.yarn.server.federation.store.records.SubClusterIdInfo; @@ -37,30 +33,15 @@ public class PriorityRouterPolicy extends AbstractRouterPolicy { @Override - public SubClusterId getHomeSubcluster( - ApplicationSubmissionContext appSubmissionContext, - List blacklist) throws YarnException { - - // null checks and default-queue behavior - validate(appSubmissionContext); - - Map activeSubclusters = - getActiveSubclusters(); - - FederationPolicyUtils.validateSubClusterAvailability( - new ArrayList(activeSubclusters.keySet()), blacklist); - + protected SubClusterId chooseSubCluster( + String queue, Map preSelectSubclusters) throws YarnException { // This finds the sub-cluster with the highest weight among the // currently active ones. - Map weights = - getPolicyInfo().getRouterPolicyWeights(); + Map weights = getPolicyInfo().getRouterPolicyWeights(); SubClusterId chosen = null; Float currentBest = Float.MIN_VALUE; - for (SubClusterId id : activeSubclusters.keySet()) { + for (SubClusterId id : preSelectSubclusters.keySet()) { SubClusterIdInfo idInfo = new SubClusterIdInfo(id); - if (blacklist != null && blacklist.contains(id)) { - continue; - } if (weights.containsKey(idInfo) && weights.get(idInfo) > currentBest) { currentBest = weights.get(idInfo); chosen = id; @@ -68,10 +49,8 @@ public SubClusterId getHomeSubcluster( } if (chosen == null) { throw new FederationPolicyException( - "No Active Subcluster with weight vector greater than zero"); + "No Active Subcluster with weight vector greater than zero."); } - return chosen; } - } \ No newline at end of file diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/router/RejectRouterPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/router/RejectRouterPolicy.java index b4c019270249c..32e31ebfec712 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/router/RejectRouterPolicy.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/router/RejectRouterPolicy.java @@ -17,15 +17,15 @@ package org.apache.hadoop.yarn.server.federation.policies.router; -import java.util.List; +import java.util.Map; -import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext; import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.server.federation.policies.FederationPolicyInitializationContext; import org.apache.hadoop.yarn.server.federation.policies.FederationPolicyInitializationContextValidator; import org.apache.hadoop.yarn.server.federation.policies.exceptions.FederationPolicyException; import org.apache.hadoop.yarn.server.federation.policies.exceptions.FederationPolicyInitializationException; import org.apache.hadoop.yarn.server.federation.store.records.SubClusterId; +import org.apache.hadoop.yarn.server.federation.store.records.SubClusterInfo; /** * This {@link FederationRouterPolicy} simply rejects all incoming requests. @@ -43,34 +43,12 @@ public void reinitialize( setPolicyContext(federationPolicyContext); } - /** - * The policy always reject requests. - * - * @param appSubmissionContext the {@link ApplicationSubmissionContext} that - * has to be routed to an appropriate subCluster for execution. - * - * @param blackListSubClusters the list of subClusters as identified by - * {@link SubClusterId} to blackList from the selection of the home - * subCluster. - * - * @return (never). - * - * @throws YarnException (always) to prevent applications in this queue to be - * run anywhere in the federated cluster. - */ @Override - public SubClusterId getHomeSubcluster( - ApplicationSubmissionContext appSubmissionContext, - List blackListSubClusters) throws YarnException { - - // run standard validation, as error might differ - validate(appSubmissionContext); - - throw new FederationPolicyException("The policy configured for this queue" - + " (" + appSubmissionContext.getQueue() + ") reject all routing " - + "requests by construction. Application " - + appSubmissionContext.getApplicationId() - + " cannot be routed to any RM."); + protected SubClusterId chooseSubCluster( + String queue, Map preSelectSubclusters) throws YarnException { + throw new FederationPolicyException( + "The policy configured for this queue (" + queue + ") " + + "reject all routing requests by construction. Application in " + + queue + " cannot be routed to any RM."); } - } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/router/UniformRandomRouterPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/router/UniformRandomRouterPolicy.java index 7a8be91fcd0f6..353329613ab97 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/router/UniformRandomRouterPolicy.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/router/UniformRandomRouterPolicy.java @@ -22,11 +22,10 @@ import java.util.Map; import java.util.Random; -import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext; import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.server.federation.policies.FederationPolicyInitializationContext; import org.apache.hadoop.yarn.server.federation.policies.FederationPolicyInitializationContextValidator; -import org.apache.hadoop.yarn.server.federation.policies.FederationPolicyUtils; +import org.apache.hadoop.yarn.server.federation.policies.exceptions.FederationPolicyException; import org.apache.hadoop.yarn.server.federation.policies.exceptions.FederationPolicyInitializationException; import org.apache.hadoop.yarn.server.federation.store.records.SubClusterId; import org.apache.hadoop.yarn.server.federation.store.records.SubClusterInfo; @@ -55,50 +54,16 @@ public void reinitialize(FederationPolicyInitializationContext policyContext) this.getClass().getCanonicalName()); // note: this overrides AbstractRouterPolicy and ignores the weights - setPolicyContext(policyContext); } - /** - * Simply picks a random active subCluster to start the AM (this does NOT - * depend on the weights in the policy). - * - * @param appSubmissionContext the {@link ApplicationSubmissionContext} that - * has to be routed to an appropriate subCluster for execution. - * - * @param blackListSubClusters the list of subClusters as identified by - * {@link SubClusterId} to blackList from the selection of the home - * subCluster. - * - * @return a randomly chosen subcluster. - * - * @throws YarnException if there are no active subclusters. - */ @Override - public SubClusterId getHomeSubcluster( - ApplicationSubmissionContext appSubmissionContext, - List blackListSubClusters) throws YarnException { - - // null checks and default-queue behavior - validate(appSubmissionContext); - - Map activeSubclusters = - getActiveSubclusters(); - - List list = new ArrayList<>(activeSubclusters.keySet()); - - FederationPolicyUtils.validateSubClusterAvailability(list, - blackListSubClusters); - - if (blackListSubClusters != null) { - - // Remove from the active SubClusters from StateStore the blacklisted ones - for (SubClusterId scId : blackListSubClusters) { - list.remove(scId); - } + protected SubClusterId chooseSubCluster( + String queue, Map preSelectSubclusters) throws YarnException { + if (preSelectSubclusters == null || preSelectSubclusters.isEmpty()) { + throw new FederationPolicyException("No available subcluster to choose from."); } - + List list = new ArrayList<>(preSelectSubclusters.keySet()); return list.get(rand.nextInt(list.size())); } - } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/router/WeightedRandomRouterPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/router/WeightedRandomRouterPolicy.java index b1434104836c0..f2acf663603f6 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/router/WeightedRandomRouterPolicy.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/router/WeightedRandomRouterPolicy.java @@ -19,10 +19,8 @@ package org.apache.hadoop.yarn.server.federation.policies.router; import java.util.ArrayList; -import java.util.List; import java.util.Map; -import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext; import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.server.federation.policies.FederationPolicyUtils; import org.apache.hadoop.yarn.server.federation.policies.exceptions.FederationPolicyException; @@ -35,47 +33,30 @@ * sub-clusters. */ public class WeightedRandomRouterPolicy extends AbstractRouterPolicy { - @Override - public SubClusterId getHomeSubcluster( - ApplicationSubmissionContext appSubmissionContext, - List blacklist) throws YarnException { - - // null checks and default-queue behavior - validate(appSubmissionContext); - - Map activeSubclusters = - getActiveSubclusters(); + protected SubClusterId chooseSubCluster( + String queue, Map preSelectSubclusters) throws YarnException { - FederationPolicyUtils.validateSubClusterAvailability( - new ArrayList(activeSubclusters.keySet()), blacklist); - - // note: we cannot pre-compute the weights, as the set of activeSubcluster + // note: we cannot pre-compute the weights, as the set of activeSubCluster // changes dynamically (and this would unfairly spread the load to // sub-clusters adjacent to an inactive one), hence we need to count/scan // the list and based on weight pick the next sub-cluster. - Map weights = - getPolicyInfo().getRouterPolicyWeights(); + Map weights = getPolicyInfo().getRouterPolicyWeights(); ArrayList weightList = new ArrayList<>(); ArrayList scIdList = new ArrayList<>(); for (Map.Entry entry : weights.entrySet()) { - if (blacklist != null && blacklist.contains(entry.getKey().toId())) { - continue; - } - if (entry.getKey() != null - && activeSubclusters.containsKey(entry.getKey().toId())) { + SubClusterIdInfo key = entry.getKey(); + if (key != null && preSelectSubclusters.containsKey(key.toId())) { weightList.add(entry.getValue()); - scIdList.add(entry.getKey().toId()); + scIdList.add(key.toId()); } } int pickedIndex = FederationPolicyUtils.getWeightedRandom(weightList); if (pickedIndex == -1) { - throw new FederationPolicyException( - "No positive weight found on active subclusters"); + throw new FederationPolicyException("No positive weight found on active subclusters"); } return scIdList.get(pickedIndex); } - } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/FederationReservationHomeSubClusterStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/FederationReservationHomeSubClusterStore.java new file mode 100644 index 0000000000000..8c2d5ffe5868f --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/FederationReservationHomeSubClusterStore.java @@ -0,0 +1,119 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package org.apache.hadoop.yarn.server.federation.store; + +import org.apache.hadoop.classification.InterfaceAudience.Private; +import org.apache.hadoop.classification.InterfaceStability.Unstable; +import org.apache.hadoop.yarn.exceptions.YarnException; +import org.apache.hadoop.yarn.server.federation.store.records.GetReservationHomeSubClusterRequest; +import org.apache.hadoop.yarn.server.federation.store.records.GetReservationHomeSubClusterResponse; +import org.apache.hadoop.yarn.server.federation.store.records.GetReservationsHomeSubClusterRequest; +import org.apache.hadoop.yarn.server.federation.store.records.GetReservationsHomeSubClusterResponse; +import org.apache.hadoop.yarn.server.federation.store.records.AddReservationHomeSubClusterResponse; +import org.apache.hadoop.yarn.server.federation.store.records.AddReservationHomeSubClusterRequest; +import org.apache.hadoop.yarn.server.federation.store.records.UpdateReservationHomeSubClusterRequest; +import org.apache.hadoop.yarn.server.federation.store.records.UpdateReservationHomeSubClusterResponse; +import org.apache.hadoop.yarn.server.federation.store.records.DeleteReservationHomeSubClusterRequest; +import org.apache.hadoop.yarn.server.federation.store.records.DeleteReservationHomeSubClusterResponse; + +/** + * FederationReservationHomeSubClusterStore maintains the state of all + * Reservations that have been submitted to the federated cluster. + * + * * + *

    + * The mapping details contains: + *

      + *
    • {@code ReservationId}
      • + *
    • {@code SubClusterId}
      • + *
    + * + */ +@Private +@Unstable +public interface FederationReservationHomeSubClusterStore { + + /** + * Register the home {@code SubClusterId} of the newly submitted + * {@code ReservationId}. Currently response is empty if the operation was + * successful, if not an exception reporting reason for a failure. If a + * mapping for the Reservation already existed, the {@code SubClusterId} in + * this response will return the existing mapping which might be different + * from that in the {@code AddReservationHomeSubClusterRequest}. + * + * @param request the request to register a new Reservation with its home + * sub-cluster + * @return upon successful registration of the Reservation in the StateStore, + * {@code AddReservationHomeSubClusterRequest} containing the home + * sub-cluster of the Reservation. Otherwise, an exception reporting + * reason for a failure + * @throws YarnException if the request is invalid/fails + */ + AddReservationHomeSubClusterResponse addReservationHomeSubCluster( + AddReservationHomeSubClusterRequest request) throws YarnException; + + /** + * Get information about the Reservation identified by the input + * {@code ReservationId}. + * + * @param request contains the Reservation queried + * @return {@code ReservationHomeSubCluster} containing the Reservation's home + * subcluster + * @throws YarnException if the request is invalid/fails + */ + GetReservationHomeSubClusterResponse getReservationHomeSubCluster( + GetReservationHomeSubClusterRequest request) throws YarnException; + + /** + * Get the {@code ReservationHomeSubCluster} list representing the mapping of + * all submitted Reservations to it's home sub-cluster. + * + * @param request empty representing all Reservations + * @return the mapping of all submitted Reservation to it's home sub-cluster + * @throws YarnException if the request is invalid/fails + */ + GetReservationsHomeSubClusterResponse getReservationsHomeSubCluster( + GetReservationsHomeSubClusterRequest request) throws YarnException; + + /** + * Update the home {@code SubClusterId} of a previously submitted + * {@code ReservationId}. Currently response is empty if the operation was + * successful, if not an exception reporting reason for a failure. + * + * @param request the request to update the home sub-cluster of a reservation. + * @return empty on successful update of the Reservation in the StateStore, if + * not an exception reporting reason for a failure + * @throws YarnException if the request is invalid/fails + */ + UpdateReservationHomeSubClusterResponse updateReservationHomeSubCluster( + UpdateReservationHomeSubClusterRequest request) throws YarnException; + + + /** + * Delete the mapping of home {@code SubClusterId} of a previously submitted + * {@code ReservationId}. Currently response is empty if the operation was + * successful, if not an exception reporting reason for a failure. + * + * @param request the request to delete the home sub-cluster of a reservation. + * @return empty on successful update of the Reservation in the StateStore, if + * not an exception reporting reason for a failure + * @throws YarnException if the request is invalid/fails + */ + DeleteReservationHomeSubClusterResponse deleteReservationHomeSubCluster( + DeleteReservationHomeSubClusterRequest request) throws YarnException; +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/FederationStateStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/FederationStateStore.java index 9397e9c240473..67461e6c30b86 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/FederationStateStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/FederationStateStore.java @@ -25,12 +25,13 @@ /** * FederationStore extends the three interfaces used to coordinate the state of * a federated cluster: {@link FederationApplicationHomeSubClusterStore}, - * {@link FederationMembershipStateStore}, and {@link FederationPolicyStore}. + * {@link FederationMembershipStateStore}, {@link FederationPolicyStore}, and + * {@link FederationReservationHomeSubClusterStore}. * */ -public interface FederationStateStore - extends FederationApplicationHomeSubClusterStore, - FederationMembershipStateStore, FederationPolicyStore { +public interface FederationStateStore extends + FederationApplicationHomeSubClusterStore, FederationMembershipStateStore, + FederationPolicyStore, FederationReservationHomeSubClusterStore { /** * Initialize the FederationStore. diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/impl/MemoryFederationStateStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/impl/MemoryFederationStateStore.java index 7c06256a41364..920b8e8912d1e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/impl/MemoryFederationStateStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/impl/MemoryFederationStateStore.java @@ -27,6 +27,7 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.yarn.api.records.ApplicationId; +import org.apache.hadoop.yarn.api.records.ReservationId; import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.server.federation.store.FederationStateStore; import org.apache.hadoop.yarn.server.federation.store.records.AddApplicationHomeSubClusterRequest; @@ -59,7 +60,19 @@ import org.apache.hadoop.yarn.server.federation.store.records.SubClusterRegisterResponse; import org.apache.hadoop.yarn.server.federation.store.records.UpdateApplicationHomeSubClusterRequest; import org.apache.hadoop.yarn.server.federation.store.records.UpdateApplicationHomeSubClusterResponse; +import org.apache.hadoop.yarn.server.federation.store.records.AddReservationHomeSubClusterRequest; +import org.apache.hadoop.yarn.server.federation.store.records.AddReservationHomeSubClusterResponse; +import org.apache.hadoop.yarn.server.federation.store.records.GetReservationHomeSubClusterResponse; +import org.apache.hadoop.yarn.server.federation.store.records.GetReservationHomeSubClusterRequest; +import org.apache.hadoop.yarn.server.federation.store.records.GetReservationsHomeSubClusterResponse; +import org.apache.hadoop.yarn.server.federation.store.records.GetReservationsHomeSubClusterRequest; +import org.apache.hadoop.yarn.server.federation.store.records.ReservationHomeSubCluster; +import org.apache.hadoop.yarn.server.federation.store.records.UpdateReservationHomeSubClusterRequest; +import org.apache.hadoop.yarn.server.federation.store.records.UpdateReservationHomeSubClusterResponse; +import org.apache.hadoop.yarn.server.federation.store.records.DeleteReservationHomeSubClusterRequest; +import org.apache.hadoop.yarn.server.federation.store.records.DeleteReservationHomeSubClusterResponse; import org.apache.hadoop.yarn.server.federation.store.utils.FederationApplicationHomeSubClusterStoreInputValidator; +import org.apache.hadoop.yarn.server.federation.store.utils.FederationReservationHomeSubClusterStoreInputValidator; import org.apache.hadoop.yarn.server.federation.store.utils.FederationMembershipStateStoreInputValidator; import org.apache.hadoop.yarn.server.federation.store.utils.FederationPolicyStoreInputValidator; import org.apache.hadoop.yarn.server.federation.store.utils.FederationStateStoreUtils; @@ -75,6 +88,7 @@ public class MemoryFederationStateStore implements FederationStateStore { private Map membership; private Map applications; + private Map reservations; private Map policies; private final MonotonicClock clock = new MonotonicClock(); @@ -86,6 +100,7 @@ public class MemoryFederationStateStore implements FederationStateStore { public void init(Configuration conf) { membership = new ConcurrentHashMap(); applications = new ConcurrentHashMap(); + reservations = new ConcurrentHashMap(); policies = new ConcurrentHashMap(); } @@ -93,6 +108,7 @@ public void init(Configuration conf) { public void close() { membership = null; applications = null; + reservations = null; policies = null; } @@ -233,8 +249,7 @@ public GetApplicationHomeSubClusterResponse getApplicationHomeSubCluster( FederationStateStoreUtils.logAndThrowStoreException(LOG, errMsg); } - return GetApplicationHomeSubClusterResponse.newInstance( - ApplicationHomeSubCluster.newInstance(appId, applications.get(appId))); + return GetApplicationHomeSubClusterResponse.newInstance(appId, applications.get(appId)); } @Override @@ -312,4 +327,72 @@ public Version loadVersion() { return null; } + @Override + public AddReservationHomeSubClusterResponse addReservationHomeSubCluster( + AddReservationHomeSubClusterRequest request) throws YarnException { + FederationReservationHomeSubClusterStoreInputValidator.validate(request); + ReservationHomeSubCluster homeSubCluster = request.getReservationHomeSubCluster(); + ReservationId reservationId = homeSubCluster.getReservationId(); + if (!reservations.containsKey(reservationId)) { + reservations.put(reservationId, homeSubCluster.getHomeSubCluster()); + } + return AddReservationHomeSubClusterResponse.newInstance(reservations.get(reservationId)); + } + + @Override + public GetReservationHomeSubClusterResponse getReservationHomeSubCluster( + GetReservationHomeSubClusterRequest request) throws YarnException { + FederationReservationHomeSubClusterStoreInputValidator.validate(request); + ReservationId reservationId = request.getReservationId(); + if (!reservations.containsKey(reservationId)) { + throw new YarnException("Reservation " + reservationId + " does not exist"); + } + SubClusterId subClusterId = reservations.get(reservationId); + ReservationHomeSubCluster homeSubCluster = + ReservationHomeSubCluster.newInstance(reservationId, subClusterId); + return GetReservationHomeSubClusterResponse.newInstance(homeSubCluster); + } + + @Override + public GetReservationsHomeSubClusterResponse getReservationsHomeSubCluster( + GetReservationsHomeSubClusterRequest request) throws YarnException { + List result = new ArrayList<>(); + + for (Entry entry : reservations.entrySet()) { + ReservationId reservationId = entry.getKey(); + SubClusterId subClusterId = entry.getValue(); + ReservationHomeSubCluster homeSubCluster = + ReservationHomeSubCluster.newInstance(reservationId, subClusterId); + result.add(homeSubCluster); + } + + return GetReservationsHomeSubClusterResponse.newInstance(result); + } + + @Override + public UpdateReservationHomeSubClusterResponse updateReservationHomeSubCluster( + UpdateReservationHomeSubClusterRequest request) throws YarnException { + FederationReservationHomeSubClusterStoreInputValidator.validate(request); + ReservationId reservationId = request.getReservationHomeSubCluster().getReservationId(); + + if (!reservations.containsKey(reservationId)) { + throw new YarnException("Reservation " + reservationId + " does not exist."); + } + + SubClusterId subClusterId = request.getReservationHomeSubCluster().getHomeSubCluster(); + reservations.put(reservationId, subClusterId); + return UpdateReservationHomeSubClusterResponse.newInstance(); + } + + @Override + public DeleteReservationHomeSubClusterResponse deleteReservationHomeSubCluster( + DeleteReservationHomeSubClusterRequest request) throws YarnException { + FederationReservationHomeSubClusterStoreInputValidator.validate(request); + ReservationId reservationId = request.getReservationId(); + if (!reservations.containsKey(reservationId)) { + throw new YarnException("Reservation " + reservationId + " does not exist"); + } + reservations.remove(reservationId); + return DeleteReservationHomeSubClusterResponse.newInstance(); + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/impl/SQLFederationStateStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/impl/SQLFederationStateStore.java index 2b3fea5609b2e..0c0b5c9e0f6b9 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/impl/SQLFederationStateStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/impl/SQLFederationStateStore.java @@ -30,8 +30,10 @@ import java.util.TimeZone; import org.apache.commons.lang3.NotImplementedException; +import org.apache.commons.lang3.StringUtils; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.yarn.api.records.ApplicationId; +import org.apache.hadoop.yarn.api.records.ReservationId; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.server.federation.store.FederationStateStore; @@ -68,10 +70,22 @@ import org.apache.hadoop.yarn.server.federation.store.records.SubClusterState; import org.apache.hadoop.yarn.server.federation.store.records.UpdateApplicationHomeSubClusterRequest; import org.apache.hadoop.yarn.server.federation.store.records.UpdateApplicationHomeSubClusterResponse; +import org.apache.hadoop.yarn.server.federation.store.records.AddReservationHomeSubClusterRequest; +import org.apache.hadoop.yarn.server.federation.store.records.AddReservationHomeSubClusterResponse; +import org.apache.hadoop.yarn.server.federation.store.records.GetReservationHomeSubClusterResponse; +import org.apache.hadoop.yarn.server.federation.store.records.GetReservationHomeSubClusterRequest; +import org.apache.hadoop.yarn.server.federation.store.records.GetReservationsHomeSubClusterResponse; +import org.apache.hadoop.yarn.server.federation.store.records.GetReservationsHomeSubClusterRequest; +import org.apache.hadoop.yarn.server.federation.store.records.DeleteReservationHomeSubClusterRequest; +import org.apache.hadoop.yarn.server.federation.store.records.DeleteReservationHomeSubClusterResponse; +import org.apache.hadoop.yarn.server.federation.store.records.UpdateReservationHomeSubClusterRequest; +import org.apache.hadoop.yarn.server.federation.store.records.UpdateReservationHomeSubClusterResponse; +import org.apache.hadoop.yarn.server.federation.store.records.ReservationHomeSubCluster; import org.apache.hadoop.yarn.server.federation.store.utils.FederationApplicationHomeSubClusterStoreInputValidator; import org.apache.hadoop.yarn.server.federation.store.utils.FederationMembershipStateStoreInputValidator; import org.apache.hadoop.yarn.server.federation.store.utils.FederationPolicyStoreInputValidator; import org.apache.hadoop.yarn.server.federation.store.utils.FederationStateStoreUtils; +import org.apache.hadoop.yarn.server.federation.store.utils.FederationReservationHomeSubClusterStoreInputValidator; import org.apache.hadoop.yarn.server.records.Version; import org.apache.hadoop.yarn.util.Clock; import org.apache.hadoop.yarn.util.MonotonicClock; @@ -130,6 +144,21 @@ public class SQLFederationStateStore implements FederationStateStore { private static final String CALL_SP_GET_POLICIES_CONFIGURATIONS = "{call sp_getPoliciesConfigurations()}"; + protected static final String CALL_SP_ADD_RESERVATION_HOME_SUBCLUSTER = + "{call sp_addReservationHomeSubCluster(?, ?, ?, ?)}"; + + protected static final String CALL_SP_GET_RESERVATION_HOME_SUBCLUSTER = + "{call sp_getReservationHomeSubCluster(?, ?)}"; + + protected static final String CALL_SP_GET_RESERVATIONS_HOME_SUBCLUSTER = + "{call sp_getReservationsHomeSubCluster()}"; + + protected static final String CALL_SP_DELETE_RESERVATION_HOME_SUBCLUSTER = + "{call sp_deleteReservationHomeSubCluster(?, ?)}"; + + protected static final String CALL_SP_UPDATE_RESERVATION_HOME_SUBCLUSTER = + "{call sp_updateReservationHomeSubCluster(?, ?, ?)}"; + private Calendar utcCalendar = Calendar.getInstance(TimeZone.getTimeZone("UTC")); @@ -710,8 +739,7 @@ public GetApplicationHomeSubClusterResponse getApplicationHomeSubCluster( FederationStateStoreUtils.returnToPool(LOG, cstmt); } return GetApplicationHomeSubClusterResponse - .newInstance(ApplicationHomeSubCluster - .newInstance(request.getApplicationId(), homeRM)); + .newInstance(request.getApplicationId(), homeRM); } @Override @@ -988,12 +1016,14 @@ public void close() throws Exception { * @return a connection from the DataSource pool. * @throws SQLException on failure */ - public Connection getConnection() throws SQLException { + @VisibleForTesting + protected Connection getConnection() throws SQLException { FederationStateStoreClientMetrics.incrConnections(); return dataSource.getConnection(); } - private CallableStatement getCallableStatement(String procedure) + @VisibleForTesting + protected CallableStatement getCallableStatement(String procedure) throws SQLException { return conn.prepareCall(procedure); } @@ -1004,4 +1034,355 @@ private static byte[] getByteArray(ByteBuffer bb) { return ba; } + @Override + public AddReservationHomeSubClusterResponse addReservationHomeSubCluster( + AddReservationHomeSubClusterRequest request) throws YarnException { + + // validate + FederationReservationHomeSubClusterStoreInputValidator.validate(request); + CallableStatement cstmt = null; + + ReservationHomeSubCluster reservationHomeSubCluster = request.getReservationHomeSubCluster(); + ReservationId reservationId = reservationHomeSubCluster.getReservationId(); + SubClusterId subClusterId = reservationHomeSubCluster.getHomeSubCluster(); + SubClusterId subClusterHomeId = null; + + try { + + // Defined the sp_addReservationHomeSubCluster procedure + // this procedure requires 4 parameters + // Input parameters + // 1）IN reservationId_IN varchar(128) + // 2）IN homeSubCluster_IN varchar(256) + // Output parameters + // 3）OUT storedHomeSubCluster_OUT varchar(256) + // 4）OUT rowCount_OUT int + + // Call procedure + cstmt = getCallableStatement(CALL_SP_ADD_RESERVATION_HOME_SUBCLUSTER); + + // Set the parameters for the stored procedure + // 1）IN reservationId_IN varchar(128) + cstmt.setString("reservationId_IN", reservationId.toString()); + // 2）IN homeSubCluster_IN varchar(256) + cstmt.setString("homeSubCluster_IN", subClusterId.getId()); + // 3) OUT storedHomeSubCluster_OUT varchar(256) + cstmt.registerOutParameter("storedHomeSubCluster_OUT", java.sql.Types.VARCHAR); + // 4) OUT rowCount_OUT int + cstmt.registerOutParameter("rowCount_OUT", java.sql.Types.INTEGER); + + // Execute the query + long startTime = clock.getTime(); + cstmt.executeUpdate(); + long stopTime = clock.getTime(); + + // Get SubClusterHome + String subClusterHomeIdString = cstmt.getString("storedHomeSubCluster_OUT"); + subClusterHomeId = SubClusterId.newInstance(subClusterHomeIdString); + + // Get rowCount + int rowCount = cstmt.getInt("rowCount_OUT"); + + // For failover reason, we check the returned subClusterId. + // 1.If it is equal to the subClusterId we sent, the call added the new + // reservation into FederationStateStore. + // 2.If the call returns a different subClusterId + // it means we already tried to insert this reservation + // but a component (Router/StateStore/RM) failed during the submission. + if (subClusterId.equals(subClusterHomeId)) { + // if it is equal to 0 + // it means the call did not add a new reservation into FederationStateStore. + if (rowCount == 0) { + LOG.info("The reservation {} was not inserted in the StateStore because it" + + " was already present in subCluster {}", reservationId, subClusterHomeId); + } else if (rowCount != 1) { + // if it is different from 1 + // it means the call had a wrong behavior. Maybe the database is not set correctly. + FederationStateStoreUtils.logAndThrowStoreException(LOG, + "Wrong behavior during the insertion of subCluster %s according to reservation %s. " + + "The database expects to insert 1 record, but the number of " + + "inserted changes is greater than 1, " + + "please check the records of the database.", + subClusterId, reservationId); + } + } else { + // If it is different from 0, + // it means that there is a data situation that does not meet the expectations, + // and an exception should be thrown at this time + if (rowCount != 0) { + FederationStateStoreUtils.logAndThrowStoreException(LOG, + "The reservation %s does exist but was overwritten.", reservationId); + } + LOG.info("Reservation: {} already present with subCluster: {}.", + reservationId, subClusterHomeId); + } + + // Record successful call time + FederationStateStoreClientMetrics.succeededStateStoreCall(stopTime - startTime); + } catch (SQLException e) { + FederationStateStoreClientMetrics.failedStateStoreCall(); + FederationStateStoreUtils.logAndThrowRetriableException(e, LOG, + "Unable to insert the newly generated reservation %s to subCluster %s.", + reservationId, subClusterId); + } finally { + // Return to the pool the CallableStatement + FederationStateStoreUtils.returnToPool(LOG, cstmt); + } + + return AddReservationHomeSubClusterResponse.newInstance(subClusterHomeId); + } + + @Override + public GetReservationHomeSubClusterResponse getReservationHomeSubCluster( + GetReservationHomeSubClusterRequest request) throws YarnException { + // validate + FederationReservationHomeSubClusterStoreInputValidator.validate(request); + + CallableStatement cstmt = null; + ReservationId reservationId = request.getReservationId(); + SubClusterId subClusterId = null; + + try { + + // Defined the sp_getReservationHomeSubCluster procedure + // this procedure requires 2 parameters + // Input parameters + // 1）IN reservationId_IN varchar(128) + // Output parameters + // 2）OUT homeSubCluster_OUT varchar(256) + + cstmt = getCallableStatement(CALL_SP_GET_RESERVATION_HOME_SUBCLUSTER); + + // Set the parameters for the stored procedure + // 1）IN reservationId_IN varchar(128) + cstmt.setString("reservationId_IN", reservationId.toString()); + // 2）OUT homeSubCluster_OUT varchar(256) + cstmt.registerOutParameter("homeSubCluster_OUT", java.sql.Types.VARCHAR); + + // Execute the query + long startTime = clock.getTime(); + cstmt.execute(); + long stopTime = clock.getTime(); + + // Get Result + String subClusterHomeIdString = cstmt.getString("homeSubCluster_OUT"); + + if (StringUtils.isNotBlank(subClusterHomeIdString)) { + subClusterId = SubClusterId.newInstance(subClusterHomeIdString); + } else { + // If subClusterHomeIdString blank, we need to throw an exception + FederationStateStoreUtils.logAndThrowRetriableException(LOG, + "Reservation %s does not exist", reservationId); + } + + LOG.info("Got the information about the specified reservation {} in subCluster = {}.", + reservationId, subClusterId); + + FederationStateStoreClientMetrics.succeededStateStoreCall(stopTime - startTime); + + ReservationHomeSubCluster homeSubCluster = + ReservationHomeSubCluster.newInstance(reservationId, subClusterId); + return GetReservationHomeSubClusterResponse.newInstance(homeSubCluster); + + } catch (SQLException e) { + FederationStateStoreClientMetrics.failedStateStoreCall(); + FederationStateStoreUtils.logAndThrowRetriableException(e, LOG, + "Unable to obtain the reservation information according to %s.", reservationId); + } finally { + // Return to the pool the CallableStatement + FederationStateStoreUtils.returnToPool(LOG, cstmt); + } + + throw new YarnException( + "Unable to obtain the reservation information according to " + reservationId); + } + + @Override + public GetReservationsHomeSubClusterResponse getReservationsHomeSubCluster( + GetReservationsHomeSubClusterRequest request) throws YarnException { + CallableStatement cstmt = null; + ResultSet rs = null; + List reservationsHomeSubClusters = new ArrayList<>(); + + try { + + // Defined the sp_getReservationsHomeSubCluster procedure + // This procedure requires no input parameters, but will have 2 output parameters + // Output parameters + // 1）OUT reservationId + // 2）OUT homeSubCluster + + cstmt = getCallableStatement(CALL_SP_GET_RESERVATIONS_HOME_SUBCLUSTER); + + // Execute the query + long startTime = clock.getTime(); + rs = cstmt.executeQuery(); + long stopTime = clock.getTime(); + + while (rs.next()) { + // Extract the output for each tuple + // 1）OUT reservationId + String dbReservationId = rs.getString("reservationId"); + // 2）OUT homeSubCluster + String dbHomeSubCluster = rs.getString("homeSubCluster"); + + // Generate parameters + ReservationId reservationId = ReservationId.parseReservationId(dbReservationId); + SubClusterId homeSubCluster = SubClusterId.newInstance(dbHomeSubCluster); + ReservationHomeSubCluster reservationHomeSubCluster = + ReservationHomeSubCluster.newInstance(reservationId, homeSubCluster); + reservationsHomeSubClusters.add(reservationHomeSubCluster); + } + + FederationStateStoreClientMetrics.succeededStateStoreCall(stopTime - startTime); + + return GetReservationsHomeSubClusterResponse.newInstance( + reservationsHomeSubClusters); + } catch (Exception e) { + FederationStateStoreClientMetrics.failedStateStoreCall(); + FederationStateStoreUtils.logAndThrowRetriableException(LOG, + "Unable to obtain the information for all the reservations.", e); + } finally { + // Return to the pool the CallableStatement + FederationStateStoreUtils.returnToPool(LOG, cstmt, null, rs); + } + + throw new YarnException("Unable to obtain the information for all the reservations."); + } + + @Override + public DeleteReservationHomeSubClusterResponse deleteReservationHomeSubCluster( + DeleteReservationHomeSubClusterRequest request) throws YarnException { + + // validate + FederationReservationHomeSubClusterStoreInputValidator.validate(request); + + CallableStatement cstmt = null; + ReservationId reservationId = request.getReservationId(); + + try { + + // Defined the sp_deleteReservationHomeSubCluster procedure + // This procedure requires 1 input parameters, 1 output parameters + // Input parameters + // 1）IN reservationId_IN varchar(128) + // Output parameters + // 2）OUT rowCount_OUT int + + cstmt = getCallableStatement(CALL_SP_DELETE_RESERVATION_HOME_SUBCLUSTER); + + // Set the parameters for the stored procedure + // 1）IN reservationId_IN varchar(128) + cstmt.setString("reservationId_IN", reservationId.toString()); + // 2）OUT rowCount_OUT int + cstmt.registerOutParameter("rowCount_OUT", java.sql.Types.INTEGER); + + // Execute the query + long startTime = clock.getTime(); + cstmt.executeUpdate(); + long stopTime = clock.getTime(); + + int rowCount = cstmt.getInt("rowCount_OUT"); + + // if it is equal to 0 it means the call + // did not delete the reservation from FederationStateStore + if (rowCount == 0) { + FederationStateStoreUtils.logAndThrowStoreException(LOG, + "Reservation %s does not exist", reservationId); + } else if (rowCount != 1) { + // if it is different from 1 it means the call + // had a wrong behavior. Maybe the database is not set correctly. + FederationStateStoreUtils.logAndThrowStoreException(LOG, + "Wrong behavior during deleting the reservation %s. " + + "The database is expected to delete 1 record, " + + "but the number of deleted records returned by the database is greater than 1, " + + "indicating that a duplicate reservationId occurred during the deletion process.", + reservationId); + } + + LOG.info("Delete from the StateStore the reservation: {}.", reservationId); + FederationStateStoreClientMetrics.succeededStateStoreCall(stopTime - startTime); + return DeleteReservationHomeSubClusterResponse.newInstance(); + } catch (SQLException e) { + FederationStateStoreClientMetrics.failedStateStoreCall(); + FederationStateStoreUtils.logAndThrowRetriableException(e, LOG, + "Unable to delete the reservation %s.", reservationId); + } finally { + // Return to the pool the CallableStatement + FederationStateStoreUtils.returnToPool(LOG, cstmt); + } + throw new YarnException("Unable to delete the reservation " + reservationId); + } + + @Override + public UpdateReservationHomeSubClusterResponse updateReservationHomeSubCluster( + UpdateReservationHomeSubClusterRequest request) throws YarnException { + + // validate + FederationReservationHomeSubClusterStoreInputValidator.validate(request); + + CallableStatement cstmt = null; + ReservationHomeSubCluster reservationHomeSubCluster = request.getReservationHomeSubCluster(); + ReservationId reservationId = reservationHomeSubCluster.getReservationId(); + SubClusterId subClusterId = reservationHomeSubCluster.getHomeSubCluster(); + + try { + + // Defined the sp_updateReservationHomeSubCluster procedure + // This procedure requires 2 input parameters, 1 output parameters + // Input parameters + // 1）IN reservationId_IN varchar(128) + // 2）IN homeSubCluster_IN varchar(256) + // Output parameters + // 3）OUT rowCount_OUT int + + cstmt = getCallableStatement(CALL_SP_UPDATE_RESERVATION_HOME_SUBCLUSTER); + + // Set the parameters for the stored procedure + // 1）IN reservationId_IN varchar(128) + cstmt.setString("reservationId_IN", reservationId.toString()); + // 2）IN homeSubCluster_IN varchar(256) + cstmt.setString("homeSubCluster_IN", subClusterId.getId()); + // 3）OUT rowCount_OUT int + cstmt.registerOutParameter("rowCount_OUT", java.sql.Types.INTEGER); + + // Execute the query + long startTime = clock.getTime(); + cstmt.executeUpdate(); + long stopTime = clock.getTime(); + + int rowCount = cstmt.getInt("rowCount_OUT"); + + // if it is equal to 0 it means the call + // did not update the reservation into FederationStateStore + if (rowCount == 0) { + FederationStateStoreUtils.logAndThrowStoreException(LOG, + "Reservation %s does not exist", reservationId); + } else if (rowCount != 1) { + // if it is different from 1 it means the call + // had a wrong behavior. Maybe the database is not set correctly. + FederationStateStoreUtils.logAndThrowStoreException(LOG, + "Wrong behavior during update the subCluster %s according to reservation %s. " + + "The database is expected to update 1 record, " + + "but the number of database update records is greater than 1, " + + "the records of the database should be checked.", + subClusterId, reservationId); + } + LOG.info("Update the subCluster to {} for reservation {} in the StateStore.", + subClusterId, reservationId); + FederationStateStoreClientMetrics.succeededStateStoreCall(stopTime - startTime); + return UpdateReservationHomeSubClusterResponse.newInstance(); + } catch (SQLException e) { + FederationStateStoreClientMetrics.failedStateStoreCall(); + FederationStateStoreUtils.logAndThrowRetriableException(e, LOG, + "Unable to update the subCluster %s according to reservation %s.", + subClusterId, reservationId); + } finally { + // Return to the pool the CallableStatement + FederationStateStoreUtils.returnToPool(LOG, cstmt); + } + throw new YarnException( + "Unable to update the subCluster " + subClusterId + + " according to reservation" + reservationId); + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/impl/ZKFederationStateStoreOpDurations.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/impl/ZKFederationStateStoreOpDurations.java new file mode 100644 index 0000000000000..113e4850a5709 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/impl/ZKFederationStateStoreOpDurations.java @@ -0,0 +1,190 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package org.apache.hadoop.yarn.server.federation.store.impl; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.metrics2.MetricsCollector; +import org.apache.hadoop.metrics2.MetricsInfo; +import org.apache.hadoop.metrics2.MetricsSource; +import org.apache.hadoop.metrics2.MetricsSystem; +import org.apache.hadoop.metrics2.annotation.Metric; +import org.apache.hadoop.metrics2.annotation.Metrics; +import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem; +import org.apache.hadoop.metrics2.lib.MetricsRegistry; +import org.apache.hadoop.metrics2.lib.MutableRate; + +import static org.apache.hadoop.metrics2.lib.Interns.info; + +@InterfaceAudience.Private +@InterfaceStability.Unstable +@Metrics(context="ZKFederationStateStore-op-durations") +public final class ZKFederationStateStoreOpDurations implements MetricsSource { + + @Metric("Duration for a add application homeSubcluster call") + private MutableRate addAppHomeSubCluster; + + @Metric("Duration for a update application homeSubcluster call") + private MutableRate updateAppHomeSubCluster; + + @Metric("Duration for a get application homeSubcluster call") + private MutableRate getAppHomeSubCluster; + + @Metric("Duration for a get applications homeSubcluster call") + private MutableRate getAppsHomeSubCluster; + + @Metric("Duration for a delete applications homeSubcluster call") + private MutableRate deleteAppHomeSubCluster; + + @Metric("Duration for a register subCluster call") + private MutableRate registerSubCluster; + + @Metric("Duration for a deregister subCluster call") + private MutableRate deregisterSubCluster; + + @Metric("Duration for a subCluster Heartbeat call") + private MutableRate subClusterHeartbeat; + + @Metric("Duration for a get SubCluster call") + private MutableRate getSubCluster; + + @Metric("Duration for a get SubClusters call") + private MutableRate getSubClusters; + + @Metric("Duration for a get PolicyConfiguration call") + private MutableRate getPolicyConfiguration; + + @Metric("Duration for a set PolicyConfiguration call") + private MutableRate setPolicyConfiguration; + + @Metric("Duration for a get PolicyConfigurations call") + private MutableRate getPoliciesConfigurations; + + @Metric("Duration for a add reservation homeSubCluster call") + private MutableRate addReservationHomeSubCluster; + + @Metric("Duration for a get reservation homeSubCluster call") + private MutableRate getReservationHomeSubCluster; + + @Metric("Duration for a get reservations homeSubCluster call") + private MutableRate getReservationsHomeSubCluster; + + @Metric("Duration for a delete reservation homeSubCluster call") + private MutableRate deleteReservationHomeSubCluster; + + @Metric("Duration for a update reservation homeSubCluster call") + private MutableRate updateReservationHomeSubCluster; + + protected static final MetricsInfo RECORD_INFO = + info("ZKFederationStateStoreOpDurations", "Durations of ZKFederationStateStore calls"); + + private final MetricsRegistry registry; + + private static final ZKFederationStateStoreOpDurations INSTANCE = + new ZKFederationStateStoreOpDurations(); + + public static ZKFederationStateStoreOpDurations getInstance() { + return INSTANCE; + } + + private ZKFederationStateStoreOpDurations() { + registry = new MetricsRegistry(RECORD_INFO); + registry.tag(RECORD_INFO, "ZKFederationStateStoreOpDurations"); + + MetricsSystem ms = DefaultMetricsSystem.instance(); + if (ms != null) { + ms.register(RECORD_INFO.name(), RECORD_INFO.description(), this); + } + } + + @Override + public synchronized void getMetrics(MetricsCollector collector, boolean all) { + registry.snapshot(collector.addRecord(registry.info()), all); + } + + public void addAppHomeSubClusterDuration(long startTime, long endTime) { + addAppHomeSubCluster.add(endTime - startTime); + } + + public void addUpdateAppHomeSubClusterDuration(long startTime, long endTime) { + updateAppHomeSubCluster.add(endTime - startTime); + } + + public void addGetAppHomeSubClusterDuration(long startTime, long endTime) { + getAppHomeSubCluster.add(endTime - startTime); + } + + public void addGetAppsHomeSubClusterDuration(long startTime, long endTime) { + getAppsHomeSubCluster.add(endTime - startTime); + } + + public void addDeleteAppHomeSubClusterDuration(long startTime, long endTime) { + deleteAppHomeSubCluster.add(endTime - startTime); + } + + public void addRegisterSubClusterDuration(long startTime, long endTime) { + registerSubCluster.add(endTime - startTime); + } + + public void addDeregisterSubClusterDuration(long startTime, long endTime) { + deregisterSubCluster.add(endTime - startTime); + } + + public void addSubClusterHeartbeatDuration(long startTime, long endTime) { + subClusterHeartbeat.add(endTime - startTime); + } + + public void addGetSubClusterDuration(long startTime, long endTime) { + getSubCluster.add(endTime - startTime); + } + + public void addGetSubClustersDuration(long startTime, long endTime) { + getSubClusters.add(endTime - startTime); + } + + public void addGetPolicyConfigurationDuration(long startTime, long endTime) { + getPolicyConfiguration.add(endTime - startTime); + } + + public void addSetPolicyConfigurationDuration(long startTime, long endTime) { + setPolicyConfiguration.add(endTime - startTime); + } + + public void addGetPoliciesConfigurationsDuration(long startTime, long endTime) { + getPoliciesConfigurations.add(endTime - startTime); + } + + public void addReservationHomeSubClusterDuration(long startTime, long endTime) { + addReservationHomeSubCluster.add(endTime - startTime); + } + + public void addGetReservationHomeSubClusterDuration(long startTime, long endTime) { + getReservationHomeSubCluster.add(endTime - startTime); + } + + public void addGetReservationsHomeSubClusterDuration(long startTime, long endTime) { + getReservationsHomeSubCluster.add(endTime - startTime); + } + + public void addDeleteReservationHomeSubClusterDuration(long startTime, long endTime) { + deleteReservationHomeSubCluster.add(endTime - startTime); + } + + public void addUpdateReservationHomeSubClusterDuration(long startTime, long endTime) { + updateReservationHomeSubCluster.add(endTime - startTime); + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/impl/ZookeeperFederationStateStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/impl/ZookeeperFederationStateStore.java index c9b5849ad68d2..d710dacd54087 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/impl/ZookeeperFederationStateStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/impl/ZookeeperFederationStateStore.java @@ -25,6 +25,7 @@ import java.util.List; import java.util.TimeZone; +import org.apache.hadoop.classification.VisibleForTesting; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.util.curator.ZKCuratorManager; import org.apache.hadoop.yarn.api.records.ApplicationId; @@ -63,8 +64,19 @@ import org.apache.hadoop.yarn.server.federation.store.records.SubClusterRegisterRequest; import org.apache.hadoop.yarn.server.federation.store.records.SubClusterRegisterResponse; import org.apache.hadoop.yarn.server.federation.store.records.SubClusterState; +import org.apache.hadoop.yarn.server.federation.store.records.ReservationHomeSubCluster; import org.apache.hadoop.yarn.server.federation.store.records.UpdateApplicationHomeSubClusterRequest; import org.apache.hadoop.yarn.server.federation.store.records.UpdateApplicationHomeSubClusterResponse; +import org.apache.hadoop.yarn.server.federation.store.records.AddReservationHomeSubClusterRequest; +import org.apache.hadoop.yarn.server.federation.store.records.AddReservationHomeSubClusterResponse; +import org.apache.hadoop.yarn.server.federation.store.records.GetReservationHomeSubClusterResponse; +import org.apache.hadoop.yarn.server.federation.store.records.GetReservationHomeSubClusterRequest; +import org.apache.hadoop.yarn.server.federation.store.records.GetReservationsHomeSubClusterResponse; +import org.apache.hadoop.yarn.server.federation.store.records.GetReservationsHomeSubClusterRequest; +import org.apache.hadoop.yarn.server.federation.store.records.DeleteReservationHomeSubClusterRequest; +import org.apache.hadoop.yarn.server.federation.store.records.DeleteReservationHomeSubClusterResponse; +import org.apache.hadoop.yarn.server.federation.store.records.UpdateReservationHomeSubClusterRequest; +import org.apache.hadoop.yarn.server.federation.store.records.UpdateReservationHomeSubClusterResponse; import org.apache.hadoop.yarn.server.federation.store.records.impl.pb.SubClusterIdPBImpl; import org.apache.hadoop.yarn.server.federation.store.records.impl.pb.SubClusterInfoPBImpl; import org.apache.hadoop.yarn.server.federation.store.records.impl.pb.SubClusterPolicyConfigurationPBImpl; @@ -72,7 +84,11 @@ import org.apache.hadoop.yarn.server.federation.store.utils.FederationMembershipStateStoreInputValidator; import org.apache.hadoop.yarn.server.federation.store.utils.FederationPolicyStoreInputValidator; import org.apache.hadoop.yarn.server.federation.store.utils.FederationStateStoreUtils; +import org.apache.hadoop.yarn.server.federation.store.utils.FederationReservationHomeSubClusterStoreInputValidator; import org.apache.hadoop.yarn.server.records.Version; +import org.apache.hadoop.yarn.api.records.ReservationId; +import org.apache.hadoop.yarn.util.Clock; +import org.apache.hadoop.yarn.util.SystemClock; import org.apache.zookeeper.data.ACL; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -91,8 +107,11 @@ * | |----- APP1 * | |----- APP2 * |--- POLICY - * |----- QUEUE1 - * |----- QUEUE1 + * | |----- QUEUE1 + * | |----- QUEUE1 + * |--- RESERVATION + * | |----- RESERVATION1 + * | |----- RESERVATION2 */ public class ZookeeperFederationStateStore implements FederationStateStore { @@ -102,6 +121,7 @@ public class ZookeeperFederationStateStore implements FederationStateStore { private final static String ROOT_ZNODE_NAME_MEMBERSHIP = "memberships"; private final static String ROOT_ZNODE_NAME_APPLICATION = "applications"; private final static String ROOT_ZNODE_NAME_POLICY = "policies"; + private final static String ROOT_ZNODE_NAME_RESERVATION = "reservation"; /** Interface to Zookeeper. */ private ZKCuratorManager zkManager; @@ -112,6 +132,13 @@ public class ZookeeperFederationStateStore implements FederationStateStore { private String appsZNode; private String membershipZNode; private String policiesZNode; + private String reservationsZNode; + + private volatile Clock clock = SystemClock.getInstance(); + + @VisibleForTesting + private ZKFederationStateStoreOpDurations opDurations = + ZKFederationStateStoreOpDurations.getInstance(); @Override public void init(Configuration conf) throws YarnException { @@ -131,6 +158,7 @@ public void init(Configuration conf) throws YarnException { membershipZNode = getNodePath(baseZNode, ROOT_ZNODE_NAME_MEMBERSHIP); appsZNode = getNodePath(baseZNode, ROOT_ZNODE_NAME_APPLICATION); policiesZNode = getNodePath(baseZNode, ROOT_ZNODE_NAME_POLICY); + reservationsZNode = getNodePath(baseZNode, ROOT_ZNODE_NAME_RESERVATION); // Create base znode for each entity try { @@ -138,11 +166,11 @@ public void init(Configuration conf) throws YarnException { zkManager.createRootDirRecursively(membershipZNode, zkAcl); zkManager.createRootDirRecursively(appsZNode, zkAcl); zkManager.createRootDirRecursively(policiesZNode, zkAcl); + zkManager.createRootDirRecursively(reservationsZNode, zkAcl); } catch (Exception e) { String errMsg = "Cannot create base directories: " + e.getMessage(); FederationStateStoreUtils.logAndThrowStoreException(LOG, errMsg); } - } @Override @@ -156,6 +184,7 @@ public void close() throws Exception { public AddApplicationHomeSubClusterResponse addApplicationHomeSubCluster( AddApplicationHomeSubClusterRequest request) throws YarnException { + long start = clock.getTime(); FederationApplicationHomeSubClusterStoreInputValidator.validate(request); ApplicationHomeSubCluster app = request.getApplicationHomeSubCluster(); ApplicationId appId = app.getApplicationId(); @@ -176,7 +205,8 @@ public AddApplicationHomeSubClusterResponse addApplicationHomeSubCluster( String errMsg = "Cannot check app home subcluster for " + appId; FederationStateStoreUtils.logAndThrowStoreException(LOG, errMsg); } - + long end = clock.getTime(); + opDurations.addAppHomeSubClusterDuration(start, end); return AddApplicationHomeSubClusterResponse .newInstance(homeSubCluster); } @@ -187,6 +217,7 @@ public AddApplicationHomeSubClusterResponse addApplicationHomeSubCluster( UpdateApplicationHomeSubClusterRequest request) throws YarnException { + long start = clock.getTime(); FederationApplicationHomeSubClusterStoreInputValidator.validate(request); ApplicationHomeSubCluster app = request.getApplicationHomeSubCluster(); ApplicationId appId = app.getApplicationId(); @@ -198,6 +229,9 @@ public AddApplicationHomeSubClusterResponse addApplicationHomeSubCluster( SubClusterId newSubClusterId = request.getApplicationHomeSubCluster().getHomeSubCluster(); putApp(appId, newSubClusterId, true); + + long end = clock.getTime(); + opDurations.addUpdateAppHomeSubClusterDuration(start, end); return UpdateApplicationHomeSubClusterResponse.newInstance(); } @@ -205,6 +239,7 @@ public AddApplicationHomeSubClusterResponse addApplicationHomeSubCluster( public GetApplicationHomeSubClusterResponse getApplicationHomeSubCluster( GetApplicationHomeSubClusterRequest request) throws YarnException { + long start = clock.getTime(); FederationApplicationHomeSubClusterStoreInputValidator.validate(request); ApplicationId appId = request.getApplicationId(); SubClusterId homeSubCluster = getApp(appId); @@ -212,13 +247,15 @@ public GetApplicationHomeSubClusterResponse getApplicationHomeSubCluster( String errMsg = "Application " + appId + " does not exist"; FederationStateStoreUtils.logAndThrowStoreException(LOG, errMsg); } - return GetApplicationHomeSubClusterResponse.newInstance( - ApplicationHomeSubCluster.newInstance(appId, homeSubCluster)); + long end = clock.getTime(); + opDurations.addGetAppHomeSubClusterDuration(start, end); + return GetApplicationHomeSubClusterResponse.newInstance(appId, homeSubCluster); } @Override public GetApplicationsHomeSubClusterResponse getApplicationsHomeSubCluster( GetApplicationsHomeSubClusterRequest request) throws YarnException { + long start = clock.getTime(); List result = new ArrayList<>(); try { @@ -233,7 +270,8 @@ public GetApplicationsHomeSubClusterResponse getApplicationsHomeSubCluster( String errMsg = "Cannot get apps: " + e.getMessage(); FederationStateStoreUtils.logAndThrowStoreException(LOG, errMsg); } - + long end = clock.getTime(); + opDurations.addGetAppsHomeSubClusterDuration(start, end); return GetApplicationsHomeSubClusterResponse.newInstance(result); } @@ -242,7 +280,7 @@ public GetApplicationsHomeSubClusterResponse getApplicationsHomeSubCluster( deleteApplicationHomeSubCluster( DeleteApplicationHomeSubClusterRequest request) throws YarnException { - + long start = clock.getTime(); FederationApplicationHomeSubClusterStoreInputValidator.validate(request); ApplicationId appId = request.getApplicationId(); String appZNode = getNodePath(appsZNode, appId.toString()); @@ -265,13 +303,15 @@ public GetApplicationsHomeSubClusterResponse getApplicationsHomeSubCluster( String errMsg = "Cannot delete app: " + e.getMessage(); FederationStateStoreUtils.logAndThrowStoreException(LOG, errMsg); } - + long end = clock.getTime(); + opDurations.addDeleteAppHomeSubClusterDuration(start, end); return DeleteApplicationHomeSubClusterResponse.newInstance(); } @Override public SubClusterRegisterResponse registerSubCluster( SubClusterRegisterRequest request) throws YarnException { + long start = clock.getTime(); FederationMembershipStateStoreInputValidator.validate(request); SubClusterInfo subClusterInfo = request.getSubClusterInfo(); SubClusterId subclusterId = subClusterInfo.getSubClusterId(); @@ -286,12 +326,15 @@ public SubClusterRegisterResponse registerSubCluster( String errMsg = "Cannot register subcluster: " + e.getMessage(); FederationStateStoreUtils.logAndThrowStoreException(LOG, errMsg); } + long end = clock.getTime(); + opDurations.addRegisterSubClusterDuration(start, end); return SubClusterRegisterResponse.newInstance(); } @Override public SubClusterDeregisterResponse deregisterSubCluster( SubClusterDeregisterRequest request) throws YarnException { + long start = clock.getTime(); FederationMembershipStateStoreInputValidator.validate(request); SubClusterId subClusterId = request.getSubClusterId(); SubClusterState state = request.getState(); @@ -305,14 +348,15 @@ public SubClusterDeregisterResponse deregisterSubCluster( subClusterInfo.setState(state); putSubclusterInfo(subClusterId, subClusterInfo, true); } - + long end = clock.getTime(); + opDurations.addDeregisterSubClusterDuration(start, end); return SubClusterDeregisterResponse.newInstance(); } @Override public SubClusterHeartbeatResponse subClusterHeartbeat( SubClusterHeartbeatRequest request) throws YarnException { - + long start = clock.getTime(); FederationMembershipStateStoreInputValidator.validate(request); SubClusterId subClusterId = request.getSubClusterId(); @@ -329,14 +373,15 @@ public SubClusterHeartbeatResponse subClusterHeartbeat( subClusterInfo.setCapability(request.getCapability()); putSubclusterInfo(subClusterId, subClusterInfo, true); - + long end = clock.getTime(); + opDurations.addSubClusterHeartbeatDuration(start, end); return SubClusterHeartbeatResponse.newInstance(); } @Override public GetSubClusterInfoResponse getSubCluster( GetSubClusterInfoRequest request) throws YarnException { - + long start = clock.getTime(); FederationMembershipStateStoreInputValidator.validate(request); SubClusterId subClusterId = request.getSubClusterId(); SubClusterInfo subClusterInfo = null; @@ -350,12 +395,15 @@ public GetSubClusterInfoResponse getSubCluster( String errMsg = "Cannot get subcluster: " + e.getMessage(); FederationStateStoreUtils.logAndThrowStoreException(LOG, errMsg); } + long end = clock.getTime(); + opDurations.addGetSubClusterDuration(start, end); return GetSubClusterInfoResponse.newInstance(subClusterInfo); } @Override public GetSubClustersInfoResponse getSubClusters( GetSubClustersInfoRequest request) throws YarnException { + long start = clock.getTime(); List result = new ArrayList<>(); try { @@ -371,6 +419,8 @@ public GetSubClustersInfoResponse getSubClusters( String errMsg = "Cannot get subclusters: " + e.getMessage(); FederationStateStoreUtils.logAndThrowStoreException(LOG, errMsg); } + long end = clock.getTime(); + opDurations.addGetSubClustersDuration(start, end); return GetSubClustersInfoResponse.newInstance(result); } @@ -378,7 +428,7 @@ public GetSubClustersInfoResponse getSubClusters( @Override public GetSubClusterPolicyConfigurationResponse getPolicyConfiguration( GetSubClusterPolicyConfigurationRequest request) throws YarnException { - + long start = clock.getTime(); FederationPolicyStoreInputValidator.validate(request); String queue = request.getQueue(); SubClusterPolicyConfiguration policy = null; @@ -393,6 +443,8 @@ public GetSubClusterPolicyConfigurationResponse getPolicyConfiguration( LOG.warn("Policy for queue: {} does not exist.", queue); return null; } + long end = clock.getTime(); + opDurations.addGetPolicyConfigurationDuration(start, end); return GetSubClusterPolicyConfigurationResponse .newInstance(policy); } @@ -400,7 +452,7 @@ public GetSubClusterPolicyConfigurationResponse getPolicyConfiguration( @Override public SetSubClusterPolicyConfigurationResponse setPolicyConfiguration( SetSubClusterPolicyConfigurationRequest request) throws YarnException { - + long start = clock.getTime(); FederationPolicyStoreInputValidator.validate(request); SubClusterPolicyConfiguration policy = request.getPolicyConfiguration(); @@ -411,12 +463,15 @@ public SetSubClusterPolicyConfigurationResponse setPolicyConfiguration( String errMsg = "Cannot set policy: " + e.getMessage(); FederationStateStoreUtils.logAndThrowStoreException(LOG, errMsg); } + long end = clock.getTime(); + opDurations.addSetPolicyConfigurationDuration(start, end); return SetSubClusterPolicyConfigurationResponse.newInstance(); } @Override public GetSubClusterPoliciesConfigurationsResponse getPoliciesConfigurations( GetSubClusterPoliciesConfigurationsRequest request) throws YarnException { + long start = clock.getTime(); List result = new ArrayList<>(); try { @@ -432,6 +487,8 @@ public GetSubClusterPoliciesConfigurationsResponse getPoliciesConfigurations( String errMsg = "Cannot get policies: " + e.getMessage(); FederationStateStoreUtils.logAndThrowStoreException(LOG, errMsg); } + long end = clock.getTime(); + opDurations.addGetPoliciesConfigurationsDuration(start, end); return GetSubClusterPoliciesConfigurationsResponse.newInstance(result); } @@ -637,4 +694,161 @@ private static long getCurrentTime() { Calendar cal = Calendar.getInstance(TimeZone.getTimeZone("UTC")); return cal.getTimeInMillis(); } -} + + private void putReservation(final ReservationId reservationId, + final SubClusterId subClusterId, boolean update) throws YarnException { + String reservationZNode = getNodePath(reservationsZNode, reservationId.toString()); + SubClusterIdProto proto = ((SubClusterIdPBImpl)subClusterId).getProto(); + byte[] data = proto.toByteArray(); + put(reservationZNode, data, update); + } + + private SubClusterId getReservation(final ReservationId reservationId) + throws YarnException { + String reservationIdZNode = getNodePath(reservationsZNode, reservationId.toString()); + SubClusterId subClusterId = null; + byte[] data = get(reservationIdZNode); + if (data != null) { + try { + subClusterId = new SubClusterIdPBImpl(SubClusterIdProto.parseFrom(data)); + } catch (InvalidProtocolBufferException e) { + String errMsg = "Cannot parse reservation at " + reservationId; + FederationStateStoreUtils.logAndThrowStoreException(LOG, errMsg); + } + } + return subClusterId; + } + + @VisibleForTesting + public ZKFederationStateStoreOpDurations getOpDurations() { + return opDurations; + } + + @Override + public AddReservationHomeSubClusterResponse addReservationHomeSubCluster( + AddReservationHomeSubClusterRequest request) throws YarnException { + + long start = clock.getTime(); + FederationReservationHomeSubClusterStoreInputValidator.validate(request); + ReservationHomeSubCluster reservationHomeSubCluster = request.getReservationHomeSubCluster(); + ReservationId reservationId = reservationHomeSubCluster.getReservationId(); + + // Try to write the subcluster + SubClusterId homeSubCluster = reservationHomeSubCluster.getHomeSubCluster(); + try { + putReservation(reservationId, homeSubCluster, false); + } catch (Exception e) { + String errMsg = "Cannot add reservation home subcluster for " + reservationId; + FederationStateStoreUtils.logAndThrowStoreException(LOG, errMsg); + } + + // Check for the actual subcluster + try { + homeSubCluster = getReservation(reservationId); + } catch (Exception e) { + String errMsg = "Cannot check app home subcluster for " + reservationId; + FederationStateStoreUtils.logAndThrowStoreException(LOG, errMsg); + } + long end = clock.getTime(); + opDurations.addReservationHomeSubClusterDuration(start, end); + return AddReservationHomeSubClusterResponse.newInstance(homeSubCluster); + } + + @Override + public GetReservationHomeSubClusterResponse getReservationHomeSubCluster( + GetReservationHomeSubClusterRequest request) throws YarnException { + + long start = clock.getTime(); + FederationReservationHomeSubClusterStoreInputValidator.validate(request); + ReservationId reservationId = request.getReservationId(); + SubClusterId homeSubCluster = getReservation(reservationId); + + if (homeSubCluster == null) { + String errMsg = "Reservation " + reservationId + " does not exist"; + FederationStateStoreUtils.logAndThrowStoreException(LOG, errMsg); + } + + ReservationHomeSubCluster reservationHomeSubCluster = + ReservationHomeSubCluster.newInstance(reservationId, homeSubCluster); + long end = clock.getTime(); + opDurations.addGetReservationHomeSubClusterDuration(start, end); + return GetReservationHomeSubClusterResponse.newInstance(reservationHomeSubCluster); + } + + @Override + public GetReservationsHomeSubClusterResponse getReservationsHomeSubCluster( + GetReservationsHomeSubClusterRequest request) throws YarnException { + long start = clock.getTime(); + List result = new ArrayList<>(); + + try { + for (String child : zkManager.getChildren(reservationsZNode)) { + ReservationId reservationId = ReservationId.parseReservationId(child); + SubClusterId homeSubCluster = getReservation(reservationId); + ReservationHomeSubCluster app = + ReservationHomeSubCluster.newInstance(reservationId, homeSubCluster); + result.add(app); + } + } catch (Exception e) { + String errMsg = "Cannot get apps: " + e.getMessage(); + FederationStateStoreUtils.logAndThrowStoreException(LOG, errMsg); + } + long end = clock.getTime(); + opDurations.addGetReservationsHomeSubClusterDuration(start, end); + return GetReservationsHomeSubClusterResponse.newInstance(result); + } + + @Override + public DeleteReservationHomeSubClusterResponse deleteReservationHomeSubCluster( + DeleteReservationHomeSubClusterRequest request) throws YarnException { + long start = clock.getTime(); + FederationReservationHomeSubClusterStoreInputValidator.validate(request); + ReservationId reservationId = request.getReservationId(); + String reservationZNode = getNodePath(reservationsZNode, reservationId.toString()); + + boolean exists = false; + try { + exists = zkManager.exists(reservationZNode); + } catch (Exception e) { + String errMsg = "Cannot check reservation: " + e.getMessage(); + FederationStateStoreUtils.logAndThrowStoreException(LOG, errMsg); + } + + if (!exists) { + String errMsg = "Reservation " + reservationId + " does not exist"; + FederationStateStoreUtils.logAndThrowStoreException(LOG, errMsg); + } + + try { + zkManager.delete(reservationZNode); + } catch (Exception e) { + String errMsg = "Cannot delete reservation: " + e.getMessage(); + FederationStateStoreUtils.logAndThrowStoreException(LOG, errMsg); + } + long end = clock.getTime(); + opDurations.addDeleteReservationHomeSubClusterDuration(start, end); + return DeleteReservationHomeSubClusterResponse.newInstance(); + } + + @Override + public UpdateReservationHomeSubClusterResponse updateReservationHomeSubCluster( + UpdateReservationHomeSubClusterRequest request) throws YarnException { + + long start = clock.getTime(); + FederationReservationHomeSubClusterStoreInputValidator.validate(request); + ReservationHomeSubCluster reservationHomeSubCluster = request.getReservationHomeSubCluster(); + ReservationId reservationId = reservationHomeSubCluster.getReservationId(); + SubClusterId homeSubCluster = getReservation(reservationId); + + if (homeSubCluster == null) { + String errMsg = "Reservation " + reservationId + " does not exist"; + FederationStateStoreUtils.logAndThrowStoreException(LOG, errMsg); + } + + SubClusterId newSubClusterId = reservationHomeSubCluster.getHomeSubCluster(); + putReservation(reservationId, newSubClusterId, true); + long end = clock.getTime(); + opDurations.addUpdateReservationHomeSubClusterDuration(start, end); + return UpdateReservationHomeSubClusterResponse.newInstance(); + } +} \ No newline at end of file diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/AddReservationHomeSubClusterRequest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/AddReservationHomeSubClusterRequest.java new file mode 100644 index 0000000000000..e51bce342849c --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/AddReservationHomeSubClusterRequest.java @@ -0,0 +1,72 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package org.apache.hadoop.yarn.server.federation.store.records; + +import org.apache.hadoop.classification.InterfaceAudience.Private; +import org.apache.hadoop.classification.InterfaceAudience.Public; +import org.apache.hadoop.classification.InterfaceStability.Unstable; +import org.apache.hadoop.yarn.util.Records; + +/** + *

    + * The request sent by the Router to Federation state + * store to map the home subcluster of a newly submitted Reservation. + * + *

    + * The request includes the mapping details, i.e.: + *

      + *
    • {@code ReservationId}
      • + *
    • {@code SubClusterId}
      • + *
    + */ +@Private +@Unstable +public abstract class AddReservationHomeSubClusterRequest { + + @Private + @Unstable + public static AddReservationHomeSubClusterRequest newInstance( + ReservationHomeSubCluster reservationHomeSubCluster) { + AddReservationHomeSubClusterRequest mapRequest = + Records.newRecord(AddReservationHomeSubClusterRequest.class); + mapRequest.setReservationHomeSubCluster(reservationHomeSubCluster); + return mapRequest; + } + + /** + * Get the {@link ReservationHomeSubCluster} representing the mapping of the + * Reservation to it's home sub-cluster. + * + * @return the mapping of the Reservation to it's home sub-cluster. + */ + @Public + @Unstable + public abstract ReservationHomeSubCluster getReservationHomeSubCluster(); + + /** + * Set the {@link ReservationHomeSubCluster} representing the mapping of the + * Reservation to it's home sub-cluster. + * + * @param reservationHomeSubCluster the mapping of the Reservation to it's + * home sub-cluster. + */ + @Private + @Unstable + public abstract void setReservationHomeSubCluster( + ReservationHomeSubCluster reservationHomeSubCluster); +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/AddReservationHomeSubClusterResponse.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/AddReservationHomeSubClusterResponse.java new file mode 100644 index 0000000000000..0486d889ac19f --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/AddReservationHomeSubClusterResponse.java @@ -0,0 +1,65 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package org.apache.hadoop.yarn.server.federation.store.records; + +import org.apache.hadoop.classification.InterfaceAudience.Private; +import org.apache.hadoop.classification.InterfaceStability.Unstable; +import org.apache.hadoop.yarn.util.Records; + +/** + * AddReservationHomeSubClusterResponse contains the answer from the + * {@code FederationReservationHomeSubClusterStore} to a request to insert a + * newly generated ReservationId and its owner. + * + * The response contains reservation's home sub-cluster as it is stored in the + * {@code FederationReservationHomeSubClusterStore}. If a mapping for the + * reservation already existed, the {@code SubClusterId} in this response will + * return the existing mapping which might be different from that in the + * {@code AddReservationHomeSubClusterRequest}. + */ +@Private +@Unstable +public abstract class AddReservationHomeSubClusterResponse { + + @Private + @Unstable + public static AddReservationHomeSubClusterResponse newInstance( + SubClusterId homeSubCluster) { + AddReservationHomeSubClusterResponse response = + Records.newRecord(AddReservationHomeSubClusterResponse.class); + response.setHomeSubCluster(homeSubCluster); + return response; + } + + /** + * Set the home sub-cluster that this Reservation has been assigned to. + * + * @param homeSubCluster the {@link SubClusterId} of this reservation's home + * sub-cluster + */ + public abstract void setHomeSubCluster(SubClusterId homeSubCluster); + + /** + * Get the home sub-cluster that this Reservation has been assigned to. This + * may not match the {@link SubClusterId} in the corresponding response, if + * the mapping for the request's reservation already existed. + * + * @return the {@link SubClusterId} of this reservation's home sub-cluster + */ + public abstract SubClusterId getHomeSubCluster(); +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/DeleteReservationHomeSubClusterRequest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/DeleteReservationHomeSubClusterRequest.java new file mode 100644 index 0000000000000..81d9fc7aa35bd --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/DeleteReservationHomeSubClusterRequest.java @@ -0,0 +1,65 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package org.apache.hadoop.yarn.server.federation.store.records; + +import org.apache.hadoop.classification.InterfaceAudience.Private; +import org.apache.hadoop.classification.InterfaceAudience.Public; +import org.apache.hadoop.classification.InterfaceStability.Unstable; +import org.apache.hadoop.yarn.api.records.ReservationId; +import org.apache.hadoop.yarn.util.Records; + +/** + * The request to Federation state store to delete the mapping of + * home subcluster of a submitted reservation. + */ +@Private +@Unstable +public abstract class DeleteReservationHomeSubClusterRequest { + + @Private + @Unstable + public static DeleteReservationHomeSubClusterRequest newInstance( + ReservationId reservationId) { + DeleteReservationHomeSubClusterRequest deleteReservationRequest = + Records.newRecord(DeleteReservationHomeSubClusterRequest.class); + deleteReservationRequest.setReservationId(reservationId); + return deleteReservationRequest; + } + + /** + * Get the identifier of the {@link ReservationId} to be removed from + * Federation state store . + * + * @return the identifier of the Reservation to be removed from Federation + * State Store. + */ + @Public + @Unstable + public abstract ReservationId getReservationId(); + + /** + * Set the identifier of the {@link ReservationId} to be removed from + * Federation state store . + * + * @param reservationId the identifier of the Reservation to be removed from + * Federation State Store. + */ + @Private + @Unstable + public abstract void setReservationId(ReservationId reservationId); +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/DeleteReservationHomeSubClusterResponse.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/DeleteReservationHomeSubClusterResponse.java new file mode 100644 index 0000000000000..0b7116904be1f --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/DeleteReservationHomeSubClusterResponse.java @@ -0,0 +1,42 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package org.apache.hadoop.yarn.server.federation.store.records; + +import org.apache.hadoop.classification.InterfaceAudience.Private; +import org.apache.hadoop.classification.InterfaceStability.Unstable; +import org.apache.hadoop.yarn.util.Records; + +/** + * DeleteReservationHomeSubClusterResponse contains the answer from the {@code + * FederationReservationHomeSubClusterStore} to a request to delete the mapping + * of home subcluster of a submitted reservation. Currently, response is empty if + * the operation was successful, if not an exception reporting reason for a + * failure. + */ +@Private +@Unstable +public abstract class DeleteReservationHomeSubClusterResponse { + + @Private + @Unstable + public static DeleteReservationHomeSubClusterResponse newInstance() { + DeleteReservationHomeSubClusterResponse response = + Records.newRecord(DeleteReservationHomeSubClusterResponse.class); + return response; + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/GetApplicationHomeSubClusterResponse.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/GetApplicationHomeSubClusterResponse.java index 60735b382f16a..6144b01e86060 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/GetApplicationHomeSubClusterResponse.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/GetApplicationHomeSubClusterResponse.java @@ -20,6 +20,7 @@ import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.classification.InterfaceAudience.Public; import org.apache.hadoop.classification.InterfaceStability.Unstable; +import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.util.Records; /** @@ -42,7 +43,9 @@ public abstract class GetApplicationHomeSubClusterResponse { @Private @Unstable public static GetApplicationHomeSubClusterResponse newInstance( - ApplicationHomeSubCluster applicationHomeSubCluster) { + ApplicationId appId, SubClusterId homeSubCluster) { + ApplicationHomeSubCluster applicationHomeSubCluster = + ApplicationHomeSubCluster.newInstance(appId, homeSubCluster); GetApplicationHomeSubClusterResponse mapResponse = Records.newRecord(GetApplicationHomeSubClusterResponse.class); mapResponse.setApplicationHomeSubCluster(applicationHomeSubCluster); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/GetReservationHomeSubClusterRequest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/GetReservationHomeSubClusterRequest.java new file mode 100644 index 0000000000000..f82fec6206440 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/GetReservationHomeSubClusterRequest.java @@ -0,0 +1,64 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package org.apache.hadoop.yarn.server.federation.store.records; + +import org.apache.hadoop.classification.InterfaceAudience.Private; +import org.apache.hadoop.classification.InterfaceAudience.Public; +import org.apache.hadoop.classification.InterfaceStability.Unstable; +import org.apache.hadoop.yarn.api.records.ReservationId; +import org.apache.hadoop.yarn.util.Records; + +/** + * Request class to obtain the home sub-cluster for the specified + * {@link ReservationId}. + */ +@Private +@Unstable +public abstract class GetReservationHomeSubClusterRequest { + + @Private + @Unstable + public static GetReservationHomeSubClusterRequest newInstance( + ReservationId reservationId) { + GetReservationHomeSubClusterRequest appMapping = + Records.newRecord(GetReservationHomeSubClusterRequest.class); + appMapping.setReservationId(reservationId); + return appMapping; + } + + /** + * Get the {@link ReservationId} representing the unique identifier of the + * application. + * + * @return the application identifier + */ + @Public + @Unstable + public abstract ReservationId getReservationId(); + + /** + * Set the {@link ReservationId} representing the unique identifier of the + * application. + * + * @param reservationId the reservatopm identifier + */ + @Private + @Unstable + public abstract void setReservationId(ReservationId reservationId); + +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/GetReservationHomeSubClusterResponse.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/GetReservationHomeSubClusterResponse.java new file mode 100644 index 0000000000000..50328a176b100 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/GetReservationHomeSubClusterResponse.java @@ -0,0 +1,72 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package org.apache.hadoop.yarn.server.federation.store.records; + +import org.apache.hadoop.classification.InterfaceAudience.Private; +import org.apache.hadoop.classification.InterfaceAudience.Public; +import org.apache.hadoop.classification.InterfaceStability.Unstable; +import org.apache.hadoop.yarn.util.Records; + +/** + *

    + * The response sent by Federation state + * store to a query for the home subcluster of a newly submitted + * reservation. + * + *

    + * The request includes the mapping details, i.e.: + *

      + *
    • {@code ReservationId}
      • + *
    • {@code SubClusterId}
      • + *
    + */ +@Private +@Unstable +public abstract class GetReservationHomeSubClusterResponse { + + @Private + @Unstable + public static GetReservationHomeSubClusterResponse newInstance( + ReservationHomeSubCluster reservationHomeSubCluster) { + GetReservationHomeSubClusterResponse mapResponse = + Records.newRecord(GetReservationHomeSubClusterResponse.class); + mapResponse.setReservationHomeSubCluster(reservationHomeSubCluster); + return mapResponse; + } + + /** + * Get the {@link ReservationHomeSubCluster} representing the mapping of the + * Reservation to it's home sub-cluster. + * + * @return the mapping of the reservation to it's home sub-cluster. + */ + @Public + @Unstable + public abstract ReservationHomeSubCluster getReservationHomeSubCluster(); + + /** + * Set the {@link ReservationHomeSubCluster} representing the mapping of the + * Reservation to it's home sub-cluster. + * + * @param reservationHomeSubCluster the mapping of the reservation to it's home sub-cluster. + */ + @Private + @Unstable + public abstract void setReservationHomeSubCluster( + ReservationHomeSubCluster reservationHomeSubCluster); +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/GetReservationsHomeSubClusterRequest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/GetReservationsHomeSubClusterRequest.java new file mode 100644 index 0000000000000..c91d25dcd4930 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/GetReservationsHomeSubClusterRequest.java @@ -0,0 +1,39 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package org.apache.hadoop.yarn.server.federation.store.records; + +import org.apache.hadoop.classification.InterfaceAudience.Private; +import org.apache.hadoop.classification.InterfaceStability.Unstable; +import org.apache.hadoop.yarn.util.Records; + +/** + * Request class to obtain the home sub-cluster mapping of all active + * reservations. + */ +@Private +@Unstable +public abstract class GetReservationsHomeSubClusterRequest { + + @Private + @Unstable + public static GetReservationsHomeSubClusterRequest newInstance() { + GetReservationsHomeSubClusterRequest request = + Records.newRecord(GetReservationsHomeSubClusterRequest.class); + return request; + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/GetReservationsHomeSubClusterResponse.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/GetReservationsHomeSubClusterResponse.java new file mode 100644 index 0000000000000..f39430dbe6322 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/GetReservationsHomeSubClusterResponse.java @@ -0,0 +1,75 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package org.apache.hadoop.yarn.server.federation.store.records; + +import java.util.List; + +import org.apache.hadoop.classification.InterfaceAudience.Private; +import org.apache.hadoop.classification.InterfaceAudience.Public; +import org.apache.hadoop.classification.InterfaceStability.Unstable; +import org.apache.hadoop.yarn.util.Records; + +/** + *

    + * The response sent by Federation state + * store to a query for the home subcluster of all submitted + * reservations. + * + *

    + * The response includes the mapping details, i.e.: + *

      + *
    • {@code ReservationId}
      • + *
    • {@code SubClusterId}
      • + *
    + */ +@Private +@Unstable +public abstract class GetReservationsHomeSubClusterResponse { + + @Private + @Unstable + public static GetReservationsHomeSubClusterResponse newInstance( + List appsHomeSubClusters) { + GetReservationsHomeSubClusterResponse mapResponse = + Records.newRecord(GetReservationsHomeSubClusterResponse.class); + mapResponse.setAppsHomeSubClusters(appsHomeSubClusters); + return mapResponse; + } + + /** + * Get the {@link ReservationHomeSubCluster} list representing the mapping of + * all submitted reservations to it's home sub-cluster. + * + * @return the mapping of all submitted reservation to it's home sub-cluster. + */ + @Public + @Unstable + public abstract List getAppsHomeSubClusters(); + + /** + * Set the {@link ReservationHomeSubCluster} list representing the mapping of + * all submitted reservations to it's home sub-cluster. + * + * @param reservationsHomeSubClusters the mapping of all submitted reservation + * to it's home sub-cluster. + */ + @Private + @Unstable + public abstract void setAppsHomeSubClusters( + List reservationsHomeSubClusters); +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/GetSubClustersInfoResponse.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/GetSubClustersInfoResponse.java index bcf75aba1aef0..0ffe4ae28a355 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/GetSubClustersInfoResponse.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/GetSubClustersInfoResponse.java @@ -19,6 +19,7 @@ package org.apache.hadoop.yarn.server.federation.store.records; import java.util.List; +import java.util.Collection; import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.classification.InterfaceAudience.Public; @@ -36,7 +37,7 @@ public abstract class GetSubClustersInfoResponse { @Public @Unstable public static GetSubClustersInfoResponse newInstance( - List subClusters) { + Collection subClusters) { GetSubClustersInfoResponse subClusterInfos = Records.newRecord(GetSubClustersInfoResponse.class); subClusterInfos.setSubClusters(subClusters); @@ -61,6 +62,5 @@ public static GetSubClustersInfoResponse newInstance( */ @Private @Unstable - public abstract void setSubClusters(List subClusters); - + public abstract void setSubClusters(Collection subClusters); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/ReservationHomeSubCluster.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/ReservationHomeSubCluster.java new file mode 100644 index 0000000000000..e080d115716dd --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/ReservationHomeSubCluster.java @@ -0,0 +1,129 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package org.apache.hadoop.yarn.server.federation.store.records; + +import org.apache.commons.lang3.builder.EqualsBuilder; +import org.apache.commons.lang3.builder.HashCodeBuilder; +import org.apache.hadoop.classification.InterfaceAudience.Private; +import org.apache.hadoop.classification.InterfaceAudience.Public; +import org.apache.hadoop.classification.InterfaceStability.Unstable; +import org.apache.hadoop.yarn.api.records.ReservationId; +import org.apache.hadoop.yarn.util.Records; + +/** + *

    + * ReservationHomeSubCluster is a report of the runtime information of the + * reservation that is running in the federated cluster. + * + *

    + * It includes information such as: + *

      + *
    • {@link ReservationId}
      • + *
    • {@link SubClusterId}
      • + *
    + * + */ +@Private +@Unstable +public abstract class ReservationHomeSubCluster { + + @Private + @Unstable + public static ReservationHomeSubCluster newInstance(ReservationId resId, + SubClusterId homeSubCluster) { + ReservationHomeSubCluster appMapping = Records.newRecord(ReservationHomeSubCluster.class); + appMapping.setReservationId(resId); + appMapping.setHomeSubCluster(homeSubCluster); + return appMapping; + } + + /** + * Get the {@link ReservationId} representing the unique identifier of the + * Reservation. + * + * @return the reservation identifier + */ + @Public + @Unstable + public abstract ReservationId getReservationId(); + + /** + * Set the {@link ReservationId} representing the unique identifier of the + * Reservation. + * + * @param resId the reservation identifier + */ + @Private + @Unstable + public abstract void setReservationId(ReservationId resId); + + /** + * Get the {@link SubClusterId} representing the unique identifier of the home + * subcluster in which the reservation is mapped to. + * + * @return the home subcluster identifier + */ + @Public + @Unstable + public abstract SubClusterId getHomeSubCluster(); + + /** + * Set the {@link SubClusterId} representing the unique identifier of the home + * subcluster in which the ReservationMaster of the reservation is running. + * + * @param subClusterId the home subcluster identifier + */ + @Private + @Unstable + public abstract void setHomeSubCluster(SubClusterId subClusterId); + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if (obj == null) { + return false; + } + if (getClass() != obj.getClass()) { + return false; + } + ReservationHomeSubCluster other = (ReservationHomeSubCluster) obj; + + return new EqualsBuilder() + .append(this.getReservationId(), other.getReservationId()) + .append(this.getHomeSubCluster(), other.getHomeSubCluster()) + .isEquals(); + } + + @Override + public int hashCode() { + return new HashCodeBuilder(). + append(this.getReservationId()). + append(this.getHomeSubCluster()). + toHashCode(); + } + + @Override + public String toString() { + return "ReservationHomeSubCluster [getReservationId()=" + + getReservationId() + ", getApplicationHomeSubcluster()=" + getHomeSubCluster() + + "]"; + } + +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/SubClusterId.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/SubClusterId.java index fec967d86bdd9..7eeb44bba55a7 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/SubClusterId.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/SubClusterId.java @@ -43,6 +43,14 @@ public static SubClusterId newInstance(String subClusterId) { return id; } + @Private + @Unstable + public static SubClusterId newInstance(Integer subClusterId) { + SubClusterId id = Records.newRecord(SubClusterId.class); + id.setId(String.valueOf(subClusterId)); + return id; + } + /** * Get the string identifier of the subcluster which is unique across * the federated cluster. The identifier is static, i.e. preserved across diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/UpdateReservationHomeSubClusterRequest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/UpdateReservationHomeSubClusterRequest.java new file mode 100644 index 0000000000000..8410d65095537 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/UpdateReservationHomeSubClusterRequest.java @@ -0,0 +1,74 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package org.apache.hadoop.yarn.server.federation.store.records; + +import org.apache.hadoop.classification.InterfaceAudience.Private; +import org.apache.hadoop.classification.InterfaceAudience.Public; +import org.apache.hadoop.classification.InterfaceStability.Unstable; +import org.apache.hadoop.yarn.util.Records; + +/** + *

    + * The request sent by the Router to + * Federation state store to update the home subcluster of a newly + * submitted reservation. + * + *

    + * The request includes the mapping details, i.e.: + *

      + *
    • {@code ReservationId}
      • + *
    • {@code SubClusterId}
      • + *
    + */ +@Private +@Unstable +public abstract class UpdateReservationHomeSubClusterRequest { + + @Private + @Unstable + public static UpdateReservationHomeSubClusterRequest newInstance( + ReservationHomeSubCluster reservationHomeSubCluster) { + UpdateReservationHomeSubClusterRequest updateReservationRequest = + Records.newRecord(UpdateReservationHomeSubClusterRequest.class); + updateReservationRequest + .setReservationHomeSubCluster(reservationHomeSubCluster); + return updateReservationRequest; + } + + /** + * Get the {@link ReservationHomeSubCluster} representing the mapping of the + * reservation to it's home sub-cluster. + * + * @return the mapping of the reservation to it's home sub-cluster. + */ + @Public + @Unstable + public abstract ReservationHomeSubCluster getReservationHomeSubCluster(); + + /** + * Set the {@link ReservationHomeSubCluster} representing the mapping of the + * reservation to it's home sub-cluster. + * + * @param reservationHomeSubCluster the mapping of the reservation to it's + * home sub-cluster. + */ + @Private + @Unstable + public abstract void setReservationHomeSubCluster( + ReservationHomeSubCluster reservationHomeSubCluster); +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/UpdateReservationHomeSubClusterResponse.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/UpdateReservationHomeSubClusterResponse.java new file mode 100644 index 0000000000000..d6fd0a481e7ec --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/UpdateReservationHomeSubClusterResponse.java @@ -0,0 +1,43 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package org.apache.hadoop.yarn.server.federation.store.records; + +import org.apache.hadoop.classification.InterfaceAudience.Private; +import org.apache.hadoop.classification.InterfaceStability.Unstable; +import org.apache.hadoop.yarn.util.Records; + +/** + * UpdateReservationHomeSubClusterResponse contains the answer from the + * {@code FederationReservationHomeSubClusterStore} to a request to register the + * home subcluster of a submitted reservation. Currently response is empty if + * the operation was successful, if not an exception reporting reason for a + * failure. + */ +@Private +@Unstable +public abstract class UpdateReservationHomeSubClusterResponse { + + @Private + @Unstable + public static UpdateReservationHomeSubClusterResponse newInstance() { + UpdateReservationHomeSubClusterResponse response = + Records.newRecord(UpdateReservationHomeSubClusterResponse.class); + return response; + } + +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/AddReservationHomeSubClusterRequestPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/AddReservationHomeSubClusterRequestPBImpl.java new file mode 100644 index 0000000000000..e6facbec9c937 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/AddReservationHomeSubClusterRequestPBImpl.java @@ -0,0 +1,130 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package org.apache.hadoop.yarn.server.federation.store.records.impl.pb; + +import org.apache.hadoop.classification.InterfaceAudience.Private; +import org.apache.hadoop.classification.InterfaceStability.Unstable; +import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.AddReservationHomeSubClusterRequestProto; +import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.AddReservationHomeSubClusterRequestProtoOrBuilder; +import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.ReservationHomeSubClusterProto; +import org.apache.hadoop.yarn.server.federation.store.records.AddReservationHomeSubClusterRequest; +import org.apache.hadoop.yarn.server.federation.store.records.ReservationHomeSubCluster; + +import org.apache.hadoop.thirdparty.protobuf.TextFormat; + +/** + * Protocol buffer based implementation of + * {@link AddReservationHomeSubClusterRequest}. + */ +@Private +@Unstable +public class AddReservationHomeSubClusterRequestPBImpl + extends AddReservationHomeSubClusterRequest { + + private AddReservationHomeSubClusterRequestProto proto = + AddReservationHomeSubClusterRequestProto.getDefaultInstance(); + private AddReservationHomeSubClusterRequestProto.Builder builder = null; + private boolean viaProto = false; + + public AddReservationHomeSubClusterRequestPBImpl() { + builder = AddReservationHomeSubClusterRequestProto.newBuilder(); + } + + public AddReservationHomeSubClusterRequestPBImpl( + AddReservationHomeSubClusterRequestProto proto) { + this.proto = proto; + viaProto = true; + } + + public AddReservationHomeSubClusterRequestProto getProto() { + mergeLocalToProto(); + proto = viaProto ? proto : builder.build(); + viaProto = true; + return proto; + } + + private void mergeLocalToProto() { + if (viaProto) { + maybeInitBuilder(); + } + mergeLocalToBuilder(); + proto = builder.build(); + viaProto = true; + } + + private void maybeInitBuilder() { + if (viaProto || builder == null) { + builder = AddReservationHomeSubClusterRequestProto.newBuilder(proto); + } + viaProto = false; + } + + private void mergeLocalToBuilder() { + } + + @Override + public int hashCode() { + return getProto().hashCode(); + } + + @Override + public boolean equals(Object other) { + if (other == null) { + return false; + } + if (other.getClass().isAssignableFrom(this.getClass())) { + return this.getProto().equals(this.getClass().cast(other).getProto()); + } + return false; + } + + @Override + public String toString() { + return TextFormat.shortDebugString(getProto()); + } + + @Override + public ReservationHomeSubCluster getReservationHomeSubCluster() { + AddReservationHomeSubClusterRequestProtoOrBuilder p = viaProto ? proto : builder; + if (!p.hasAppSubclusterMap()) { + return null; + } + return convertFromProtoFormat(p.getAppSubclusterMap()); + } + + @Override + public void setReservationHomeSubCluster( + ReservationHomeSubCluster reservationInfo) { + maybeInitBuilder(); + if (reservationInfo == null) { + builder.clearAppSubclusterMap(); + return; + } + builder.setAppSubclusterMap(convertToProtoFormat(reservationInfo)); + } + + private ReservationHomeSubCluster convertFromProtoFormat( + ReservationHomeSubClusterProto sc) { + return new ReservationHomeSubClusterPBImpl(sc); + } + + private ReservationHomeSubClusterProto convertToProtoFormat( + ReservationHomeSubCluster sc) { + return ((ReservationHomeSubClusterPBImpl) sc).getProto(); + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/AddReservationHomeSubClusterResponsePBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/AddReservationHomeSubClusterResponsePBImpl.java new file mode 100644 index 0000000000000..e64e740634dec --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/AddReservationHomeSubClusterResponsePBImpl.java @@ -0,0 +1,115 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package org.apache.hadoop.yarn.server.federation.store.records.impl.pb; + +import org.apache.hadoop.classification.InterfaceAudience.Private; +import org.apache.hadoop.classification.InterfaceStability.Unstable; +import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.AddReservationHomeSubClusterResponseProto; +import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.AddReservationHomeSubClusterResponseProtoOrBuilder; +import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.SubClusterIdProto; +import org.apache.hadoop.yarn.server.federation.store.records.AddReservationHomeSubClusterResponse; +import org.apache.hadoop.yarn.server.federation.store.records.SubClusterId; + +import org.apache.hadoop.thirdparty.protobuf.TextFormat; + +/** + * Protocol buffer based implementation of + * {@link AddReservationHomeSubClusterResponse}. + */ +@Private +@Unstable +public class AddReservationHomeSubClusterResponsePBImpl + extends AddReservationHomeSubClusterResponse { + + private AddReservationHomeSubClusterResponseProto proto = + AddReservationHomeSubClusterResponseProto.getDefaultInstance(); + private AddReservationHomeSubClusterResponseProto.Builder builder = null; + private boolean viaProto = false; + + public AddReservationHomeSubClusterResponsePBImpl() { + builder = AddReservationHomeSubClusterResponseProto.newBuilder(); + } + + public AddReservationHomeSubClusterResponsePBImpl( + AddReservationHomeSubClusterResponseProto proto) { + this.proto = proto; + viaProto = true; + } + + private void maybeInitBuilder() { + if (viaProto || builder == null) { + builder = AddReservationHomeSubClusterResponseProto.newBuilder(proto); + } + viaProto = false; + } + + public AddReservationHomeSubClusterResponseProto getProto() { + proto = viaProto ? proto : builder.build(); + viaProto = true; + return proto; + } + + @Override + public void setHomeSubCluster(SubClusterId homeSubCluster) { + maybeInitBuilder(); + if (homeSubCluster == null) { + builder.clearHomeSubCluster(); + return; + } + builder.setHomeSubCluster(convertToProtoFormat(homeSubCluster)); + } + + @Override + public SubClusterId getHomeSubCluster() { + AddReservationHomeSubClusterResponseProtoOrBuilder p = viaProto ? proto : builder; + + if (!p.hasHomeSubCluster()) { + return null; + } + return convertFromProtoFormat(p.getHomeSubCluster()); + } + + @Override + public int hashCode() { + return getProto().hashCode(); + } + + @Override + public boolean equals(Object other) { + if (other == null) { + return false; + } + if (other.getClass().isAssignableFrom(this.getClass())) { + return this.getProto().equals(this.getClass().cast(other).getProto()); + } + return false; + } + + @Override + public String toString() { + return TextFormat.shortDebugString(getProto()); + } + + private SubClusterId convertFromProtoFormat(SubClusterIdProto sc) { + return new SubClusterIdPBImpl(sc); + } + + private SubClusterIdProto convertToProtoFormat(SubClusterId sc) { + return ((SubClusterIdPBImpl) sc).getProto(); + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/DeleteReservationHomeSubClusterRequestPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/DeleteReservationHomeSubClusterRequestPBImpl.java new file mode 100644 index 0000000000000..b6970b04e7fca --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/DeleteReservationHomeSubClusterRequestPBImpl.java @@ -0,0 +1,129 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package org.apache.hadoop.yarn.server.federation.store.records.impl.pb; + +import org.apache.hadoop.classification.InterfaceAudience.Private; +import org.apache.hadoop.classification.InterfaceStability.Unstable; +import org.apache.hadoop.yarn.api.records.ReservationId; +import org.apache.hadoop.yarn.api.records.impl.pb.ReservationIdPBImpl; +import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.DeleteReservationHomeSubClusterRequestProto; +import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.DeleteReservationHomeSubClusterRequestProtoOrBuilder; +import org.apache.hadoop.yarn.proto.YarnProtos.ReservationIdProto; +import org.apache.hadoop.yarn.server.federation.store.records.DeleteReservationHomeSubClusterRequest; + +import org.apache.hadoop.thirdparty.protobuf.TextFormat; + +/** + * Protocol buffer based implementation of + * {@link DeleteReservationHomeSubClusterRequest}. + */ +@Private +@Unstable +public class DeleteReservationHomeSubClusterRequestPBImpl + extends DeleteReservationHomeSubClusterRequest { + + private DeleteReservationHomeSubClusterRequestProto proto = + DeleteReservationHomeSubClusterRequestProto.getDefaultInstance(); + private DeleteReservationHomeSubClusterRequestProto.Builder builder = null; + private boolean viaProto = false; + + public DeleteReservationHomeSubClusterRequestPBImpl() { + builder = DeleteReservationHomeSubClusterRequestProto.newBuilder(); + } + + public DeleteReservationHomeSubClusterRequestPBImpl( + DeleteReservationHomeSubClusterRequestProto proto) { + this.proto = proto; + viaProto = true; + } + + public DeleteReservationHomeSubClusterRequestProto getProto() { + mergeLocalToProto(); + proto = viaProto ? proto : builder.build(); + viaProto = true; + return proto; + } + + private void mergeLocalToProto() { + if (viaProto) { + maybeInitBuilder(); + } + mergeLocalToBuilder(); + proto = builder.build(); + viaProto = true; + } + + private void maybeInitBuilder() { + if (viaProto || builder == null) { + builder = DeleteReservationHomeSubClusterRequestProto.newBuilder(proto); + } + viaProto = false; + } + + private void mergeLocalToBuilder() { + } + + @Override + public int hashCode() { + return getProto().hashCode(); + } + + @Override + public boolean equals(Object other) { + if (other == null) { + return false; + } + if (other.getClass().isAssignableFrom(this.getClass())) { + return this.getProto().equals(this.getClass().cast(other).getProto()); + } + return false; + } + + @Override + public String toString() { + return TextFormat.shortDebugString(getProto()); + } + + @Override + public ReservationId getReservationId() { + DeleteReservationHomeSubClusterRequestProtoOrBuilder p = + viaProto ? proto : builder; + if (!p.hasReservationId()) { + return null; + } + return convertFromProtoFormat(p.getReservationId()); + } + + @Override + public void setReservationId(ReservationId reservationId) { + maybeInitBuilder(); + if (reservationId == null) { + builder.clearReservationId(); + return; + } + builder.setReservationId(convertToProtoFormat(reservationId)); + } + + private ReservationId convertFromProtoFormat(ReservationIdProto appId) { + return new ReservationIdPBImpl(appId); + } + + private ReservationIdProto convertToProtoFormat(ReservationId appId) { + return ((ReservationIdPBImpl) appId).getProto(); + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/DeleteReservationHomeSubClusterResponsePBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/DeleteReservationHomeSubClusterResponsePBImpl.java new file mode 100644 index 0000000000000..1bbbe40d63a6c --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/DeleteReservationHomeSubClusterResponsePBImpl.java @@ -0,0 +1,76 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package org.apache.hadoop.yarn.server.federation.store.records.impl.pb; + +import org.apache.hadoop.classification.InterfaceAudience.Private; +import org.apache.hadoop.classification.InterfaceStability.Unstable; +import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.DeleteReservationHomeSubClusterResponseProto; +import org.apache.hadoop.yarn.server.federation.store.records.DeleteReservationHomeSubClusterResponse; + +import org.apache.hadoop.thirdparty.protobuf.TextFormat; + +/** + * Protocol buffer based implementation of + * {@link DeleteReservationHomeSubClusterResponse}. + */ +@Private +@Unstable +public class DeleteReservationHomeSubClusterResponsePBImpl + extends DeleteReservationHomeSubClusterResponse { + private DeleteReservationHomeSubClusterResponseProto proto = + DeleteReservationHomeSubClusterResponseProto.getDefaultInstance(); + private DeleteReservationHomeSubClusterResponseProto.Builder builder = null; + private boolean viaProto = false; + + public DeleteReservationHomeSubClusterResponsePBImpl() { + builder = DeleteReservationHomeSubClusterResponseProto.newBuilder(); + } + + public DeleteReservationHomeSubClusterResponsePBImpl( + DeleteReservationHomeSubClusterResponseProto proto) { + this.proto = proto; + viaProto = true; + } + + public DeleteReservationHomeSubClusterResponseProto getProto() { + proto = viaProto ? proto : builder.build(); + viaProto = true; + return proto; + } + + @Override + public int hashCode() { + return getProto().hashCode(); + } + + @Override + public boolean equals(Object other) { + if (other == null) { + return false; + } + if (other.getClass().isAssignableFrom(this.getClass())) { + return this.getProto().equals(this.getClass().cast(other).getProto()); + } + return false; + } + + @Override + public String toString() { + return TextFormat.shortDebugString(getProto()); + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/GetReservationHomeSubClusterRequestPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/GetReservationHomeSubClusterRequestPBImpl.java new file mode 100644 index 0000000000000..d3011d84a65f3 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/GetReservationHomeSubClusterRequestPBImpl.java @@ -0,0 +1,138 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package org.apache.hadoop.yarn.server.federation.store.records.impl.pb; + +import org.apache.hadoop.classification.InterfaceAudience.Private; +import org.apache.hadoop.classification.InterfaceStability.Unstable; +import org.apache.hadoop.yarn.api.records.ReservationId; +import org.apache.hadoop.yarn.api.records.impl.pb.ReservationIdPBImpl; +import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.GetReservationHomeSubClusterRequestProto; +import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.GetReservationHomeSubClusterRequestProtoOrBuilder; +import org.apache.hadoop.yarn.proto.YarnProtos.ReservationIdProto; +import org.apache.hadoop.yarn.server.federation.store.records.GetReservationHomeSubClusterRequest; + +import org.apache.hadoop.thirdparty.protobuf.TextFormat; + +/** + * Protocol buffer based implementation of + * {@link GetReservationHomeSubClusterRequest}. + */ +@Private +@Unstable +public class GetReservationHomeSubClusterRequestPBImpl + extends GetReservationHomeSubClusterRequest { + + private GetReservationHomeSubClusterRequestProto proto = + GetReservationHomeSubClusterRequestProto.getDefaultInstance(); + private GetReservationHomeSubClusterRequestProto.Builder builder = null; + private boolean viaProto = false; + + private ReservationId reservationId = null; + + public GetReservationHomeSubClusterRequestPBImpl() { + builder = GetReservationHomeSubClusterRequestProto.newBuilder(); + } + + public GetReservationHomeSubClusterRequestPBImpl( + GetReservationHomeSubClusterRequestProto proto) { + this.proto = proto; + viaProto = true; + } + + public GetReservationHomeSubClusterRequestProto getProto() { + mergeLocalToProto(); + proto = viaProto ? proto : builder.build(); + viaProto = true; + return proto; + } + + private void mergeLocalToProto() { + if (viaProto) { + maybeInitBuilder(); + } + mergeLocalToBuilder(); + proto = builder.build(); + viaProto = true; + } + + private void maybeInitBuilder() { + if (viaProto || builder == null) { + builder = GetReservationHomeSubClusterRequestProto.newBuilder(proto); + } + viaProto = false; + } + + private void mergeLocalToBuilder() { + if (this.reservationId != null) { + builder.setReservationId(convertToProtoFormat(this.reservationId)); + } + } + + @Override + public int hashCode() { + return getProto().hashCode(); + } + + @Override + public boolean equals(Object other) { + if (other == null) { + return false; + } + if (other.getClass().isAssignableFrom(this.getClass())) { + return this.getProto().equals(this.getClass().cast(other).getProto()); + } + return false; + } + + @Override + public String toString() { + return TextFormat.shortDebugString(getProto()); + } + + @Override + public ReservationId getReservationId() { + GetReservationHomeSubClusterRequestProtoOrBuilder p = viaProto ? proto : builder; + if (reservationId != null) { + return reservationId; + } + + if (!p.hasReservationId()) { + return null; + } + this.reservationId = convertFromProtoFormat(p.getReservationId()); + return reservationId; + } + + @Override + public void setReservationId(ReservationId reservationId) { + maybeInitBuilder(); + if (reservationId == null) { + builder.clearReservationId(); + return; + } + this.reservationId = reservationId; + } + + private ReservationId convertFromProtoFormat(ReservationIdProto appId) { + return new ReservationIdPBImpl(appId); + } + + private ReservationIdProto convertToProtoFormat(ReservationId appId) { + return ((ReservationIdPBImpl) appId).getProto(); + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/GetReservationHomeSubClusterResponsePBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/GetReservationHomeSubClusterResponsePBImpl.java new file mode 100644 index 0000000000000..f72b61b650dcb --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/GetReservationHomeSubClusterResponsePBImpl.java @@ -0,0 +1,130 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package org.apache.hadoop.yarn.server.federation.store.records.impl.pb; + +import org.apache.hadoop.classification.InterfaceAudience.Private; +import org.apache.hadoop.classification.InterfaceStability.Unstable; +import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.ReservationHomeSubClusterProto; +import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.GetReservationHomeSubClusterResponseProto; +import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.GetReservationHomeSubClusterResponseProtoOrBuilder; +import org.apache.hadoop.yarn.server.federation.store.records.ReservationHomeSubCluster; +import org.apache.hadoop.yarn.server.federation.store.records.GetReservationHomeSubClusterResponse; + +import org.apache.hadoop.thirdparty.protobuf.TextFormat; + +/** + * Protocol buffer based implementation of + * {@link GetReservationHomeSubClusterResponse}. + */ +@Private +@Unstable +public class GetReservationHomeSubClusterResponsePBImpl + extends GetReservationHomeSubClusterResponse { + + private GetReservationHomeSubClusterResponseProto proto = + GetReservationHomeSubClusterResponseProto.getDefaultInstance(); + private GetReservationHomeSubClusterResponseProto.Builder builder = null; + private boolean viaProto = false; + + public GetReservationHomeSubClusterResponsePBImpl() { + builder = GetReservationHomeSubClusterResponseProto.newBuilder(); + } + + public GetReservationHomeSubClusterResponsePBImpl( + GetReservationHomeSubClusterResponseProto proto) { + this.proto = proto; + viaProto = true; + } + + public GetReservationHomeSubClusterResponseProto getProto() { + mergeLocalToProto(); + proto = viaProto ? proto : builder.build(); + viaProto = true; + return proto; + } + + private void mergeLocalToProto() { + if (viaProto) { + maybeInitBuilder(); + } + mergeLocalToBuilder(); + proto = builder.build(); + viaProto = true; + } + + private void maybeInitBuilder() { + if (viaProto || builder == null) { + builder = GetReservationHomeSubClusterResponseProto.newBuilder(proto); + } + viaProto = false; + } + + private void mergeLocalToBuilder() { + } + + @Override + public int hashCode() { + return getProto().hashCode(); + } + + @Override + public boolean equals(Object other) { + if (other == null) { + return false; + } + if (other.getClass().isAssignableFrom(this.getClass())) { + return this.getProto().equals(this.getClass().cast(other).getProto()); + } + return false; + } + + @Override + public String toString() { + return TextFormat.shortDebugString(getProto()); + } + + @Override + public ReservationHomeSubCluster getReservationHomeSubCluster() { + GetReservationHomeSubClusterResponseProtoOrBuilder p = viaProto ? proto : builder; + if (!p.hasAppSubclusterMap()) { + return null; + } + return convertFromProtoFormat(p.getAppSubclusterMap()); + } + + @Override + public void setReservationHomeSubCluster( + ReservationHomeSubCluster reservationInfo) { + maybeInitBuilder(); + if (reservationInfo == null) { + builder.clearAppSubclusterMap(); + return; + } + builder.setAppSubclusterMap(convertToProtoFormat(reservationInfo)); + } + + private ReservationHomeSubCluster convertFromProtoFormat( + ReservationHomeSubClusterProto sc) { + return new ReservationHomeSubClusterPBImpl(sc); + } + + private ReservationHomeSubClusterProto convertToProtoFormat( + ReservationHomeSubCluster sc) { + return ((ReservationHomeSubClusterPBImpl) sc).getProto(); + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/GetReservationsHomeSubClusterRequestPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/GetReservationsHomeSubClusterRequestPBImpl.java new file mode 100644 index 0000000000000..889941ede1b68 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/GetReservationsHomeSubClusterRequestPBImpl.java @@ -0,0 +1,77 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package org.apache.hadoop.yarn.server.federation.store.records.impl.pb; + +import org.apache.hadoop.classification.InterfaceAudience.Private; +import org.apache.hadoop.classification.InterfaceStability.Unstable; +import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.GetReservationsHomeSubClusterRequestProto; +import org.apache.hadoop.yarn.server.federation.store.records.GetReservationsHomeSubClusterRequest; + +import org.apache.hadoop.thirdparty.protobuf.TextFormat; + +/** + * Protocol buffer based implementation of + * {@link GetReservationsHomeSubClusterRequest}. + */ +@Private +@Unstable +public class GetReservationsHomeSubClusterRequestPBImpl + extends GetReservationsHomeSubClusterRequest { + + private GetReservationsHomeSubClusterRequestProto proto = + GetReservationsHomeSubClusterRequestProto.getDefaultInstance(); + private GetReservationsHomeSubClusterRequestProto.Builder builder = null; + private boolean viaProto = false; + + public GetReservationsHomeSubClusterRequestPBImpl() { + builder = GetReservationsHomeSubClusterRequestProto.newBuilder(); + } + + public GetReservationsHomeSubClusterRequestPBImpl( + GetReservationsHomeSubClusterRequestProto proto) { + this.proto = proto; + viaProto = true; + } + + public GetReservationsHomeSubClusterRequestProto getProto() { + proto = viaProto ? proto : builder.build(); + viaProto = true; + return proto; + } + + @Override + public int hashCode() { + return getProto().hashCode(); + } + + @Override + public boolean equals(Object other) { + if (other == null) { + return false; + } + if (other.getClass().isAssignableFrom(this.getClass())) { + return this.getProto().equals(this.getClass().cast(other).getProto()); + } + return false; + } + + @Override + public String toString() { + return TextFormat.shortDebugString(getProto()); + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/GetReservationsHomeSubClusterResponsePBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/GetReservationsHomeSubClusterResponsePBImpl.java new file mode 100644 index 0000000000000..c380997fa1db9 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/GetReservationsHomeSubClusterResponsePBImpl.java @@ -0,0 +1,181 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package org.apache.hadoop.yarn.server.federation.store.records.impl.pb; + +import java.util.ArrayList; +import java.util.Iterator; +import java.util.List; + +import org.apache.hadoop.classification.InterfaceAudience.Private; +import org.apache.hadoop.classification.InterfaceStability.Unstable; +import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.ReservationHomeSubClusterProto; +import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.GetReservationsHomeSubClusterResponseProto; +import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.GetReservationsHomeSubClusterResponseProtoOrBuilder; +import org.apache.hadoop.yarn.server.federation.store.records.ReservationHomeSubCluster; +import org.apache.hadoop.yarn.server.federation.store.records.GetReservationsHomeSubClusterResponse; + +import org.apache.hadoop.thirdparty.protobuf.TextFormat; + +/** + * Protocol buffer based implementation of + * {@link GetReservationsHomeSubClusterResponse}. + */ +@Private +@Unstable +public class GetReservationsHomeSubClusterResponsePBImpl + extends GetReservationsHomeSubClusterResponse { + + private GetReservationsHomeSubClusterResponseProto proto = + GetReservationsHomeSubClusterResponseProto.getDefaultInstance(); + private GetReservationsHomeSubClusterResponseProto.Builder builder = null; + private boolean viaProto = false; + + private List appsHomeSubCluster; + + public GetReservationsHomeSubClusterResponsePBImpl() { + builder = GetReservationsHomeSubClusterResponseProto.newBuilder(); + } + + public GetReservationsHomeSubClusterResponsePBImpl( + GetReservationsHomeSubClusterResponseProto proto) { + this.proto = proto; + viaProto = true; + } + + public GetReservationsHomeSubClusterResponseProto getProto() { + mergeLocalToProto(); + proto = viaProto ? proto : builder.build(); + viaProto = true; + return proto; + } + + private void mergeLocalToProto() { + if (viaProto) { + maybeInitBuilder(); + } + mergeLocalToBuilder(); + proto = builder.build(); + viaProto = true; + } + + private void maybeInitBuilder() { + if (viaProto || builder == null) { + builder = GetReservationsHomeSubClusterResponseProto.newBuilder(proto); + } + viaProto = false; + } + + private void mergeLocalToBuilder() { + if (this.appsHomeSubCluster != null) { + addSubClustersInfoToProto(); + } + } + + @Override + public int hashCode() { + return getProto().hashCode(); + } + + @Override + public boolean equals(Object other) { + if (other == null) { + return false; + } + if (other.getClass().isAssignableFrom(this.getClass())) { + return this.getProto().equals(this.getClass().cast(other).getProto()); + } + return false; + } + + @Override + public String toString() { + return TextFormat.shortDebugString(getProto()); + } + + @Override + public List getAppsHomeSubClusters() { + initSubClustersInfoList(); + return appsHomeSubCluster; + } + + @Override + public void setAppsHomeSubClusters( + List appsHomeSubClusters) { + maybeInitBuilder(); + if (appsHomeSubClusters == null) { + builder.clearAppSubclusterMap(); + return; + } + this.appsHomeSubCluster = appsHomeSubClusters; + } + + private void initSubClustersInfoList() { + if (this.appsHomeSubCluster != null) { + return; + } + GetReservationsHomeSubClusterResponseProtoOrBuilder p = viaProto ? proto : builder; + List subClusterInfosList = p.getAppSubclusterMapList(); + appsHomeSubCluster = new ArrayList<>(); + + for (ReservationHomeSubClusterProto r : subClusterInfosList) { + appsHomeSubCluster.add(convertFromProtoFormat(r)); + } + } + + private void addSubClustersInfoToProto() { + maybeInitBuilder(); + builder.clearAppSubclusterMap(); + if (appsHomeSubCluster == null) { + return; + } + Iterable iterable = + new Iterable() { + @Override + public Iterator iterator() { + return new Iterator() { + + private Iterator iter = appsHomeSubCluster.iterator(); + + @Override + public boolean hasNext() { + return iter.hasNext(); + } + + @Override + public ReservationHomeSubClusterProto next() { + return convertToProtoFormat(iter.next()); + } + + @Override + public void remove() { + throw new UnsupportedOperationException(); + } + }; + } + }; + builder.addAllAppSubclusterMap(iterable); + } + + private ReservationHomeSubCluster convertFromProtoFormat(ReservationHomeSubClusterProto sc) { + return new ReservationHomeSubClusterPBImpl(sc); + } + + private ReservationHomeSubClusterProto convertToProtoFormat(ReservationHomeSubCluster sc) { + return ((ReservationHomeSubClusterPBImpl) sc).getProto(); + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/GetSubClustersInfoResponsePBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/GetSubClustersInfoResponsePBImpl.java index 5ecc3e249b2fc..271570882f922 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/GetSubClustersInfoResponsePBImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/GetSubClustersInfoResponsePBImpl.java @@ -19,8 +19,10 @@ package org.apache.hadoop.yarn.server.federation.store.records.impl.pb; import java.util.ArrayList; +import java.util.Collection; import java.util.Iterator; import java.util.List; +import java.util.stream.Collectors; import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.classification.InterfaceStability.Unstable; @@ -93,12 +95,12 @@ public List getSubClusters() { } @Override - public void setSubClusters(List subClusters) { + public void setSubClusters(Collection subClusters) { if (subClusters == null) { builder.clearSubClusterInfos(); return; } - this.subClusterInfos = subClusters; + this.subClusterInfos = subClusters.stream().collect(Collectors.toList()); } private void initSubClustersInfoList() { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/ReservationHomeSubClusterPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/ReservationHomeSubClusterPBImpl.java new file mode 100644 index 0000000000000..ba7f9d185ffca --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/ReservationHomeSubClusterPBImpl.java @@ -0,0 +1,168 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package org.apache.hadoop.yarn.server.federation.store.records.impl.pb; + +import org.apache.hadoop.classification.InterfaceAudience.Private; +import org.apache.hadoop.classification.InterfaceStability.Unstable; +import org.apache.hadoop.yarn.api.records.ReservationId; +import org.apache.hadoop.yarn.api.records.impl.pb.ReservationIdPBImpl; +import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.ReservationHomeSubClusterProto; +import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.ReservationHomeSubClusterProtoOrBuilder; +import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.SubClusterIdProto; +import org.apache.hadoop.yarn.proto.YarnProtos.ReservationIdProto; +import org.apache.hadoop.yarn.server.federation.store.records.ReservationHomeSubCluster; +import org.apache.hadoop.yarn.server.federation.store.records.SubClusterId; + +import org.apache.hadoop.thirdparty.protobuf.TextFormat; + +/** + * Protocol buffer based implementation of {@link ReservationHomeSubCluster}. + */ +@Private +@Unstable +public class ReservationHomeSubClusterPBImpl extends ReservationHomeSubCluster { + + private ReservationHomeSubClusterProto proto = + ReservationHomeSubClusterProto.getDefaultInstance(); + private ReservationHomeSubClusterProto.Builder builder = null; + private boolean viaProto = false; + + private ReservationId reservationId = null; + private SubClusterId homeSubCluster = null; + + public ReservationHomeSubClusterPBImpl() { + builder = ReservationHomeSubClusterProto.newBuilder(); + } + + public ReservationHomeSubClusterPBImpl(ReservationHomeSubClusterProto proto) { + this.proto = proto; + viaProto = true; + } + + public ReservationHomeSubClusterProto getProto() { + mergeLocalToProto(); + proto = viaProto ? proto : builder.build(); + viaProto = true; + return proto; + } + + private void mergeLocalToProto() { + if (viaProto) { + maybeInitBuilder(); + } + mergeLocalToBuilder(); + proto = builder.build(); + viaProto = true; + } + + private void maybeInitBuilder() { + if (viaProto || builder == null) { + builder = ReservationHomeSubClusterProto.newBuilder(proto); + } + viaProto = false; + } + + private void mergeLocalToBuilder() { + if (this.reservationId != null) { + builder.setReservationId(convertToProtoFormat(this.reservationId)); + } + if (this.homeSubCluster != null) { + builder.setHomeSubCluster(convertToProtoFormat(this.homeSubCluster)); + } + } + + @Override + public int hashCode() { + return getProto().hashCode(); + } + + @Override + public boolean equals(Object other) { + if (other == null) { + return false; + } + if (other.getClass().isAssignableFrom(this.getClass())) { + return this.getProto().equals(this.getClass().cast(other).getProto()); + } + return false; + } + + @Override + public String toString() { + return TextFormat.shortDebugString(getProto()); + } + + @Override + public ReservationId getReservationId() { + ReservationHomeSubClusterProtoOrBuilder p = viaProto ? proto : builder; + if (!p.hasReservationId()) { + return null; + } + this.reservationId = convertFromProtoFormat(p.getReservationId()); + return this.reservationId; + } + + @Override + public void setReservationId(ReservationId resId) { + maybeInitBuilder(); + if (resId == null) { + builder.clearReservationId(); + return; + } + this.reservationId = resId; + } + + @Override + public SubClusterId getHomeSubCluster() { + ReservationHomeSubClusterProtoOrBuilder p = viaProto ? proto : builder; + if (this.homeSubCluster != null) { + return this.homeSubCluster; + } + if (!p.hasHomeSubCluster()) { + return null; + } + this.homeSubCluster = convertFromProtoFormat(p.getHomeSubCluster()); + return this.homeSubCluster; + } + + @Override + public void setHomeSubCluster(SubClusterId subClusterId) { + maybeInitBuilder(); + if (subClusterId == null) { + builder.clearHomeSubCluster(); + return; + } + this.homeSubCluster = subClusterId; + } + + private SubClusterId convertFromProtoFormat(SubClusterIdProto subClusterId) { + return new SubClusterIdPBImpl(subClusterId); + } + + private SubClusterIdProto convertToProtoFormat(SubClusterId subClusterId) { + return ((SubClusterIdPBImpl) subClusterId).getProto(); + } + + private ReservationId convertFromProtoFormat(ReservationIdProto appId) { + return new ReservationIdPBImpl(appId); + } + + private ReservationIdProto convertToProtoFormat(ReservationId appId) { + return ((ReservationIdPBImpl) appId).getProto(); + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/UpdateReservationHomeSubClusterRequestPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/UpdateReservationHomeSubClusterRequestPBImpl.java new file mode 100644 index 0000000000000..c9d9a5aa0a646 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/UpdateReservationHomeSubClusterRequestPBImpl.java @@ -0,0 +1,131 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package org.apache.hadoop.yarn.server.federation.store.records.impl.pb; + +import org.apache.hadoop.classification.InterfaceAudience.Private; +import org.apache.hadoop.classification.InterfaceStability.Unstable; +import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.ReservationHomeSubClusterProto; +import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.UpdateReservationHomeSubClusterRequestProto; +import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.UpdateReservationHomeSubClusterRequestProtoOrBuilder; +import org.apache.hadoop.yarn.server.federation.store.records.ReservationHomeSubCluster; +import org.apache.hadoop.yarn.server.federation.store.records.UpdateReservationHomeSubClusterRequest; + +import org.apache.hadoop.thirdparty.protobuf.TextFormat; + +/** + * Protocol buffer based implementation of + * {@link UpdateReservationHomeSubClusterRequest} . + */ +@Private +@Unstable +public class UpdateReservationHomeSubClusterRequestPBImpl + extends UpdateReservationHomeSubClusterRequest { + + private UpdateReservationHomeSubClusterRequestProto proto = + UpdateReservationHomeSubClusterRequestProto.getDefaultInstance(); + private UpdateReservationHomeSubClusterRequestProto.Builder builder = null; + private boolean viaProto = false; + + public UpdateReservationHomeSubClusterRequestPBImpl() { + builder = UpdateReservationHomeSubClusterRequestProto.newBuilder(); + } + + public UpdateReservationHomeSubClusterRequestPBImpl( + UpdateReservationHomeSubClusterRequestProto proto) { + this.proto = proto; + viaProto = true; + } + + public UpdateReservationHomeSubClusterRequestProto getProto() { + mergeLocalToProto(); + proto = viaProto ? proto : builder.build(); + viaProto = true; + return proto; + } + + private void mergeLocalToProto() { + if (viaProto) { + maybeInitBuilder(); + } + mergeLocalToBuilder(); + proto = builder.build(); + viaProto = true; + } + + private void maybeInitBuilder() { + if (viaProto || builder == null) { + builder = UpdateReservationHomeSubClusterRequestProto.newBuilder(proto); + } + viaProto = false; + } + + private void mergeLocalToBuilder() { + } + + @Override + public int hashCode() { + return getProto().hashCode(); + } + + @Override + public boolean equals(Object other) { + if (other == null) { + return false; + } + if (other.getClass().isAssignableFrom(this.getClass())) { + return this.getProto().equals(this.getClass().cast(other).getProto()); + } + return false; + } + + @Override + public String toString() { + return TextFormat.shortDebugString(getProto()); + } + + @Override + public ReservationHomeSubCluster getReservationHomeSubCluster() { + UpdateReservationHomeSubClusterRequestProtoOrBuilder p = + viaProto ? proto : builder; + if (!p.hasAppSubclusterMap()) { + return null; + } + return convertFromProtoFormat(p.getAppSubclusterMap()); + } + + @Override + public void setReservationHomeSubCluster( + ReservationHomeSubCluster reservationInfo) { + maybeInitBuilder(); + if (reservationInfo == null) { + builder.clearAppSubclusterMap(); + return; + } + builder.setAppSubclusterMap(convertToProtoFormat(reservationInfo)); + } + + private ReservationHomeSubCluster convertFromProtoFormat( + ReservationHomeSubClusterProto sc) { + return new ReservationHomeSubClusterPBImpl(sc); + } + + private ReservationHomeSubClusterProto convertToProtoFormat( + ReservationHomeSubCluster sc) { + return ((ReservationHomeSubClusterPBImpl) sc).getProto(); + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/UpdateReservationHomeSubClusterResponsePBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/UpdateReservationHomeSubClusterResponsePBImpl.java new file mode 100644 index 0000000000000..16f75480a7065 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/UpdateReservationHomeSubClusterResponsePBImpl.java @@ -0,0 +1,77 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package org.apache.hadoop.yarn.server.federation.store.records.impl.pb; + +import org.apache.hadoop.classification.InterfaceAudience.Private; +import org.apache.hadoop.classification.InterfaceStability.Unstable; +import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.UpdateReservationHomeSubClusterResponseProto; +import org.apache.hadoop.yarn.server.federation.store.records.UpdateReservationHomeSubClusterResponse; + +import org.apache.hadoop.thirdparty.protobuf.TextFormat; + +/** + * Protocol buffer based implementation of + * {@link UpdateReservationHomeSubClusterResponse}. + */ +@Private +@Unstable +public class UpdateReservationHomeSubClusterResponsePBImpl + extends UpdateReservationHomeSubClusterResponse { + + private UpdateReservationHomeSubClusterResponseProto proto = + UpdateReservationHomeSubClusterResponseProto.getDefaultInstance(); + private UpdateReservationHomeSubClusterResponseProto.Builder builder = null; + private boolean viaProto = false; + + public UpdateReservationHomeSubClusterResponsePBImpl() { + builder = UpdateReservationHomeSubClusterResponseProto.newBuilder(); + } + + public UpdateReservationHomeSubClusterResponsePBImpl( + UpdateReservationHomeSubClusterResponseProto proto) { + this.proto = proto; + viaProto = true; + } + + public UpdateReservationHomeSubClusterResponseProto getProto() { + proto = viaProto ? proto : builder.build(); + viaProto = true; + return proto; + } + + @Override + public int hashCode() { + return getProto().hashCode(); + } + + @Override + public boolean equals(Object other) { + if (other == null) { + return false; + } + if (other.getClass().isAssignableFrom(this.getClass())) { + return this.getProto().equals(this.getClass().cast(other).getProto()); + } + return false; + } + + @Override + public String toString() { + return TextFormat.shortDebugString(getProto()); + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/utils/FederationReservationHomeSubClusterStoreInputValidator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/utils/FederationReservationHomeSubClusterStoreInputValidator.java new file mode 100644 index 0000000000000..d792ca6b668ec --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/utils/FederationReservationHomeSubClusterStoreInputValidator.java @@ -0,0 +1,171 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.federation.store.utils; + +import org.apache.hadoop.yarn.api.records.ReservationId; +import org.apache.hadoop.yarn.server.federation.store.exception.FederationStateStoreInvalidInputException; +import org.apache.hadoop.yarn.server.federation.store.records.AddReservationHomeSubClusterRequest; +import org.apache.hadoop.yarn.server.federation.store.records.ReservationHomeSubCluster; +import org.apache.hadoop.yarn.server.federation.store.records.GetReservationHomeSubClusterRequest; +import org.apache.hadoop.yarn.server.federation.store.records.UpdateReservationHomeSubClusterRequest; +import org.apache.hadoop.yarn.server.federation.store.records.DeleteReservationHomeSubClusterRequest; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Utility class to validate the inputs to + * {@code FederationReservationHomeSubClusterStore}, allows a fail fast + * mechanism for invalid user inputs. + * + */ +public final class FederationReservationHomeSubClusterStoreInputValidator { + + private static final Logger LOG = LoggerFactory + .getLogger(FederationReservationHomeSubClusterStoreInputValidator.class); + + private FederationReservationHomeSubClusterStoreInputValidator() { + } + + /** + * Quick validation on the input to check some obvious fail conditions (fail + * fast). Check if the provided {@link AddReservationHomeSubClusterRequest} + * for adding a new reservation is valid or not. + * + * @param request the {@link AddReservationHomeSubClusterRequest} to validate against + * @throws FederationStateStoreInvalidInputException if the request is invalid + */ + public static void validate(AddReservationHomeSubClusterRequest request) + throws FederationStateStoreInvalidInputException { + if (request == null) { + String message = "Missing AddReservationHomeSubCluster Request." + + " Please try again by specifying" + + " an AddReservationHomeSubCluster information."; + LOG.warn(message); + throw new FederationStateStoreInvalidInputException(message); + } + + // validate ReservationHomeSubCluster info + checkReservationHomeSubCluster(request.getReservationHomeSubCluster()); + } + + /** + * Quick validation on the input to check some obvious fail conditions (fail + * fast). Check if the provided {@link GetReservationHomeSubClusterRequest} + * for querying reservation's information is valid or not. + * + * @param request the {@link GetReservationHomeSubClusterRequest} to validate against + * @throws FederationStateStoreInvalidInputException if the request is invalid + */ + public static void validate(GetReservationHomeSubClusterRequest request) + throws FederationStateStoreInvalidInputException { + if (request == null) { + String message = "Missing GetReservationHomeSubCluster Request." + + " Please try again by specifying an Reservation Id information."; + LOG.warn(message); + throw new FederationStateStoreInvalidInputException(message); + } + + // validate Reservation Id + checkReservationId(request.getReservationId()); + } + + /** + * Validate if the ReservationHomeSubCluster info are present or not. + * + * @param reservationHomeSubCluster the information of the Reservation to be verified + * @throws FederationStateStoreInvalidInputException if the SubCluster Info are invalid + */ + private static void checkReservationHomeSubCluster( + ReservationHomeSubCluster reservationHomeSubCluster) + throws FederationStateStoreInvalidInputException { + if (reservationHomeSubCluster == null) { + String message = "Missing ReservationHomeSubCluster Info." + + " Please try again by specifying" + + " an ReservationHomeSubCluster information."; + LOG.warn(message); + throw new FederationStateStoreInvalidInputException(message); + } + + // validate Reservation Id + checkReservationId(reservationHomeSubCluster.getReservationId()); + + // validate subcluster Id + FederationMembershipStateStoreInputValidator + .checkSubClusterId(reservationHomeSubCluster.getHomeSubCluster()); + } + + /** + * Validate if the Reservation id is present or not. + * + * @param reservationId the id of the Reservation to be verified + * @throws FederationStateStoreInvalidInputException if the Reservation Id is invalid + */ + private static void checkReservationId(ReservationId reservationId) + throws FederationStateStoreInvalidInputException { + if (reservationId == null) { + String message = "Missing ReservationId. Please try again by specifying an ReservationId."; + LOG.warn(message); + throw new FederationStateStoreInvalidInputException(message); + } + } + + /** + * Quick validation on the input to check some obvious fail conditions (fail + * fast). Check if the provided {@link UpdateReservationHomeSubClusterRequest} + * for updating an reservation is valid or not. + * + * @param request the {@link UpdateReservationHomeSubClusterRequest} to + * validate against + * @throws FederationStateStoreInvalidInputException if the request is invalid + */ + public static void validate(UpdateReservationHomeSubClusterRequest request) + throws FederationStateStoreInvalidInputException { + if (request == null) { + String message = "Missing UpdateReservationHomeSubCluster Request." + + " Please try again by specifying an ReservationHomeSubCluster information."; + LOG.warn(message); + throw new FederationStateStoreInvalidInputException(message); + } + + // validate ReservationHomeSubCluster info + checkReservationHomeSubCluster(request.getReservationHomeSubCluster()); + } + + /** + * Quick validation on the input to check some obvious fail conditions (fail + * fast). Check if the provided {@link DeleteReservationHomeSubClusterRequest} + * for deleting an Reservation is valid or not. + * + * @param request the {@link DeleteReservationHomeSubClusterRequest} to + * validate against + * @throws FederationStateStoreInvalidInputException if the request is invalid + */ + public static void validate(DeleteReservationHomeSubClusterRequest request) + throws FederationStateStoreInvalidInputException { + if (request == null) { + String message = "Missing DeleteReservationHomeSubCluster Request." + + " Please try again by specifying an ReservationHomeSubCluster information."; + LOG.warn(message); + throw new FederationStateStoreInvalidInputException(message); + } + + // validate Reservation Id + checkReservationId(request.getReservationId()); + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/utils/FederationStateStoreUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/utils/FederationStateStoreUtils.java index 27a4f7dba5327..7dc53f8e0acfc 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/utils/FederationStateStoreUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/utils/FederationStateStoreUtils.java @@ -145,6 +145,22 @@ public static void logAndThrowStoreException(Logger log, String errMsg) throw new FederationStateStoreException(errMsg); } + /** + * Throws an FederationStateStoreException due to an error in + * FederationStateStore. + * + * @param log the logger interface + * @param errMsgFormat the error message format string. + * @param args referenced by the format specifiers in the format string. + * @throws YarnException on failure + */ + public static void logAndThrowStoreException(Logger log, String errMsgFormat, Object... args) + throws YarnException { + String errMsg = String.format(errMsgFormat, args); + log.error(errMsg); + throw new FederationStateStoreException(errMsg); + } + /** * Throws an FederationStateStoreInvalidInputException due to an * error in FederationStateStore. @@ -179,6 +195,44 @@ public static void logAndThrowRetriableException(Logger log, String errMsg, } } + /** + * Throws an FederationStateStoreRetriableException due to an + * error in FederationStateStore. + * + * @param t the throwable raised in the called class. + * @param log the logger interface. + * @param errMsgFormat the error message format string. + * @param args referenced by the format specifiers in the format string. + * @throws YarnException on failure + */ + public static void logAndThrowRetriableException( + Throwable t, Logger log, String errMsgFormat, Object... args) throws YarnException { + String errMsg = String.format(errMsgFormat, args); + if (t != null) { + log.error(errMsg, t); + throw new FederationStateStoreRetriableException(errMsg, t); + } else { + log.error(errMsg); + throw new FederationStateStoreRetriableException(errMsg); + } + } + + /** + * Throws an FederationStateStoreRetriableException due to an + * error in FederationStateStore. + * + * @param log the logger interface. + * @param errMsgFormat the error message format string. + * @param args referenced by the format specifiers in the format string. + * @throws YarnException on failure + */ + public static void logAndThrowRetriableException( + Logger log, String errMsgFormat, Object... args) throws YarnException { + String errMsg = String.format(errMsgFormat, args); + log.error(errMsg); + throw new FederationStateStoreRetriableException(errMsg); + } + /** * Sets a specific value for a specific property of * HikariDataSource SQL connections. diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/utils/FederationStateStoreFacade.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/utils/FederationStateStoreFacade.java index fa3c8b8d9819f..d95be57576665 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/utils/FederationStateStoreFacade.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/utils/FederationStateStoreFacade.java @@ -43,6 +43,7 @@ import org.apache.hadoop.io.retry.RetryProxy; import org.apache.hadoop.util.ReflectionUtils; import org.apache.hadoop.yarn.api.records.ApplicationId; +import org.apache.hadoop.yarn.api.records.ReservationId; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.exceptions.YarnRuntimeException; @@ -51,9 +52,13 @@ import org.apache.hadoop.yarn.server.federation.store.exception.FederationStateStoreRetriableException; import org.apache.hadoop.yarn.server.federation.store.records.AddApplicationHomeSubClusterRequest; import org.apache.hadoop.yarn.server.federation.store.records.AddApplicationHomeSubClusterResponse; +import org.apache.hadoop.yarn.server.federation.store.records.AddReservationHomeSubClusterRequest; +import org.apache.hadoop.yarn.server.federation.store.records.AddReservationHomeSubClusterResponse; import org.apache.hadoop.yarn.server.federation.store.records.ApplicationHomeSubCluster; import org.apache.hadoop.yarn.server.federation.store.records.GetApplicationHomeSubClusterRequest; import org.apache.hadoop.yarn.server.federation.store.records.GetApplicationHomeSubClusterResponse; +import org.apache.hadoop.yarn.server.federation.store.records.GetReservationHomeSubClusterRequest; +import org.apache.hadoop.yarn.server.federation.store.records.GetReservationHomeSubClusterResponse; import org.apache.hadoop.yarn.server.federation.store.records.GetSubClusterInfoRequest; import org.apache.hadoop.yarn.server.federation.store.records.GetSubClusterInfoResponse; import org.apache.hadoop.yarn.server.federation.store.records.GetSubClusterPoliciesConfigurationsRequest; @@ -62,10 +67,13 @@ import org.apache.hadoop.yarn.server.federation.store.records.GetSubClusterPolicyConfigurationResponse; import org.apache.hadoop.yarn.server.federation.store.records.GetSubClustersInfoRequest; import org.apache.hadoop.yarn.server.federation.store.records.GetSubClustersInfoResponse; +import org.apache.hadoop.yarn.server.federation.store.records.ReservationHomeSubCluster; import org.apache.hadoop.yarn.server.federation.store.records.SubClusterId; import org.apache.hadoop.yarn.server.federation.store.records.SubClusterInfo; import org.apache.hadoop.yarn.server.federation.store.records.SubClusterPolicyConfiguration; import org.apache.hadoop.yarn.server.federation.store.records.UpdateApplicationHomeSubClusterRequest; +import org.apache.hadoop.yarn.server.federation.store.records.UpdateReservationHomeSubClusterRequest; +import org.apache.hadoop.yarn.server.federation.store.records.DeleteReservationHomeSubClusterRequest; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -86,6 +94,8 @@ public final class FederationStateStoreFacade { private static final String GET_SUBCLUSTERS_CACHEID = "getSubClusters"; private static final String GET_POLICIES_CONFIGURATIONS_CACHEID = "getPoliciesConfigurations"; + private static final String GET_APPLICATION_HOME_SUBCLUSTER_CACHEID = + "getApplicationHomeSubCluster"; private static final FederationStateStoreFacade FACADE = new FederationStateStoreFacade(); @@ -376,10 +386,19 @@ public void updateApplicationHomeSubCluster( */ public SubClusterId getApplicationHomeSubCluster(ApplicationId appId) throws YarnException { - GetApplicationHomeSubClusterResponse response = - stateStore.getApplicationHomeSubCluster( + try { + if (isCachingEnabled()) { + SubClusterId value = SubClusterId.class.cast( + cache.get(buildGetApplicationHomeSubClusterRequest(appId))); + return value; + } else { + GetApplicationHomeSubClusterResponse response = stateStore.getApplicationHomeSubCluster( GetApplicationHomeSubClusterRequest.newInstance(appId)); - return response.getApplicationHomeSubCluster().getHomeSubCluster(); + return response.getApplicationHomeSubCluster().getHomeSubCluster(); + } + } catch (Throwable ex) { + throw new YarnException(ex); + } } /** @@ -400,6 +419,63 @@ public Configuration getConf() { return this.conf; } + /** + * Adds the home {@link SubClusterId} for the specified {@link ReservationId}. + * + * @param appHomeSubCluster the mapping of the reservation to it's home + * sub-cluster + * @return the stored subCluster from StateStore + * @throws YarnException if the call to the state store is unsuccessful + */ + public SubClusterId addReservationHomeSubCluster(ReservationHomeSubCluster appHomeSubCluster) + throws YarnException { + AddReservationHomeSubClusterResponse response = stateStore.addReservationHomeSubCluster( + AddReservationHomeSubClusterRequest.newInstance(appHomeSubCluster)); + return response.getHomeSubCluster(); + } + + /** + * Returns the home {@link SubClusterId} for the specified {@link ReservationId}. + * + * @param reservationId the identifier of the reservation + * @return the home subCluster identifier + * @throws YarnException if the call to the state store is unsuccessful + */ + public SubClusterId getReservationHomeSubCluster(ReservationId reservationId) + throws YarnException { + GetReservationHomeSubClusterResponse response = stateStore.getReservationHomeSubCluster( + GetReservationHomeSubClusterRequest.newInstance(reservationId)); + return response.getReservationHomeSubCluster().getHomeSubCluster(); + } + + /** + * Updates the home {@link SubClusterId} for the specified + * {@link ReservationId}. + * + * @param appHomeSubCluster the mapping of the reservation to it's home + * sub-cluster + * @throws YarnException if the call to the state store is unsuccessful + */ + public void updateReservationHomeSubCluster(ReservationHomeSubCluster appHomeSubCluster) + throws YarnException { + UpdateReservationHomeSubClusterRequest request = + UpdateReservationHomeSubClusterRequest.newInstance(appHomeSubCluster); + stateStore.updateReservationHomeSubCluster(request); + } + + /** + * Delete the home {@link SubClusterId} for the specified + * {@link ReservationId}. + * + * @param reservationId the identifier of the reservation + * @throws YarnException if the call to the state store is unsuccessful + */ + public void deleteReservationHomeSubCluster(ReservationId reservationId) throws YarnException { + DeleteReservationHomeSubClusterRequest request = + DeleteReservationHomeSubClusterRequest.newInstance(reservationId); + stateStore.deleteReservationHomeSubCluster(request); + } + /** * Helper method to create instances of Object using the class name defined in * the configuration object. The instances creates {@link RetryProxy} using @@ -513,6 +589,26 @@ public Map invoke( return cacheRequest; } + private Object buildGetApplicationHomeSubClusterRequest(ApplicationId applicationId) { + final String cacheKey = buildCacheKey(getClass().getSimpleName(), + GET_APPLICATION_HOME_SUBCLUSTER_CACHEID, applicationId.toString()); + CacheRequest cacheRequest = new CacheRequest<>( + cacheKey, + input -> { + + GetApplicationHomeSubClusterRequest request = + GetApplicationHomeSubClusterRequest.newInstance(applicationId); + GetApplicationHomeSubClusterResponse response = + stateStore.getApplicationHomeSubCluster(request); + + ApplicationHomeSubCluster appHomeSubCluster = response.getApplicationHomeSubCluster(); + SubClusterId subClusterId = appHomeSubCluster.getHomeSubCluster(); + + return subClusterId; + }); + return cacheRequest; + } + protected String buildCacheKey(String typeName, String methodName, String argName) { StringBuilder buffer = new StringBuilder(); @@ -560,7 +656,7 @@ private static class CacheRequest { private K key; private Func func; - public CacheRequest(K key, Func func) { + CacheRequest(K key, Func func) { this.key = key; this.func = func; } @@ -609,4 +705,19 @@ public boolean equals(Object obj) { protected interface Func { TResult invoke(T input) throws Exception; } + + @VisibleForTesting + public Cache getCache() { + return cache; + } + + @VisibleForTesting + protected Object getAppHomeSubClusterCacheRequest(ApplicationId applicationId) { + return buildGetApplicationHomeSubClusterRequest(applicationId); + } + + @VisibleForTesting + public FederationStateStore getStateStore() { + return stateStore; + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/uam/UnmanagedAMPoolManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/uam/UnmanagedAMPoolManager.java index 47203b257eb4d..1b72e4065d37c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/uam/UnmanagedAMPoolManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/uam/UnmanagedAMPoolManager.java @@ -22,13 +22,15 @@ import java.util.HashSet; import java.util.Map; import java.util.Set; -import java.util.concurrent.Callable; +import java.util.HashMap; +import java.util.Collections; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ExecutorCompletionService; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.Future; +import org.apache.commons.lang3.tuple.Pair; import org.apache.hadoop.classification.InterfaceAudience.Public; import org.apache.hadoop.classification.InterfaceStability.Unstable; import org.apache.hadoop.conf.Configuration; @@ -74,6 +76,10 @@ public class UnmanagedAMPoolManager extends AbstractService { private ExecutorService threadpool; + private String dispatcherThreadName = "UnmanagedAMPoolManager-Finish-Thread"; + + private Thread finishApplicationThread; + public UnmanagedAMPoolManager(ExecutorService threadpool) { super(UnmanagedAMPoolManager.class.getName()); this.threadpool = threadpool; @@ -94,48 +100,16 @@ protected void serviceStart() throws Exception { * UAMs running, force kill all of them. Do parallel kill because of * performance reasons. * - * TODO: move waiting for the kill to finish into a separate thread, without - * blocking the serviceStop. */ @Override protected void serviceStop() throws Exception { - ExecutorCompletionService completionService = - new ExecutorCompletionService<>(this.threadpool); - if (this.unmanagedAppMasterMap.isEmpty()) { - return; - } - - // Save a local copy of the key set so that it won't change with the map - Set addressList = - new HashSet<>(this.unmanagedAppMasterMap.keySet()); - LOG.warn("Abnormal shutdown of UAMPoolManager, still {} UAMs in map", - addressList.size()); - for (final String uamId : addressList) { - completionService.submit(new Callable() { - @Override - public KillApplicationResponse call() throws Exception { - try { - LOG.info("Force-killing UAM id " + uamId + " for application " - + appIdMap.get(uamId)); - return unmanagedAppMasterMap.remove(uamId).forceKillApplication(); - } catch (Exception e) { - LOG.error("Failed to kill unmanaged application master", e); - return null; - } - } - }); + if (!this.unmanagedAppMasterMap.isEmpty()) { + finishApplicationThread = new Thread(createForceFinishApplicationThread()); + finishApplicationThread.setName(dispatcherThreadName); + finishApplicationThread.start(); } - for (int i = 0; i < addressList.size(); ++i) { - try { - Future future = completionService.take(); - future.get(); - } catch (Exception e) { - LOG.error("Failed to kill unmanaged application master", e); - } - } - this.appIdMap.clear(); super.serviceStop(); } @@ -450,4 +424,111 @@ public void drainUAMHeartbeats() { uam.drainHeartbeatThread(); } } + + /** + * Complete FinishApplicationMaster interface calls in batches. + * + * @param request FinishApplicationMasterRequest + * @param appId application Id + * @return Returns the Map map, + * the key is subClusterId, the value is FinishApplicationMasterResponse + */ + public Map batchFinishApplicationMaster( + FinishApplicationMasterRequest request, String appId) { + + Map responseMap = new HashMap<>(); + Set subClusterIds = this.unmanagedAppMasterMap.keySet(); + + if (subClusterIds != null && !subClusterIds.isEmpty()) { + ExecutorCompletionService> finishAppService = + new ExecutorCompletionService<>(this.threadpool); + LOG.info("Sending finish application request to {} sub-cluster RMs", subClusterIds.size()); + + for (final String subClusterId : subClusterIds) { + finishAppService.submit(() -> { + LOG.info("Sending finish application request to RM {}", subClusterId); + try { + FinishApplicationMasterResponse uamResponse = + finishApplicationMaster(subClusterId, request); + return Collections.singletonMap(subClusterId, uamResponse); + } catch (Throwable e) { + LOG.warn("Failed to finish unmanaged application master: " + + " RM address: {} ApplicationId: {}", subClusterId, appId, e); + return Collections.singletonMap(subClusterId, null); + } + }); + } + + for (int i = 0; i < subClusterIds.size(); ++i) { + try { + Future> future = finishAppService.take(); + Map uamResponse = future.get(); + LOG.debug("Received finish application response from RM: {}", uamResponse.keySet()); + responseMap.putAll(uamResponse); + } catch (Throwable e) { + LOG.warn("Failed to finish unmanaged application master: ApplicationId: {}", appId, e); + } + } + } + + return responseMap; + } + + Runnable createForceFinishApplicationThread() { + return () -> { + + ExecutorCompletionService> completionService = + new ExecutorCompletionService<>(threadpool); + + // Save a local copy of the key set so that it won't change with the map + Set addressList = new HashSet<>(unmanagedAppMasterMap.keySet()); + + LOG.warn("Abnormal shutdown of UAMPoolManager, still {} UAMs in map", addressList.size()); + + for (final String uamId : addressList) { + completionService.submit(() -> { + try { + ApplicationId appId = appIdMap.get(uamId); + LOG.info("Force-killing UAM id {} for application {}", uamId, appId); + UnmanagedApplicationManager applicationManager = unmanagedAppMasterMap.remove(uamId); + KillApplicationResponse response = applicationManager.forceKillApplication(); + return Pair.of(uamId, response); + } catch (Exception e) { + LOG.error("Failed to kill unmanaged application master", e); + return Pair.of(uamId, null); + } + }); + } + + for (int i = 0; i < addressList.size(); ++i) { + try { + Future> future = completionService.take(); + Pair pairs = future.get(); + String uamId = pairs.getLeft(); + ApplicationId appId = appIdMap.get(uamId); + KillApplicationResponse response = pairs.getRight(); + if (response == null) { + throw new YarnException( + "Failed Force-killing UAM id " + uamId + " for application " + appId); + } + LOG.info("Force-killing UAM id = {} for application {} KillCompleted {}.", + uamId, appId, response.getIsKillCompleted()); + } catch (Exception e) { + LOG.error("Failed to kill unmanaged application master", e); + } + } + + appIdMap.clear(); + }; + } + + @VisibleForTesting + protected Map getUnmanagedAppMasterMap() { + return unmanagedAppMasterMap; + } + + @VisibleForTesting + protected Thread getFinishApplicationThread() { + return finishApplicationThread; + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/uam/UnmanagedApplicationManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/uam/UnmanagedApplicationManager.java index 61ea864af7190..c70a2db25f161 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/uam/UnmanagedApplicationManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/uam/UnmanagedApplicationManager.java @@ -382,8 +382,13 @@ protected T createRMProxy(Class protocol, Configuration config, protected Token initializeUnmanagedAM( ApplicationId appId) throws IOException, YarnException { try { - UserGroupInformation appSubmitter = - UserGroupInformation.createRemoteUser(this.submitter); + UserGroupInformation appSubmitter; + if (UserGroupInformation.isSecurityEnabled()) { + appSubmitter = UserGroupInformation.createProxyUser(this.submitter, + UserGroupInformation.getLoginUser()); + } else { + appSubmitter = UserGroupInformation.createRemoteUser(this.submitter); + } this.rmClient = createRMProxy(ApplicationClientProtocol.class, this.conf, appSubmitter, null); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/YarnServerBuilderUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/YarnServerBuilderUtils.java index 9ee68d12c7797..6a5d22affae8c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/YarnServerBuilderUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/YarnServerBuilderUtils.java @@ -79,7 +79,7 @@ public static NodeHeartbeatResponse newNodeHeartbeatResponse(int responseId, * * @param applicationId Application ID * @param credentials HDFS Tokens - * @return systemCredentialsForAppsProto SystemCredentialsForAppsProto + * @return systemCredentialsForAppsProto */ public static SystemCredentialsForAppsProto newSystemCredentialsForAppsProto( ApplicationId applicationId, ByteBuffer credentials) { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/dao/ContainersInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/dao/ContainersInfo.java index 545f84e3232cb..a1627152d5f9b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/dao/ContainersInfo.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/dao/ContainersInfo.java @@ -18,6 +18,7 @@ package org.apache.hadoop.yarn.server.webapp.dao; import java.util.ArrayList; +import java.util.Collection; import javax.xml.bind.annotation.XmlAccessType; import javax.xml.bind.annotation.XmlAccessorType; @@ -46,4 +47,7 @@ public ArrayList getContainers() { return container; } + public void addAll(Collection containersInfo) { + container.addAll(containersInfo); + } } \ No newline at end of file diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/proto/yarn_server_federation_protos.proto b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/proto/yarn_server_federation_protos.proto index 114a60df87b6a..33f5cb3fc14e0 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/proto/yarn_server_federation_protos.proto +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/proto/yarn_server_federation_protos.proto @@ -137,6 +137,56 @@ message DeleteApplicationHomeSubClusterRequestProto { message DeleteApplicationHomeSubClusterResponseProto { } + +// ---- reservations ---- + +message ReservationHomeSubClusterProto { + optional ReservationIdProto reservation_id = 1; + optional SubClusterIdProto home_sub_cluster = 2; +} + +message AddReservationHomeSubClusterRequestProto { + optional ReservationHomeSubClusterProto app_subcluster_map = 1; +} + +message AddReservationHomeSubClusterResponseProto { + optional SubClusterIdProto home_sub_cluster = 1; +} + +message UpdateReservationHomeSubClusterRequestProto { + optional ReservationHomeSubClusterProto app_subcluster_map = 1; +} + +message UpdateReservationHomeSubClusterResponseProto { +} + +message GetReservationHomeSubClusterRequestProto { + optional ReservationIdProto reservation_id = 1; +} + +message GetReservationHomeSubClusterResponseProto { + optional ReservationHomeSubClusterProto app_subcluster_map = 1; +} + +message GetReservationsHomeSubClusterRequestProto { + +} + +message GetReservationsHomeSubClusterResponseProto { + repeated ReservationHomeSubClusterProto app_subcluster_map = 1; +} + + +message DeleteReservationHomeSubClusterRequestProto { + optional ReservationIdProto reservation_id = 1; +} + +message DeleteReservationHomeSubClusterResponseProto { +} + + +//----- configurations --- + message SubClusterPolicyConfigurationProto { optional string queue = 1; optional string type = 2; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/BaseFederationPoliciesTest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/BaseFederationPoliciesTest.java index 249efd324b4a0..d9ebd2f1c5e92 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/BaseFederationPoliciesTest.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/BaseFederationPoliciesTest.java @@ -19,7 +19,6 @@ package org.apache.hadoop.yarn.server.federation.policies; import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; import java.nio.ByteBuffer; import java.util.HashMap; @@ -28,20 +27,26 @@ import java.util.Map; import java.util.Random; +import org.apache.hadoop.yarn.api.protocolrecords.ReservationSubmissionRequest; import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext; import org.apache.hadoop.yarn.api.records.ResourceRequest; +import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.server.federation.policies.amrmproxy.FederationAMRMProxyPolicy; import org.apache.hadoop.yarn.server.federation.policies.dao.WeightedPolicyInfo; import org.apache.hadoop.yarn.server.federation.policies.exceptions.FederationPolicyException; import org.apache.hadoop.yarn.server.federation.policies.exceptions.FederationPolicyInitializationException; import org.apache.hadoop.yarn.server.federation.policies.router.FederationRouterPolicy; +import org.apache.hadoop.yarn.server.federation.store.FederationStateStore; +import org.apache.hadoop.yarn.server.federation.store.impl.MemoryFederationStateStore; import org.apache.hadoop.yarn.server.federation.store.records.SubClusterId; import org.apache.hadoop.yarn.server.federation.store.records.SubClusterIdInfo; import org.apache.hadoop.yarn.server.federation.store.records.SubClusterInfo; import org.apache.hadoop.yarn.server.federation.store.records.SubClusterPolicyConfiguration; import org.apache.hadoop.yarn.server.federation.store.records.SubClusterState; +import org.apache.hadoop.yarn.server.federation.store.records.SubClusterRegisterRequest; import org.apache.hadoop.yarn.server.federation.utils.FederationPoliciesTestUtil; +import org.apache.hadoop.yarn.server.federation.utils.FederationStateStoreFacade; import org.junit.Test; /** @@ -58,6 +63,9 @@ public abstract class BaseFederationPoliciesTest { private Random rand = new Random(); private SubClusterId homeSubCluster; + private ReservationSubmissionRequest reservationSubmissionRequest = + mock(ReservationSubmissionRequest.class); + @Test public void testReinitilialize() throws YarnException { FederationPolicyInitializationContext fpc = @@ -177,11 +185,60 @@ public void setHomeSubCluster(SubClusterId homeSubCluster) { public void setMockActiveSubclusters(int numSubclusters) { for (int i = 1; i <= numSubclusters; i++) { SubClusterIdInfo sc = new SubClusterIdInfo("sc" + i); - SubClusterInfo sci = mock(SubClusterInfo.class); - when(sci.getState()).thenReturn(SubClusterState.SC_RUNNING); - when(sci.getSubClusterId()).thenReturn(sc.toId()); + SubClusterInfo sci = SubClusterInfo.newInstance( + sc.toId(), "dns1:80", "dns1:81", "dns1:82", "dns1:83", SubClusterState.SC_RUNNING, + System.currentTimeMillis(), "something"); getActiveSubclusters().put(sc.toId(), sci); } } + public String generateClusterMetricsInfo(int id) { + long mem = 1024 * getRand().nextInt(277 * 100 - 1); + // plant a best cluster + if (id == 5) { + mem = 1024 * 277 * 100; + } + String clusterMetrics = + "{\"clusterMetrics\":{\"appsSubmitted\":65, \"appsCompleted\":64,\"appsPending\":0," + + "\"appsRunning\":0, \"appsFailed\":0, \"appsKilled\":1,\"reservedMB\":0,\"availableMB\":" + + mem + ", \"allocatedMB\":0,\"reservedVirtualCores\":0, \"availableVirtualCores\":2216," + + "\"allocatedVirtualCores\":0, \"containersAllocated\":0,\"containersReserved\":0," + + "\"containersPending\":0,\"totalMB\":28364800, \"totalVirtualCores\":2216," + + "\"totalNodes\":278, \"lostNodes\":1,\"unhealthyNodes\":0,\"decommissionedNodes\":0, " + + "\"rebootedNodes\":0, \"activeNodes\":277}}"; + return clusterMetrics; + } + + public FederationStateStoreFacade getMemoryFacade() throws YarnException { + + // setting up a store and its facade (with caching off) + FederationStateStoreFacade fedFacade = FederationStateStoreFacade.getInstance(); + YarnConfiguration conf = new YarnConfiguration(); + conf.setInt(YarnConfiguration.FEDERATION_CACHE_TIME_TO_LIVE_SECS, 0); + FederationStateStore store = new MemoryFederationStateStore(); + store.init(conf); + fedFacade.reinitialize(store, conf); + + for (SubClusterInfo sinfo : getActiveSubclusters().values()) { + store.registerSubCluster(SubClusterRegisterRequest.newInstance(sinfo)); + } + + return fedFacade; + } + + public ReservationSubmissionRequest getReservationSubmissionRequest() { + return reservationSubmissionRequest; + } + + public void setReservationSubmissionRequest( + ReservationSubmissionRequest reservationSubmissionRequest) { + this.reservationSubmissionRequest = reservationSubmissionRequest; + } + + public void setupContext() throws YarnException { + FederationPolicyInitializationContext context = + FederationPoliciesTestUtil.initializePolicyContext2(getPolicy(), + getPolicyInfo(), getActiveSubclusters(), getMemoryFacade()); + this.setFederationPolicyContext(context); + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/router/BaseRouterPoliciesTest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/router/BaseRouterPoliciesTest.java index d09ba754d55a7..b73578c6deec7 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/router/BaseRouterPoliciesTest.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/router/BaseRouterPoliciesTest.java @@ -19,23 +19,35 @@ package org.apache.hadoop.yarn.server.federation.policies.router; import java.util.ArrayList; +import java.util.Collections; import java.util.List; import java.util.Map; import java.util.Random; +import org.apache.hadoop.test.LambdaTestUtils; +import org.apache.hadoop.util.Time; +import org.apache.hadoop.yarn.api.protocolrecords.ReservationSubmissionRequest; +import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext; +import org.apache.hadoop.yarn.api.records.Priority; +import org.apache.hadoop.yarn.api.records.ReservationId; import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.server.federation.policies.BaseFederationPoliciesTest; +import org.apache.hadoop.yarn.server.federation.policies.FederationPolicyInitializationContext; import org.apache.hadoop.yarn.server.federation.policies.FederationPolicyUtils; import org.apache.hadoop.yarn.server.federation.policies.exceptions.FederationPolicyException; import org.apache.hadoop.yarn.server.federation.store.records.SubClusterId; import org.apache.hadoop.yarn.server.federation.store.records.SubClusterIdInfo; import org.apache.hadoop.yarn.server.federation.store.records.SubClusterInfo; +import org.apache.hadoop.yarn.server.federation.store.records.ReservationHomeSubCluster; import org.apache.hadoop.yarn.server.federation.utils.FederationPoliciesTestUtil; +import org.apache.hadoop.yarn.server.federation.utils.FederationStateStoreFacade; import org.apache.hadoop.yarn.util.resource.Resources; import org.junit.Assert; import org.junit.Test; +import static org.mockito.Mockito.when; + /** * Base class for router policies tests, tests for null input cases. */ @@ -115,4 +127,143 @@ public void testAllBlacklistSubcluster() throws YarnException { } } } + + @Test + public void testNullReservationContext() throws Exception { + FederationRouterPolicy policy = ((FederationRouterPolicy) getPolicy()); + + LambdaTestUtils.intercept(FederationPolicyException.class, + "The ReservationSubmissionRequest cannot be null.", + () -> policy.getReservationHomeSubcluster(null)); + } + + @Test + public void testUnknownReservation() throws Exception { + + long now = Time.now(); + ReservationSubmissionRequest resReq = getReservationSubmissionRequest(); + ReservationId reservationId = ReservationId.newInstance(now, 1); + when(resReq.getQueue()).thenReturn("queue1"); + when(resReq.getReservationId()).thenReturn(reservationId); + + // route an application that uses this app + ApplicationSubmissionContext applicationSubmissionContext = + ApplicationSubmissionContext.newInstance( + ApplicationId.newInstance(now, 1), "app1", "queue1", Priority.newInstance(1), + null, false, false, 1, null, null, false); + + applicationSubmissionContext.setReservationID(resReq.getReservationId()); + FederationRouterPolicy policy = (FederationRouterPolicy) getPolicy(); + + LambdaTestUtils.intercept(YarnException.class, + "Reservation " + reservationId + " does not exist", + () -> policy.getHomeSubcluster(applicationSubmissionContext, new ArrayList<>())); + } + + @Test + public void testFollowReservation() throws YarnException { + + long now = Time.now(); + ReservationSubmissionRequest resReq = getReservationSubmissionRequest(); + when(resReq.getQueue()).thenReturn("queue1"); + when(resReq.getReservationId()).thenReturn(ReservationId.newInstance(now, 1)); + + FederationRouterPolicy routerPolicy = (FederationRouterPolicy) getPolicy(); + FederationPolicyInitializationContext fdContext = getFederationPolicyContext(); + FederationStateStoreFacade storeFacade = fdContext.getFederationStateStoreFacade(); + + // first we invoke a reservation placement + SubClusterId chosen = routerPolicy.getReservationHomeSubcluster(resReq); + + // add this to the store + ReservationHomeSubCluster homeSubCluster = + ReservationHomeSubCluster.newInstance(resReq.getReservationId(), chosen); + storeFacade.addReservationHomeSubCluster(homeSubCluster); + + // route an application that uses this app + ApplicationSubmissionContext applicationSubmissionContext = + ApplicationSubmissionContext.newInstance( + ApplicationId.newInstance(now, 1), "app1", "queue1", Priority.newInstance(1), + null, false, false, 1, null, null, false); + + applicationSubmissionContext.setReservationID(resReq.getReservationId()); + SubClusterId chosen2 = routerPolicy.getHomeSubcluster( + applicationSubmissionContext, Collections.emptyList()); + + // application follows reservation + Assert.assertEquals(chosen, chosen2); + } + + @Test + public void testUpdateReservation() throws YarnException { + long now = Time.now(); + ReservationSubmissionRequest resReq = getReservationSubmissionRequest(); + when(resReq.getQueue()).thenReturn("queue1"); + when(resReq.getReservationId()).thenReturn(ReservationId.newInstance(now, 1)); + + // first we invoke a reservation placement + FederationRouterPolicy routerPolicy = (FederationRouterPolicy) getPolicy(); + SubClusterId chosen = routerPolicy.getReservationHomeSubcluster(resReq); + + // add this to the store + FederationStateStoreFacade facade = + getFederationPolicyContext().getFederationStateStoreFacade(); + ReservationHomeSubCluster subCluster = + ReservationHomeSubCluster.newInstance(resReq.getReservationId(), chosen); + facade.addReservationHomeSubCluster(subCluster); + + // get all activeSubClusters + Map activeSubClusters = getActiveSubclusters(); + + // Update ReservationHomeSubCluster + List subClusterIds = new ArrayList<>(activeSubClusters.keySet()); + SubClusterId chosen2 = subClusterIds.get(this.getRand().nextInt(subClusterIds.size())); + ReservationHomeSubCluster subCluster2 = + ReservationHomeSubCluster.newInstance(resReq.getReservationId(), chosen2); + facade.updateReservationHomeSubCluster(subCluster2); + + // route an application that uses this app + ApplicationSubmissionContext applicationSubmissionContext = + ApplicationSubmissionContext.newInstance( + ApplicationId.newInstance(now, 1), "app1", "queue1", Priority.newInstance(1), + null, false, false, 1, null, null, false); + + applicationSubmissionContext.setReservationID(resReq.getReservationId()); + SubClusterId chosen3 = routerPolicy.getHomeSubcluster( + applicationSubmissionContext, new ArrayList<>()); + + Assert.assertEquals(chosen2, chosen3); + } + + @Test + public void testDeleteReservation() throws Exception { + long now = Time.now(); + ReservationSubmissionRequest resReq = getReservationSubmissionRequest(); + when(resReq.getQueue()).thenReturn("queue1"); + when(resReq.getReservationId()).thenReturn(ReservationId.newInstance(now, 1)); + + // first we invoke a reservation placement + FederationRouterPolicy routerPolicy = (FederationRouterPolicy) getPolicy(); + SubClusterId chosen = routerPolicy.getReservationHomeSubcluster(resReq); + + // add this to the store + FederationStateStoreFacade facade = + getFederationPolicyContext().getFederationStateStoreFacade(); + ReservationHomeSubCluster subCluster = + ReservationHomeSubCluster.newInstance(resReq.getReservationId(), chosen); + facade.addReservationHomeSubCluster(subCluster); + + // delete this to the store + facade.deleteReservationHomeSubCluster(resReq.getReservationId()); + + ApplicationSubmissionContext applicationSubmissionContext = + ApplicationSubmissionContext.newInstance( + ApplicationId.newInstance(now, 1), "app1", "queue1", Priority.newInstance(1), + null, false, false, 1, null, null, false); + applicationSubmissionContext.setReservationID(resReq.getReservationId()); + + LambdaTestUtils.intercept(YarnException.class, + "Reservation " + resReq.getReservationId() + " does not exist", + () -> routerPolicy.getHomeSubcluster(applicationSubmissionContext, new ArrayList<>())); + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/router/TestHashBasedRouterPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/router/TestHashBasedRouterPolicy.java index ee3e09d2b93ed..57f0b59ffe5fd 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/router/TestHashBasedRouterPolicy.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/router/TestHashBasedRouterPolicy.java @@ -27,7 +27,6 @@ import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext; import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.server.federation.store.records.SubClusterId; -import org.apache.hadoop.yarn.server.federation.utils.FederationPoliciesTestUtil; import org.junit.Assert; import org.junit.Before; import org.junit.Test; @@ -50,8 +49,7 @@ public void setUp() throws Exception { setMockActiveSubclusters(numSubclusters); // initialize policy with context - FederationPoliciesTestUtil.initializePolicyContext(getPolicy(), - getPolicyInfo(), getActiveSubclusters()); + setupContext(); } @Test diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/router/TestLoadBasedRouterPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/router/TestLoadBasedRouterPolicy.java index 58f1b9947bd81..d5acc5c9ee4f4 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/router/TestLoadBasedRouterPolicy.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/router/TestLoadBasedRouterPolicy.java @@ -17,22 +17,34 @@ package org.apache.hadoop.yarn.server.federation.policies.router; -import static org.junit.Assert.fail; - +import java.util.ArrayList; import java.util.HashMap; +import java.util.List; import java.util.Map; +import org.apache.hadoop.test.LambdaTestUtils; +import org.apache.hadoop.util.Time; +import org.apache.hadoop.yarn.api.protocolrecords.ReservationSubmissionRequest; +import org.apache.hadoop.yarn.api.records.ApplicationId; +import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext; +import org.apache.hadoop.yarn.api.records.Priority; +import org.apache.hadoop.yarn.api.records.ReservationId; import org.apache.hadoop.yarn.exceptions.YarnException; +import org.apache.hadoop.yarn.server.federation.policies.ConfigurableFederationPolicy; import org.apache.hadoop.yarn.server.federation.policies.dao.WeightedPolicyInfo; import org.apache.hadoop.yarn.server.federation.store.records.SubClusterId; import org.apache.hadoop.yarn.server.federation.store.records.SubClusterIdInfo; import org.apache.hadoop.yarn.server.federation.store.records.SubClusterInfo; import org.apache.hadoop.yarn.server.federation.store.records.SubClusterState; +import org.apache.hadoop.yarn.server.federation.store.records.ReservationHomeSubCluster; import org.apache.hadoop.yarn.server.federation.utils.FederationPoliciesTestUtil; +import org.apache.hadoop.yarn.server.federation.utils.FederationStateStoreFacade; import org.junit.Assert; import org.junit.Before; import org.junit.Test; +import static org.mockito.Mockito.when; + /** * Simple test class for the {@link LoadBasedRouterPolicy}. Test that the load * is properly considered for allocation. @@ -46,12 +58,14 @@ public void setUp() throws Exception { Map routerWeights = new HashMap<>(); Map amrmWeights = new HashMap<>(); + long now = Time.now(); + // simulate 20 active subclusters for (int i = 0; i < 20; i++) { SubClusterIdInfo sc = new SubClusterIdInfo(String.format("sc%02d", i)); - SubClusterInfo federationSubClusterInfo = - SubClusterInfo.newInstance(sc.toId(), null, null, null, null, -1, - SubClusterState.SC_RUNNING, -1, generateClusterMetricsInfo(i)); + SubClusterInfo federationSubClusterInfo = SubClusterInfo.newInstance( + sc.toId(), "dns1:80", "dns1:81", "dns1:82", "dns1:83", + now - 1000, SubClusterState.SC_RUNNING, now - 2000, generateClusterMetricsInfo(i)); getActiveSubclusters().put(sc.toId(), federationSubClusterInfo); float weight = getRand().nextInt(2); if (i == 5) { @@ -67,12 +81,11 @@ public void setUp() throws Exception { getPolicyInfo().setRouterPolicyWeights(routerWeights); getPolicyInfo().setAMRMPolicyWeights(amrmWeights); - FederationPoliciesTestUtil.initializePolicyContext(getPolicy(), - getPolicyInfo(), getActiveSubclusters()); - + // initialize policy with context + setupContext(); } - private String generateClusterMetricsInfo(int id) { + public String generateClusterMetricsInfo(int id) { long mem = 1024 * getRand().nextInt(277 * 100 - 1); // plant a best cluster @@ -106,7 +119,7 @@ public void testLoadIsRespected() throws YarnException { } @Test - public void testIfNoSubclustersWithWeightOne() { + public void testIfNoSubclustersWithWeightOne() throws Exception { setPolicy(new LoadBasedRouterPolicy()); setPolicyInfo(new WeightedPolicyInfo()); Map routerWeights = new HashMap<>(); @@ -123,15 +136,66 @@ public void testIfNoSubclustersWithWeightOne() { getPolicyInfo().setRouterPolicyWeights(routerWeights); getPolicyInfo().setAMRMPolicyWeights(amrmWeights); - try { - FederationPoliciesTestUtil.initializePolicyContext(getPolicy(), - getPolicyInfo(), getActiveSubclusters()); - ((FederationRouterPolicy) getPolicy()) - .getHomeSubcluster(getApplicationSubmissionContext(), null); - fail(); - } catch (YarnException ex) { - Assert.assertTrue( - ex.getMessage().contains("Zero Active Subcluster with weight 1")); + + ConfigurableFederationPolicy policy = getPolicy(); + FederationPoliciesTestUtil.initializePolicyContext(policy, + getPolicyInfo(), getActiveSubclusters()); + + LambdaTestUtils.intercept(YarnException.class, "Zero Active Subcluster with weight 1.", + () -> ((FederationRouterPolicy) policy). + getHomeSubcluster(getApplicationSubmissionContext(), null)); + } + + @Test + public void testUpdateReservation() throws YarnException { + long now = Time.now(); + ReservationSubmissionRequest resReq = getReservationSubmissionRequest(); + when(resReq.getQueue()).thenReturn("queue1"); + when(resReq.getReservationId()).thenReturn(ReservationId.newInstance(now, 1)); + + // first we invoke a reservation placement + FederationRouterPolicy routerPolicy = (FederationRouterPolicy) getPolicy(); + SubClusterId chosen = routerPolicy.getReservationHomeSubcluster(resReq); + + // add this to the store + FederationStateStoreFacade facade = + getFederationPolicyContext().getFederationStateStoreFacade(); + ReservationHomeSubCluster subCluster = + ReservationHomeSubCluster.newInstance(resReq.getReservationId(), chosen); + facade.addReservationHomeSubCluster(subCluster); + + // get all activeSubClusters + Map activeSubClusters = getActiveSubclusters(); + + // Update ReservationHomeSubCluster + // Cannot be randomly selected, SubCluster with Weight >= 1.0 needs to be selected + WeightedPolicyInfo weightedPolicyInfo = this.getPolicyInfo(); + Map routerPolicyWeights = weightedPolicyInfo.getRouterPolicyWeights(); + + List subClusterIds = new ArrayList<>(); + for (Map.Entry entry : routerPolicyWeights.entrySet()) { + SubClusterIdInfo subClusterIdInfo = entry.getKey(); + Float subClusterWeight = entry.getValue(); + if (subClusterWeight >= 1.0) { + subClusterIds.add(subClusterIdInfo.toId()); + } } + + SubClusterId chosen2 = subClusterIds.get(this.getRand().nextInt(subClusterIds.size())); + ReservationHomeSubCluster subCluster2 = + ReservationHomeSubCluster.newInstance(resReq.getReservationId(), chosen2); + facade.updateReservationHomeSubCluster(subCluster2); + + // route an application that uses this app + ApplicationSubmissionContext applicationSubmissionContext = + ApplicationSubmissionContext.newInstance( + ApplicationId.newInstance(now, 1), "app1", "queue1", Priority.newInstance(1), + null, false, false, 1, null, null, false); + + applicationSubmissionContext.setReservationID(resReq.getReservationId()); + SubClusterId chosen3 = routerPolicy.getHomeSubcluster( + applicationSubmissionContext, new ArrayList<>()); + + Assert.assertEquals(chosen2, chosen3); } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/router/TestLocalityRouterPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/router/TestLocalityRouterPolicy.java index 05939329a0681..3af0d037fcae6 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/router/TestLocalityRouterPolicy.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/router/TestLocalityRouterPolicy.java @@ -73,6 +73,7 @@ public void setUp() throws Exception { configureWeights(4); + // initialize policy with context initializePolicy(new YarnConfiguration()); } @@ -86,9 +87,7 @@ private void initializePolicy(Configuration conf) throws YarnException { .newInstance("queue1", getPolicy().getClass().getCanonicalName(), buf)); getFederationPolicyContext().setHomeSubcluster(getHomeSubCluster()); - FederationPoliciesTestUtil - .initializePolicyContext(getFederationPolicyContext(), getPolicy(), - getPolicyInfo(), getActiveSubclusters(), conf); + setupContext(); } /** diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/router/TestPriorityRouterPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/router/TestPriorityRouterPolicy.java index e1799d321083c..0b78a10c0a18e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/router/TestPriorityRouterPolicy.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/router/TestPriorityRouterPolicy.java @@ -23,6 +23,7 @@ import java.util.HashMap; import java.util.Map; +import org.apache.hadoop.util.Time; import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.server.federation.policies.dao.WeightedPolicyInfo; import org.apache.hadoop.yarn.server.federation.policies.exceptions.FederationPolicyException; @@ -54,10 +55,11 @@ public void setUp() throws Exception { // with 5% omit a subcluster if (getRand().nextFloat() < 0.95f || i == 5) { - SubClusterInfo sci = mock(SubClusterInfo.class); - when(sci.getState()).thenReturn(SubClusterState.SC_RUNNING); - when(sci.getSubClusterId()).thenReturn(sc.toId()); - getActiveSubclusters().put(sc.toId(), sci); + long now = Time.now(); + SubClusterInfo federationSubClusterInfo = SubClusterInfo.newInstance( + sc.toId(), "dns1:80", "dns1:81", "dns1:82", "dns1:83", + now - 1000, SubClusterState.SC_RUNNING, now - 2000, generateClusterMetricsInfo(i)); + getActiveSubclusters().put(sc.toId(), federationSubClusterInfo); } float weight = getRand().nextFloat(); if (i == 5) { @@ -72,9 +74,8 @@ public void setUp() throws Exception { } getPolicyInfo().setRouterPolicyWeights(routerWeights); getPolicyInfo().setAMRMPolicyWeights(amrmWeights); - FederationPoliciesTestUtil.initializePolicyContext(getPolicy(), - getPolicyInfo(), getActiveSubclusters()); + setupContext(); } @Test @@ -105,7 +106,7 @@ public void testZeroSubClustersWithPositiveWeight() throws Exception { getPolicyInfo(), getActiveSubclusters()); intercept(FederationPolicyException.class, - "No Active Subcluster with weight vector greater than zero", + "No Active Subcluster with weight vector greater than zero.", () -> ((FederationRouterPolicy) getPolicy()) .getHomeSubcluster(getApplicationSubmissionContext(), null)); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/router/TestRejectRouterPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/router/TestRejectRouterPolicy.java index 1747f73715cda..4b3b3392b4908 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/router/TestRejectRouterPolicy.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/router/TestRejectRouterPolicy.java @@ -20,7 +20,6 @@ import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext; import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.server.federation.policies.exceptions.FederationPolicyException; -import org.apache.hadoop.yarn.server.federation.utils.FederationPoliciesTestUtil; import org.apache.hadoop.yarn.util.resource.Resources; import org.junit.Before; import org.junit.Test; @@ -39,8 +38,7 @@ public void setUp() throws Exception { setMockActiveSubclusters(2); // initialize policy with context - FederationPoliciesTestUtil.initializePolicyContext(getPolicy(), - getPolicyInfo(), getActiveSubclusters()); + setupContext(); } @@ -60,4 +58,21 @@ public void testNullQueueRouting() throws YarnException { localPolicy.getHomeSubcluster(applicationSubmissionContext, null); } + @Override + @Test(expected = FederationPolicyException.class) + public void testFollowReservation() throws YarnException { + super.testFollowReservation(); + } + + @Override + @Test(expected = FederationPolicyException.class) + public void testUpdateReservation() throws YarnException { + super.testUpdateReservation(); + } + + @Override + @Test(expected = FederationPolicyException.class) + public void testDeleteReservation() throws Exception { + super.testDeleteReservation(); + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/router/TestUniformRandomRouterPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/router/TestUniformRandomRouterPolicy.java index 05490aba67247..8346277505ecb 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/router/TestUniformRandomRouterPolicy.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/router/TestUniformRandomRouterPolicy.java @@ -18,15 +18,14 @@ package org.apache.hadoop.yarn.server.federation.policies.router; import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; +import org.apache.hadoop.util.Time; import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.server.federation.policies.dao.WeightedPolicyInfo; import org.apache.hadoop.yarn.server.federation.store.records.SubClusterId; import org.apache.hadoop.yarn.server.federation.store.records.SubClusterIdInfo; import org.apache.hadoop.yarn.server.federation.store.records.SubClusterInfo; import org.apache.hadoop.yarn.server.federation.store.records.SubClusterState; -import org.apache.hadoop.yarn.server.federation.utils.FederationPoliciesTestUtil; import org.junit.Assert; import org.junit.Before; import org.junit.Test; @@ -44,14 +43,14 @@ public void setUp() throws Exception { setPolicyInfo(mock(WeightedPolicyInfo.class)); for (int i = 1; i <= 2; i++) { SubClusterIdInfo sc = new SubClusterIdInfo("sc" + i); - SubClusterInfo sci = mock(SubClusterInfo.class); - when(sci.getState()).thenReturn(SubClusterState.SC_RUNNING); - when(sci.getSubClusterId()).thenReturn(sc.toId()); - getActiveSubclusters().put(sc.toId(), sci); + long now = Time.now(); + SubClusterInfo federationSubClusterInfo = SubClusterInfo.newInstance( + sc.toId(), "dns1:80", "dns1:81", "dns1:82", "dns1:83", + now - 1000, SubClusterState.SC_RUNNING, now - 2000, generateClusterMetricsInfo(i)); + getActiveSubclusters().put(sc.toId(), federationSubClusterInfo); } - FederationPoliciesTestUtil.initializePolicyContext(getPolicy(), - mock(WeightedPolicyInfo.class), getActiveSubclusters()); + setupContext(); } @Test diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/router/TestWeightedRandomRouterPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/router/TestWeightedRandomRouterPolicy.java index d549250f07256..8121ce282cd6a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/router/TestWeightedRandomRouterPolicy.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/router/TestWeightedRandomRouterPolicy.java @@ -24,6 +24,7 @@ import java.util.Map; import java.util.concurrent.atomic.AtomicLong; +import org.apache.hadoop.util.Time; import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext; import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.server.federation.policies.FederationPolicyUtils; @@ -32,7 +33,6 @@ import org.apache.hadoop.yarn.server.federation.store.records.SubClusterIdInfo; import org.apache.hadoop.yarn.server.federation.store.records.SubClusterInfo; import org.apache.hadoop.yarn.server.federation.store.records.SubClusterState; -import org.apache.hadoop.yarn.server.federation.utils.FederationPoliciesTestUtil; import org.junit.Assert; import org.junit.Before; import org.junit.Test; @@ -51,8 +51,7 @@ public void setUp() throws Exception { configureWeights(20); - FederationPoliciesTestUtil.initializePolicyContext(getPolicy(), - getPolicyInfo(), getActiveSubclusters()); + setupContext(); } public void configureWeights(float numSubClusters) { @@ -68,10 +67,11 @@ public void configureWeights(float numSubClusters) { SubClusterIdInfo sc = new SubClusterIdInfo("sc" + i); // with 5% omit a subcluster if (getRand().nextFloat() < 0.95f) { - SubClusterInfo sci = mock(SubClusterInfo.class); - when(sci.getState()).thenReturn(SubClusterState.SC_RUNNING); - when(sci.getSubClusterId()).thenReturn(sc.toId()); - getActiveSubclusters().put(sc.toId(), sci); + long now = Time.now(); + SubClusterInfo federationSubClusterInfo = SubClusterInfo.newInstance( + sc.toId(), "dns1:80", "dns1:81", "dns1:82", "dns1:83", + now - 1000, SubClusterState.SC_RUNNING, now - 2000, generateClusterMetricsInfo(i)); + getActiveSubclusters().put(sc.toId(), federationSubClusterInfo); } // 80% of the weight is evenly spread, 20% is randomly generated diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/store/impl/FederationStateStoreBaseTest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/store/impl/FederationStateStoreBaseTest.java index d0e6485b02883..296e4846ea4c8 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/store/impl/FederationStateStoreBaseTest.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/store/impl/FederationStateStoreBaseTest.java @@ -24,13 +24,17 @@ import java.util.TimeZone; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.test.LambdaTestUtils; +import org.apache.hadoop.util.Time; import org.apache.hadoop.yarn.api.records.ApplicationId; +import org.apache.hadoop.yarn.api.records.ReservationId; import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.server.federation.store.FederationStateStore; import org.apache.hadoop.yarn.server.federation.store.exception.FederationStateStoreException; import org.apache.hadoop.yarn.server.federation.store.records.AddApplicationHomeSubClusterRequest; import org.apache.hadoop.yarn.server.federation.store.records.AddApplicationHomeSubClusterResponse; import org.apache.hadoop.yarn.server.federation.store.records.ApplicationHomeSubCluster; +import org.apache.hadoop.yarn.server.federation.store.records.ReservationHomeSubCluster; import org.apache.hadoop.yarn.server.federation.store.records.DeleteApplicationHomeSubClusterRequest; import org.apache.hadoop.yarn.server.federation.store.records.DeleteApplicationHomeSubClusterResponse; import org.apache.hadoop.yarn.server.federation.store.records.GetApplicationHomeSubClusterRequest; @@ -56,6 +60,14 @@ import org.apache.hadoop.yarn.server.federation.store.records.SubClusterState; import org.apache.hadoop.yarn.server.federation.store.records.UpdateApplicationHomeSubClusterRequest; import org.apache.hadoop.yarn.server.federation.store.records.UpdateApplicationHomeSubClusterResponse; +import org.apache.hadoop.yarn.server.federation.store.records.AddReservationHomeSubClusterRequest; +import org.apache.hadoop.yarn.server.federation.store.records.AddReservationHomeSubClusterResponse; +import org.apache.hadoop.yarn.server.federation.store.records.GetReservationHomeSubClusterResponse; +import org.apache.hadoop.yarn.server.federation.store.records.GetReservationHomeSubClusterRequest; +import org.apache.hadoop.yarn.server.federation.store.records.DeleteReservationHomeSubClusterRequest; +import org.apache.hadoop.yarn.server.federation.store.records.DeleteReservationHomeSubClusterResponse; +import org.apache.hadoop.yarn.server.federation.store.records.UpdateReservationHomeSubClusterRequest; +import org.apache.hadoop.yarn.server.federation.store.records.UpdateReservationHomeSubClusterResponse; import org.apache.hadoop.yarn.util.MonotonicClock; import org.junit.After; import org.junit.Assert; @@ -599,4 +611,153 @@ protected FederationStateStore getStateStore() { return stateStore; } + SubClusterId queryReservationHomeSC(ReservationId reservationId) + throws YarnException { + + GetReservationHomeSubClusterRequest request = + GetReservationHomeSubClusterRequest.newInstance(reservationId); + + GetReservationHomeSubClusterResponse response = + stateStore.getReservationHomeSubCluster(request); + + return response.getReservationHomeSubCluster().getHomeSubCluster(); + } + + @Test + public void testAddReservationHomeSubCluster() throws Exception { + + ReservationId reservationId = ReservationId.newInstance(Time.now(), 1); + SubClusterId subClusterId = SubClusterId.newInstance("SC"); + + ReservationHomeSubCluster reservationHomeSubCluster = + ReservationHomeSubCluster.newInstance(reservationId, subClusterId); + + AddReservationHomeSubClusterRequest request = + AddReservationHomeSubClusterRequest.newInstance(reservationHomeSubCluster); + AddReservationHomeSubClusterResponse response = + stateStore.addReservationHomeSubCluster(request); + + Assert.assertEquals(subClusterId, response.getHomeSubCluster()); + Assert.assertEquals(subClusterId, queryReservationHomeSC(reservationId)); + } + + private void addReservationHomeSC(ReservationId reservationId, SubClusterId subClusterId) + throws YarnException { + + ReservationHomeSubCluster reservationHomeSubCluster = + ReservationHomeSubCluster.newInstance(reservationId, subClusterId); + AddReservationHomeSubClusterRequest request = + AddReservationHomeSubClusterRequest.newInstance(reservationHomeSubCluster); + stateStore.addReservationHomeSubCluster(request); + } + + @Test + public void testAddReservationHomeSubClusterReservationAlreadyExists() throws Exception { + + ReservationId reservationId = ReservationId.newInstance(Time.now(), 1); + SubClusterId subClusterId1 = SubClusterId.newInstance("SC1"); + addReservationHomeSC(reservationId, subClusterId1); + + SubClusterId subClusterId2 = SubClusterId.newInstance("SC2"); + ReservationHomeSubCluster reservationHomeSubCluster2 = + ReservationHomeSubCluster.newInstance(reservationId, subClusterId2); + AddReservationHomeSubClusterRequest request2 = + AddReservationHomeSubClusterRequest.newInstance(reservationHomeSubCluster2); + AddReservationHomeSubClusterResponse response = + stateStore.addReservationHomeSubCluster(request2); + + Assert.assertNotNull(response); + Assert.assertEquals(subClusterId1, response.getHomeSubCluster()); + Assert.assertEquals(subClusterId1, queryReservationHomeSC(reservationId)); + } + + @Test + public void testAddReservationHomeSubClusterAppAlreadyExistsInTheSameSC() + throws Exception { + + ReservationId reservationId = ReservationId.newInstance(Time.now(), 1); + SubClusterId subClusterId1 = SubClusterId.newInstance("SC1"); + addReservationHomeSC(reservationId, subClusterId1); + + ReservationHomeSubCluster reservationHomeSubCluster2 = + ReservationHomeSubCluster.newInstance(reservationId, subClusterId1); + AddReservationHomeSubClusterRequest request2 = + AddReservationHomeSubClusterRequest.newInstance(reservationHomeSubCluster2); + AddReservationHomeSubClusterResponse response = + stateStore.addReservationHomeSubCluster(request2); + + Assert.assertNotNull(response); + Assert.assertEquals(subClusterId1, response.getHomeSubCluster()); + Assert.assertEquals(subClusterId1, queryReservationHomeSC(reservationId)); + } + + @Test + public void testDeleteReservationHomeSubCluster() throws Exception { + + ReservationId reservationId = ReservationId.newInstance(Time.now(), 1); + SubClusterId subClusterId1 = SubClusterId.newInstance("SC"); + addReservationHomeSC(reservationId, subClusterId1); + + DeleteReservationHomeSubClusterRequest delReservationRequest = + DeleteReservationHomeSubClusterRequest.newInstance(reservationId); + DeleteReservationHomeSubClusterResponse delReservationResponse = + stateStore.deleteReservationHomeSubCluster(delReservationRequest); + + Assert.assertNotNull(delReservationResponse); + + LambdaTestUtils.intercept(YarnException.class, + "Reservation " + reservationId + " does not exist", + () -> queryReservationHomeSC(reservationId)); + } + + @Test + public void testDeleteReservationHomeSubClusterUnknownApp() throws Exception { + + ReservationId reservationId = ReservationId.newInstance(Time.now(), 1); + + DeleteReservationHomeSubClusterRequest delReservationRequest = + DeleteReservationHomeSubClusterRequest.newInstance(reservationId); + + LambdaTestUtils.intercept(YarnException.class, + "Reservation " + reservationId + " does not exist", + () -> stateStore.deleteReservationHomeSubCluster(delReservationRequest)); + } + + @Test + public void testUpdateReservationHomeSubCluster() throws Exception { + + ReservationId reservationId = ReservationId.newInstance(Time.now(), 1); + SubClusterId subClusterId1 = SubClusterId.newInstance("SC"); + addReservationHomeSC(reservationId, subClusterId1); + + SubClusterId subClusterId2 = SubClusterId.newInstance("SC2"); + ReservationHomeSubCluster reservationHomeSubCluster = + ReservationHomeSubCluster.newInstance(reservationId, subClusterId2); + + UpdateReservationHomeSubClusterRequest updateReservationRequest = + UpdateReservationHomeSubClusterRequest.newInstance(reservationHomeSubCluster); + + UpdateReservationHomeSubClusterResponse updateReservationResponse = + stateStore.updateReservationHomeSubCluster(updateReservationRequest); + + Assert.assertNotNull(updateReservationResponse); + Assert.assertEquals(subClusterId2, queryReservationHomeSC(reservationId)); + } + + @Test + public void testUpdateReservationHomeSubClusterUnknownApp() throws Exception { + + ReservationId reservationId = ReservationId.newInstance(Time.now(), 1); + SubClusterId subClusterId1 = SubClusterId.newInstance("SC1"); + + ReservationHomeSubCluster reservationHomeSubCluster = + ReservationHomeSubCluster.newInstance(reservationId, subClusterId1); + + UpdateReservationHomeSubClusterRequest updateReservationRequest = + UpdateReservationHomeSubClusterRequest.newInstance(reservationHomeSubCluster); + + LambdaTestUtils.intercept(YarnException.class, + "Reservation " + reservationId + " does not exist", + () -> stateStore.updateReservationHomeSubCluster(updateReservationRequest)); + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/store/impl/HSQLDBFederationStateStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/store/impl/HSQLDBFederationStateStore.java index c3d0a9e1bbd53..2eb95bbd6f6f3 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/store/impl/HSQLDBFederationStateStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/store/impl/HSQLDBFederationStateStore.java @@ -58,6 +58,12 @@ public class HSQLDBFederationStateStore extends SQLFederationStateStore { + " policyType varchar(256) NOT NULL, params varbinary(512)," + " CONSTRAINT pk_queue PRIMARY KEY (queue))"; + private static final String TABLE_RESERVATIONSHOMESUBCLUSTER = + " CREATE TABLE reservationsHomeSubCluster (" + + " reservationId varchar(128) NOT NULL," + + " homeSubCluster varchar(256) NOT NULL," + + " CONSTRAINT pk_reservationId PRIMARY KEY (reservationId))"; + private static final String SP_REGISTERSUBCLUSTER = "CREATE PROCEDURE sp_registerSubCluster(" + " IN subClusterId_IN varchar(256)," @@ -201,6 +207,101 @@ public class HSQLDBFederationStateStore extends SQLFederationStateStore { + " DECLARE result CURSOR FOR" + " SELECT * FROM policies; OPEN result; END"; + private static final String SP_ADDRESERVATIONHOMESUBCLUSTER = + "CREATE PROCEDURE sp_addReservationHomeSubCluster(" + + " IN reservationId_IN varchar(128)," + + " IN homeSubCluster_IN varchar(256)," + + " OUT storedHomeSubCluster_OUT varchar(256), OUT rowCount_OUT int)" + + " MODIFIES SQL DATA BEGIN ATOMIC" + + " INSERT INTO reservationsHomeSubCluster " + + " (reservationId,homeSubCluster) " + + " (SELECT reservationId_IN, homeSubCluster_IN" + + " FROM reservationsHomeSubCluster" + + " WHERE reservationId = reservationId_IN" + + " HAVING COUNT(*) = 0 );" + + " GET DIAGNOSTICS rowCount_OUT = ROW_COUNT;" + + " SELECT homeSubCluster INTO storedHomeSubCluster_OUT" + + " FROM reservationsHomeSubCluster" + + " WHERE reservationId = reservationId_IN; END"; + + private static final String SP_GETRESERVATIONHOMESUBCLUSTER = + "CREATE PROCEDURE sp_getReservationHomeSubCluster(" + + " IN reservationId_IN varchar(128)," + + " OUT homeSubCluster_OUT varchar(256))" + + " MODIFIES SQL DATA BEGIN ATOMIC" + + " SELECT homeSubCluster INTO homeSubCluster_OUT" + + " FROM reservationsHomeSubCluster" + + " WHERE reservationId = reservationId_IN; END"; + + private static final String SP_GETRESERVATIONSHOMESUBCLUSTER = + "CREATE PROCEDURE sp_getReservationsHomeSubCluster()" + + " MODIFIES SQL DATA DYNAMIC RESULT SETS 1 BEGIN ATOMIC" + + " DECLARE result CURSOR FOR" + + " SELECT reservationId, homeSubCluster" + + " FROM reservationsHomeSubCluster; OPEN result; END"; + + private static final String SP_DELETERESERVATIONHOMESUBCLUSTER = + "CREATE PROCEDURE sp_deleteReservationHomeSubCluster(" + + " IN reservationId_IN varchar(128), OUT rowCount_OUT int)" + + " MODIFIES SQL DATA BEGIN ATOMIC" + + " DELETE FROM reservationsHomeSubCluster" + + " WHERE reservationId = reservationId_IN;" + + " GET DIAGNOSTICS rowCount_OUT = ROW_COUNT; END"; + + private static final String SP_UPDATERESERVATIONHOMESUBCLUSTER = + "CREATE PROCEDURE sp_updateReservationHomeSubCluster(" + + " IN reservationId_IN varchar(128)," + + " IN homeSubCluster_IN varchar(256), OUT rowCount_OUT int)" + + " MODIFIES SQL DATA BEGIN ATOMIC" + + " UPDATE reservationsHomeSubCluster" + + " SET homeSubCluster = homeSubCluster_IN" + + " WHERE reservationId = reservationId_IN;" + + " GET DIAGNOSTICS rowCount_OUT = ROW_COUNT; END"; + + protected static final String SP_DROP_ADDRESERVATIONHOMESUBCLUSTER = + "DROP PROCEDURE sp_addReservationHomeSubCluster"; + + protected static final String SP_ADDRESERVATIONHOMESUBCLUSTER2 = + "CREATE PROCEDURE sp_addReservationHomeSubCluster(" + + " IN reservationId_IN varchar(128)," + + " IN homeSubCluster_IN varchar(256)," + + " OUT storedHomeSubCluster_OUT varchar(256), OUT rowCount_OUT int)" + + " MODIFIES SQL DATA BEGIN ATOMIC" + + " INSERT INTO reservationsHomeSubCluster " + + " (reservationId,homeSubCluster) " + + " (SELECT reservationId_IN, homeSubCluster_IN" + + " FROM reservationsHomeSubCluster" + + " WHERE reservationId = reservationId_IN" + + " HAVING COUNT(*) = 0 );" + + " SELECT homeSubCluster, 2 INTO storedHomeSubCluster_OUT, rowCount_OUT" + + " FROM reservationsHomeSubCluster" + + " WHERE reservationId = reservationId_IN; END"; + + protected static final String SP_DROP_UPDATERESERVATIONHOMESUBCLUSTER = + "DROP PROCEDURE sp_updateReservationHomeSubCluster"; + + protected static final String SP_UPDATERESERVATIONHOMESUBCLUSTER2 = + "CREATE PROCEDURE sp_updateReservationHomeSubCluster(" + + " IN reservationId_IN varchar(128)," + + " IN homeSubCluster_IN varchar(256), OUT rowCount_OUT int)" + + " MODIFIES SQL DATA BEGIN ATOMIC" + + " UPDATE reservationsHomeSubCluster" + + " SET homeSubCluster = homeSubCluster_IN" + + " WHERE reservationId = reservationId_IN;" + + " SET rowCount_OUT = 2; END"; + + protected static final String SP_DROP_DELETERESERVATIONHOMESUBCLUSTER = + "DROP PROCEDURE sp_deleteReservationHomeSubCluster"; + + protected static final String SP_DELETERESERVATIONHOMESUBCLUSTER2 = + "CREATE PROCEDURE sp_deleteReservationHomeSubCluster(" + + " IN reservationId_IN varchar(128), OUT rowCount_OUT int)" + + " MODIFIES SQL DATA BEGIN ATOMIC" + + " DELETE FROM reservationsHomeSubCluster" + + " WHERE reservationId = reservationId_IN;" + + " SET rowCount_OUT = 2; END"; + + @Override public void init(Configuration conf) { try { @@ -216,6 +317,7 @@ public void init(Configuration conf) { conn.prepareStatement(TABLE_APPLICATIONSHOMESUBCLUSTER).execute(); conn.prepareStatement(TABLE_MEMBERSHIP).execute(); conn.prepareStatement(TABLE_POLICIES).execute(); + conn.prepareStatement(TABLE_RESERVATIONSHOMESUBCLUSTER).execute(); conn.prepareStatement(SP_REGISTERSUBCLUSTER).execute(); conn.prepareStatement(SP_DEREGISTERSUBCLUSTER).execute(); @@ -233,6 +335,12 @@ public void init(Configuration conf) { conn.prepareStatement(SP_GETPOLICYCONFIGURATION).execute(); conn.prepareStatement(SP_GETPOLICIESCONFIGURATIONS).execute(); + conn.prepareStatement(SP_ADDRESERVATIONHOMESUBCLUSTER).execute(); + conn.prepareStatement(SP_GETRESERVATIONHOMESUBCLUSTER).execute(); + conn.prepareStatement(SP_GETRESERVATIONSHOMESUBCLUSTER).execute(); + conn.prepareStatement(SP_DELETERESERVATIONHOMESUBCLUSTER).execute(); + conn.prepareStatement(SP_UPDATERESERVATIONHOMESUBCLUSTER).execute(); + LOG.info("Database Init: Complete"); } catch (SQLException e) { LOG.error("ERROR: failed to inizialize HSQLDB " + e.getMessage()); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/store/impl/TestSQLFederationStateStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/store/impl/TestSQLFederationStateStore.java index 3c1d327b393aa..d257b870d0770 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/store/impl/TestSQLFederationStateStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/store/impl/TestSQLFederationStateStore.java @@ -17,7 +17,10 @@ package org.apache.hadoop.yarn.server.federation.store.impl; +import org.apache.hadoop.test.LambdaTestUtils; +import org.apache.hadoop.util.Time; import org.apache.hadoop.yarn.api.records.ApplicationId; +import org.apache.hadoop.yarn.api.records.ReservationId; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.server.federation.store.FederationStateStore; @@ -25,14 +28,43 @@ import org.apache.hadoop.yarn.server.federation.store.records.SubClusterId; import org.apache.hadoop.yarn.server.federation.store.records.SubClusterInfo; import org.apache.hadoop.yarn.server.federation.store.records.SubClusterRegisterRequest; +import org.apache.hadoop.yarn.server.federation.store.records.ReservationHomeSubCluster; +import org.apache.hadoop.yarn.server.federation.store.records.AddReservationHomeSubClusterRequest; +import org.apache.hadoop.yarn.server.federation.store.records.UpdateReservationHomeSubClusterRequest; +import org.apache.hadoop.yarn.server.federation.store.records.DeleteReservationHomeSubClusterRequest; +import org.apache.hadoop.yarn.server.federation.store.utils.FederationStateStoreUtils; import org.junit.Assert; import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.sql.CallableStatement; +import java.sql.Connection; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.List; + +import static org.apache.hadoop.yarn.server.federation.store.impl.SQLFederationStateStore.CALL_SP_ADD_RESERVATION_HOME_SUBCLUSTER; +import static org.apache.hadoop.yarn.server.federation.store.impl.SQLFederationStateStore.CALL_SP_GET_RESERVATION_HOME_SUBCLUSTER; +import static org.apache.hadoop.yarn.server.federation.store.impl.SQLFederationStateStore.CALL_SP_GET_RESERVATIONS_HOME_SUBCLUSTER; +import static org.apache.hadoop.yarn.server.federation.store.impl.SQLFederationStateStore.CALL_SP_UPDATE_RESERVATION_HOME_SUBCLUSTER; +import static org.apache.hadoop.yarn.server.federation.store.impl.SQLFederationStateStore.CALL_SP_DELETE_RESERVATION_HOME_SUBCLUSTER; +import static org.apache.hadoop.yarn.server.federation.store.impl.HSQLDBFederationStateStore.SP_DROP_ADDRESERVATIONHOMESUBCLUSTER; +import static org.apache.hadoop.yarn.server.federation.store.impl.HSQLDBFederationStateStore.SP_ADDRESERVATIONHOMESUBCLUSTER2; +import static org.apache.hadoop.yarn.server.federation.store.impl.HSQLDBFederationStateStore.SP_DROP_UPDATERESERVATIONHOMESUBCLUSTER; +import static org.apache.hadoop.yarn.server.federation.store.impl.HSQLDBFederationStateStore.SP_UPDATERESERVATIONHOMESUBCLUSTER2; +import static org.apache.hadoop.yarn.server.federation.store.impl.HSQLDBFederationStateStore.SP_DROP_DELETERESERVATIONHOMESUBCLUSTER; +import static org.apache.hadoop.yarn.server.federation.store.impl.HSQLDBFederationStateStore.SP_DELETERESERVATIONHOMESUBCLUSTER2; /** * Unit tests for SQLFederationStateStore. */ public class TestSQLFederationStateStore extends FederationStateStoreBaseTest { + public static final Logger LOG = + LoggerFactory.getLogger(TestSQLFederationStateStore.class); private static final String HSQLDB_DRIVER = "org.hsqldb.jdbc.JDBCDataSource"; private static final String DATABASE_URL = "jdbc:hsqldb:mem:state"; private static final String DATABASE_USERNAME = "SA"; @@ -74,4 +106,453 @@ public void testSqlConnectionsCreatedCount() throws YarnException { Assert.assertEquals(1, FederationStateStoreClientMetrics.getNumConnections()); } + + class ReservationHomeSC { + private String reservationId; + private String subHomeClusterId; + private int dbUpdateCount; + + ReservationHomeSC(String rId, String subHomeSCId, int dbUpdateCount) { + this.reservationId = rId; + this.subHomeClusterId = subHomeSCId; + this.dbUpdateCount = dbUpdateCount; + } + } + + private ReservationHomeSC addReservationHomeSubCluster( + SQLFederationStateStore sqlFederationStateStore, String procedure, + String reservationId, String subHomeClusterId) throws SQLException, YarnException { + // procedure call parameter preparation + CallableStatement cstmt = sqlFederationStateStore.getCallableStatement(procedure); + cstmt.setString("reservationId_IN", reservationId); + cstmt.setString("homeSubCluster_IN", subHomeClusterId); + cstmt.registerOutParameter("storedHomeSubCluster_OUT", java.sql.Types.VARCHAR); + cstmt.registerOutParameter("rowCount_OUT", java.sql.Types.INTEGER); + + // execute procedure + cstmt.executeUpdate(); + + // get call result + String dbStoredHomeSubCluster = cstmt.getString("storedHomeSubCluster_OUT"); + int dbRowCount = cstmt.getInt("rowCount_OUT"); + + // return cstmt to pool + FederationStateStoreUtils.returnToPool(LOG, cstmt); + + return new ReservationHomeSC(reservationId, dbStoredHomeSubCluster, dbRowCount); + } + + private ReservationHomeSC getReservationHomeSubCluster( + SQLFederationStateStore sqlFederationStateStore, String procedure, + String reservationId) throws SQLException, YarnException { + + // procedure call parameter preparation + CallableStatement cstmt = sqlFederationStateStore.getCallableStatement(procedure); + cstmt.setString("reservationId_IN", reservationId.toString()); + cstmt.registerOutParameter("homeSubCluster_OUT", java.sql.Types.VARCHAR); + + // execute procedure + cstmt.execute(); + + // get call result + String dBSubClusterHomeId = cstmt.getString("homeSubCluster_OUT"); + + // return cstmt to pool + FederationStateStoreUtils.returnToPool(LOG, cstmt); + + // returns the ReservationHomeSubCluster object + return new ReservationHomeSC(reservationId, dBSubClusterHomeId, 0); + } + + private List getReservationsHomeSubCluster( + SQLFederationStateStore sqlFederationStateStore, String procedure) + throws SQLException, IOException, YarnException { + + List results = new ArrayList<>(); + + // procedure call parameter preparation + CallableStatement cstmt = sqlFederationStateStore.getCallableStatement(procedure); + + // execute procedure + ResultSet rs = cstmt.executeQuery(); + while (rs.next()) { + // 1）OUT reservationId + String dbReservationId = rs.getString("reservationId"); + + // 2）OUT homeSubCluster + String dbHomeSubCluster = rs.getString("homeSubCluster"); + results.add(new ReservationHomeSC(dbReservationId, dbHomeSubCluster, 0)); + } + + // return cstmt to pool + FederationStateStoreUtils.returnToPool(LOG, cstmt); + + // return ReservationHomeSubCluster List + return results; + } + + private ReservationHomeSC updateReservationHomeSubCluster( + SQLFederationStateStore sqlFederationStateStore, String procedure, + String reservationId, String subHomeClusterId) + throws SQLException, IOException { + + // procedure call parameter preparation + CallableStatement cstmt = sqlFederationStateStore.getCallableStatement(procedure); + + // 1）IN reservationId_IN varchar(128) + cstmt.setString("reservationId_IN", reservationId); + // 2）IN homeSubCluster_IN varchar(256) + cstmt.setString("homeSubCluster_IN", subHomeClusterId); + // 3）OUT rowCount_OUT int + cstmt.registerOutParameter("rowCount_OUT", java.sql.Types.INTEGER); + + // execute procedure + cstmt.executeUpdate(); + + // get rowcount + int rowCount = cstmt.getInt("rowCount_OUT"); + + // returns the ReservationHomeSubCluster object + return new ReservationHomeSC(reservationId, subHomeClusterId, rowCount); + } + + private ReservationHomeSC deleteReservationHomeSubCluster( + SQLFederationStateStore sqlFederationStateStore, String procedure, + String reservationId) throws SQLException { + // procedure call parameter preparation + CallableStatement cstmt = sqlFederationStateStore.getCallableStatement(procedure); + + // Set the parameters for the stored procedure + // 1）IN reservationId_IN varchar(128) + cstmt.setString("reservationId_IN", reservationId); + // 2）OUT rowCount_OUT int + cstmt.registerOutParameter("rowCount_OUT", java.sql.Types.INTEGER); + + // execute procedure + cstmt.executeUpdate(); + + // get rowcount + int rowCount = cstmt.getInt("rowCount_OUT"); + + // returns the ReservationHomeSubCluster object + return new ReservationHomeSC(reservationId, "-", rowCount); + } + + /** + * This test case is used to check whether the procedure + * sp_addReservationHomeSubCluster can be executed normally. + * + * This test case will write 1 record to the database, and check returns the result. + * + * @throws Exception when the error occurs + */ + @Test + public void testCheckAddReservationHomeSubCluster() throws Exception { + FederationStateStore stateStore = getStateStore(); + Assert.assertTrue(stateStore instanceof SQLFederationStateStore); + + SQLFederationStateStore sqlFederationStateStore = (SQLFederationStateStore) stateStore; + + // procedure call parameter preparation + ReservationId reservationId = ReservationId.newInstance(Time.now(), 1); + String subHomeClusterId = "SC-1"; + ReservationHomeSC resultHC = addReservationHomeSubCluster(sqlFederationStateStore, + CALL_SP_ADD_RESERVATION_HOME_SUBCLUSTER, reservationId.toString(), subHomeClusterId); + + // validation results + Assert.assertNotNull(resultHC); + Assert.assertEquals(subHomeClusterId, resultHC.subHomeClusterId); + Assert.assertEquals(1, resultHC.dbUpdateCount); + } + + /** + * This test case is used to check whether the procedure + * sp_getReservationHomeSubCluster can be executed normally. + * + * Query according to reservationId, expect accurate query results, + * and check the homeSubCluster field. + * + * @throws Exception when the error occurs + */ + @Test + public void testCheckGetReservationHomeSubCluster() throws Exception { + FederationStateStore stateStore = getStateStore(); + Assert.assertTrue(stateStore instanceof SQLFederationStateStore); + + SQLFederationStateStore sqlFederationStateStore = (SQLFederationStateStore) stateStore; + + // procedure call parameter preparation + ReservationId reservationId = ReservationId.newInstance(Time.now(), 1); + String subHomeClusterId = "SC-1"; + addReservationHomeSubCluster(sqlFederationStateStore, + CALL_SP_ADD_RESERVATION_HOME_SUBCLUSTER, reservationId.toString(), subHomeClusterId); + + // Call getReservationHomeSubCluster to get the result + ReservationHomeSC resultHC = getReservationHomeSubCluster(sqlFederationStateStore, + CALL_SP_GET_RESERVATION_HOME_SUBCLUSTER, reservationId.toString()); + + Assert.assertNotNull(resultHC); + Assert.assertEquals(subHomeClusterId, resultHC.subHomeClusterId); + Assert.assertEquals(reservationId.toString(), resultHC.reservationId); + } + + /** + * This test case is used to check whether the procedure + * sp_getReservationsHomeSubCluster can be executed normally. + * + * This test case will write 2 record to the database, and check returns the result. + * + * The test case will compare the number of returned records from the database + * and whether the content of each returned record is accurate. + * + * @throws Exception when the error occurs + */ + @Test + public void testCheckGetReservationsHomeSubCluster() throws Exception { + FederationStateStore stateStore = getStateStore(); + Assert.assertTrue(stateStore instanceof SQLFederationStateStore); + + SQLFederationStateStore sqlFederationStateStore = (SQLFederationStateStore) stateStore; + + // add 1st record + ReservationId reservationId1 = ReservationId.newInstance(Time.now(), 1); + String subHomeClusterId1 = "SC-1"; + addReservationHomeSubCluster(sqlFederationStateStore, + CALL_SP_ADD_RESERVATION_HOME_SUBCLUSTER, reservationId1.toString(), subHomeClusterId1); + + // add 2nd record + ReservationId reservationId2 = ReservationId.newInstance(Time.now(), 2); + String subHomeClusterId2 = "SC-2"; + addReservationHomeSubCluster(sqlFederationStateStore, + CALL_SP_ADD_RESERVATION_HOME_SUBCLUSTER, reservationId2.toString(), subHomeClusterId2); + + List reservationHomeSubClusters = getReservationsHomeSubCluster( + sqlFederationStateStore, CALL_SP_GET_RESERVATIONS_HOME_SUBCLUSTER); + + Assert.assertNotNull(reservationHomeSubClusters); + Assert.assertEquals(2, reservationHomeSubClusters.size()); + + ReservationHomeSC resultHC1 = reservationHomeSubClusters.get(0); + Assert.assertNotNull(resultHC1); + Assert.assertEquals(reservationId1.toString(), resultHC1.reservationId); + Assert.assertEquals(subHomeClusterId1, resultHC1.subHomeClusterId); + + ReservationHomeSC resultHC2 = reservationHomeSubClusters.get(1); + Assert.assertNotNull(resultHC2); + Assert.assertEquals(reservationId2.toString(), resultHC2.reservationId); + Assert.assertEquals(subHomeClusterId2, resultHC2.subHomeClusterId); + } + + /** + * This test case is used to check whether the procedure + * sp_updateReservationHomeSubCluster can be executed normally. + * + * This test case will first insert 1 record into the database, + * and then update the new SubHomeClusterId according to the reservationId. + * + * It will check whether the SubHomeClusterId is as expected after the addition and update. + * For the first time, the HomeClusterId of the database should be SC-1, + * and for the second time, the HomeClusterId of the database should be SC-2. + * + * @throws Exception when the error occurs + */ + @Test + public void testCheckUpdateReservationHomeSubCluster() throws Exception { + FederationStateStore stateStore = getStateStore(); + Assert.assertTrue(stateStore instanceof SQLFederationStateStore); + + SQLFederationStateStore sqlFederationStateStore = (SQLFederationStateStore) stateStore; + + // procedure call parameter preparation + ReservationId reservationId = ReservationId.newInstance(Time.now(), 1); + String subHomeClusterId = "SC-1"; + addReservationHomeSubCluster(sqlFederationStateStore, + CALL_SP_ADD_RESERVATION_HOME_SUBCLUSTER, reservationId.toString(), subHomeClusterId); + + // verify that the subHomeClusterId corresponding to reservationId is SC-1 + ReservationHomeSC resultHC = getReservationHomeSubCluster(sqlFederationStateStore, + CALL_SP_GET_RESERVATION_HOME_SUBCLUSTER, reservationId.toString()); + Assert.assertNotNull(resultHC); + Assert.assertEquals(subHomeClusterId, resultHC.subHomeClusterId); + + // prepare to update parameters + String newSubHomeClusterId = "SC-2"; + ReservationHomeSC reservationHomeSubCluster = + updateReservationHomeSubCluster(sqlFederationStateStore, + CALL_SP_UPDATE_RESERVATION_HOME_SUBCLUSTER, reservationId.toString(), newSubHomeClusterId); + + Assert.assertNotNull(reservationHomeSubCluster); + Assert.assertEquals(1, reservationHomeSubCluster.dbUpdateCount); + + // verify that the subHomeClusterId corresponding to reservationId is SC-2 + ReservationHomeSC resultHC2 = getReservationHomeSubCluster(sqlFederationStateStore, + CALL_SP_GET_RESERVATION_HOME_SUBCLUSTER, reservationId.toString()); + Assert.assertNotNull(resultHC2); + Assert.assertEquals(newSubHomeClusterId, resultHC2.subHomeClusterId); + } + + /** + * This test case is used to check whether the procedure + * sp_deleteReservationHomeSubCluster can be executed normally. + * + * This test case will first write 1 record to the database, + * and then delete the corresponding record according to reservationId. + * + * Query the corresponding homeSubCluster according to reservationId, + * we should get a NULL at this time. + * + * @throws Exception when the error occurs + */ + @Test + public void testCheckDeleteReservationHomeSubCluster() throws Exception { + FederationStateStore stateStore = getStateStore(); + Assert.assertTrue(stateStore instanceof SQLFederationStateStore); + + SQLFederationStateStore sqlFederationStateStore = (SQLFederationStateStore) stateStore; + + // procedure call parameter preparation + ReservationId reservationId = ReservationId.newInstance(Time.now(), 1); + String subHomeClusterId = "SC-1"; + addReservationHomeSubCluster(sqlFederationStateStore, + CALL_SP_ADD_RESERVATION_HOME_SUBCLUSTER, reservationId.toString(), subHomeClusterId); + + // call the delete method of the reservation + ReservationHomeSC resultHC = deleteReservationHomeSubCluster(sqlFederationStateStore, + CALL_SP_DELETE_RESERVATION_HOME_SUBCLUSTER, reservationId.toString()); + + Assert.assertNotNull(resultHC); + Assert.assertEquals(1, resultHC.dbUpdateCount); + + // call getReservationHomeSubCluster to get the result + ReservationHomeSC resultHC1 = getReservationHomeSubCluster(sqlFederationStateStore, + CALL_SP_GET_RESERVATION_HOME_SUBCLUSTER, reservationId.toString()); + Assert.assertNotNull(resultHC1); + Assert.assertEquals(null, resultHC1.subHomeClusterId); + } + + /** + * This test case is used to verify the processing logic of the incorrect number of + * updated records returned by the database when AddReservationHomeSubCluster is used. + * + * The probability of the database returning an update record greater than 1 is very low. + * + * @throws Exception when the error occurs + */ + @Test + public void testAddReservationHomeSubClusterAbnormalSituation() throws Exception { + FederationStateStore stateStore = getStateStore(); + Assert.assertTrue(stateStore instanceof SQLFederationStateStore); + + SQLFederationStateStore sqlFederationStateStore = (SQLFederationStateStore) stateStore; + + Connection conn = sqlFederationStateStore.conn; + conn.prepareStatement(SP_DROP_ADDRESERVATIONHOMESUBCLUSTER).execute(); + conn.prepareStatement(SP_ADDRESERVATIONHOMESUBCLUSTER2).execute(); + + ReservationId reservationId = ReservationId.newInstance(Time.now(), 1); + SubClusterId subClusterId = SubClusterId.newInstance("SC"); + + ReservationHomeSubCluster reservationHomeSubCluster = + ReservationHomeSubCluster.newInstance(reservationId, subClusterId); + AddReservationHomeSubClusterRequest request = + AddReservationHomeSubClusterRequest.newInstance(reservationHomeSubCluster); + + String errorMsg = String.format( + "Wrong behavior during the insertion of subCluster %s according to reservation %s. " + + "The database expects to insert 1 record, but the number of " + + "inserted changes is greater than 1, " + + "please check the records of the database.", subClusterId, reservationId); + + LambdaTestUtils.intercept(YarnException.class, errorMsg, + () -> stateStore.addReservationHomeSubCluster(request)); + } + + /** + * This test case is used to verify the logic when calling the updateReservationHomeSubCluster + * method if the database returns an inaccurate result. + * + * The probability of the database returning an update record greater than 1 is very low. + * + * @throws Exception when the error occurs + */ + @Test + public void testUpdateReservationHomeSubClusterAbnormalSituation() throws Exception { + FederationStateStore stateStore = getStateStore(); + Assert.assertTrue(stateStore instanceof SQLFederationStateStore); + + SQLFederationStateStore sqlFederationStateStore = (SQLFederationStateStore) stateStore; + + Connection conn = sqlFederationStateStore.conn; + conn.prepareStatement(SP_DROP_UPDATERESERVATIONHOMESUBCLUSTER).execute(); + conn.prepareStatement(SP_UPDATERESERVATIONHOMESUBCLUSTER2).execute(); + + ReservationId reservationId = ReservationId.newInstance(Time.now(), 1); + SubClusterId subClusterId1 = SubClusterId.newInstance("SC"); + + // add Reservation data. + ReservationHomeSubCluster reservationHomeSubCluster = + ReservationHomeSubCluster.newInstance(reservationId, subClusterId1); + AddReservationHomeSubClusterRequest addRequest = + AddReservationHomeSubClusterRequest.newInstance(reservationHomeSubCluster); + stateStore.addReservationHomeSubCluster(addRequest); + + SubClusterId subClusterId2 = SubClusterId.newInstance("SC2"); + ReservationHomeSubCluster reservationHomeSubCluster2 = + ReservationHomeSubCluster.newInstance(reservationId, subClusterId2); + UpdateReservationHomeSubClusterRequest updateRequest = + UpdateReservationHomeSubClusterRequest.newInstance(reservationHomeSubCluster2); + + String errorMsg = String.format( + "Wrong behavior during update the subCluster %s according to reservation %s. " + + "The database is expected to update 1 record, " + + "but the number of database update records is greater than 1, " + + "the records of the database should be checked.", + subClusterId2, reservationId); + + LambdaTestUtils.intercept(YarnException.class, errorMsg, + () -> stateStore.updateReservationHomeSubCluster(updateRequest)); + } + + /** + * This test case is used to verify the logic when calling the deleteReservationHomeSubCluster + * method if the database returns an inaccurate result. + * + * The probability of the database returning an update record greater than 1 is very low. + * + * @throws Exception when the error occurs + */ + @Test + public void testDeleteReservationHomeSubClusterAbnormalSituation() throws Exception { + FederationStateStore stateStore = getStateStore(); + Assert.assertTrue(stateStore instanceof SQLFederationStateStore); + + SQLFederationStateStore sqlFederationStateStore = (SQLFederationStateStore) stateStore; + + Connection conn = sqlFederationStateStore.conn; + conn.prepareStatement(SP_DROP_DELETERESERVATIONHOMESUBCLUSTER).execute(); + conn.prepareStatement(SP_DELETERESERVATIONHOMESUBCLUSTER2).execute(); + + ReservationId reservationId = ReservationId.newInstance(Time.now(), 1); + SubClusterId subClusterId1 = SubClusterId.newInstance("SC"); + + // add Reservation data. + ReservationHomeSubCluster reservationHomeSubCluster = + ReservationHomeSubCluster.newInstance(reservationId, subClusterId1); + AddReservationHomeSubClusterRequest addRequest = + AddReservationHomeSubClusterRequest.newInstance(reservationHomeSubCluster); + stateStore.addReservationHomeSubCluster(addRequest); + + DeleteReservationHomeSubClusterRequest delRequest = + DeleteReservationHomeSubClusterRequest.newInstance(reservationId); + + String errorMsg = String.format( + "Wrong behavior during deleting the reservation %s. " + + "The database is expected to delete 1 record, " + + "but the number of deleted records returned by the database is greater than 1, " + + "indicating that a duplicate reservationId occurred during the deletion process.", + reservationId); + + LambdaTestUtils.intercept(YarnException.class, errorMsg, + () -> stateStore.deleteReservationHomeSubCluster(delRequest)); + } } \ No newline at end of file diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/store/impl/TestZookeeperFederationStateStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/store/impl/TestZookeeperFederationStateStore.java index fe28641eb2560..272394b6b285b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/store/impl/TestZookeeperFederationStateStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/store/impl/TestZookeeperFederationStateStore.java @@ -25,14 +25,21 @@ import org.apache.curator.test.TestingServer; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.CommonConfigurationKeys; +import org.apache.hadoop.metrics2.MetricsRecord; +import org.apache.hadoop.metrics2.impl.MetricsCollectorImpl; +import org.apache.hadoop.metrics2.impl.MetricsRecords; +import org.apache.hadoop.util.Time; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.server.federation.store.FederationStateStore; import org.junit.After; import org.junit.Before; +import org.junit.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import static org.junit.Assert.assertEquals; + /** * Unit tests for ZookeeperFederationStateStore. */ @@ -84,4 +91,81 @@ protected FederationStateStore createStateStore() { super.setConf(getConf()); return new ZookeeperFederationStateStore(); } + + @Test + public void testMetricsInited() throws Exception { + ZookeeperFederationStateStore zkStateStore = (ZookeeperFederationStateStore) createStateStore(); + ZKFederationStateStoreOpDurations zkStateStoreOpDurations = zkStateStore.getOpDurations(); + MetricsCollectorImpl collector = new MetricsCollectorImpl(); + + long anyDuration = 10; + long start = Time.now(); + long end = start + anyDuration; + + zkStateStoreOpDurations.addAppHomeSubClusterDuration(start, end); + zkStateStoreOpDurations.addUpdateAppHomeSubClusterDuration(start, end); + zkStateStoreOpDurations.addGetAppHomeSubClusterDuration(start, end); + zkStateStoreOpDurations.addGetAppsHomeSubClusterDuration(start, end); + zkStateStoreOpDurations.addDeleteAppHomeSubClusterDuration(start, end); + zkStateStoreOpDurations.addRegisterSubClusterDuration(start, end); + zkStateStoreOpDurations.addDeregisterSubClusterDuration(start, end); + zkStateStoreOpDurations.addSubClusterHeartbeatDuration(start, end); + zkStateStoreOpDurations.addGetSubClusterDuration(start, end); + zkStateStoreOpDurations.addGetSubClustersDuration(start, end); + zkStateStoreOpDurations.addGetPolicyConfigurationDuration(start, end); + zkStateStoreOpDurations.addSetPolicyConfigurationDuration(start, end); + zkStateStoreOpDurations.addGetPoliciesConfigurationsDuration(start, end); + zkStateStoreOpDurations.addReservationHomeSubClusterDuration(start, end); + zkStateStoreOpDurations.addGetReservationHomeSubClusterDuration(start, end); + zkStateStoreOpDurations.addGetReservationsHomeSubClusterDuration(start, end); + zkStateStoreOpDurations.addDeleteReservationHomeSubClusterDuration(start, end); + zkStateStoreOpDurations.addUpdateReservationHomeSubClusterDuration(start, end); + + zkStateStoreOpDurations.getMetrics(collector, true); + assertEquals("Incorrect number of perf metrics", 1, collector.getRecords().size()); + + MetricsRecord record = collector.getRecords().get(0); + MetricsRecords.assertTag(record, ZKFederationStateStoreOpDurations.RECORD_INFO.name(), + "ZKFederationStateStoreOpDurations"); + + double expectAvgTime = anyDuration; + MetricsRecords.assertMetric(record, "AddAppHomeSubClusterAvgTime", expectAvgTime); + MetricsRecords.assertMetric(record, "UpdateAppHomeSubClusterAvgTime", expectAvgTime); + MetricsRecords.assertMetric(record, "GetAppHomeSubClusterAvgTime", expectAvgTime); + MetricsRecords.assertMetric(record, "GetAppsHomeSubClusterAvgTime", expectAvgTime); + MetricsRecords.assertMetric(record, "DeleteAppHomeSubClusterAvgTime", expectAvgTime); + MetricsRecords.assertMetric(record, "RegisterSubClusterAvgTime", expectAvgTime); + MetricsRecords.assertMetric(record, "DeregisterSubClusterAvgTime", expectAvgTime); + MetricsRecords.assertMetric(record, "SubClusterHeartbeatAvgTime", expectAvgTime); + MetricsRecords.assertMetric(record, "GetSubClusterAvgTime", expectAvgTime); + MetricsRecords.assertMetric(record, "GetSubClustersAvgTime", expectAvgTime); + MetricsRecords.assertMetric(record, "GetPolicyConfigurationAvgTime", expectAvgTime); + MetricsRecords.assertMetric(record, "SetPolicyConfigurationAvgTime", expectAvgTime); + MetricsRecords.assertMetric(record, "GetPoliciesConfigurationsAvgTime", expectAvgTime); + MetricsRecords.assertMetric(record, "AddReservationHomeSubClusterAvgTime", expectAvgTime); + MetricsRecords.assertMetric(record, "GetReservationHomeSubClusterAvgTime", expectAvgTime); + MetricsRecords.assertMetric(record, "GetReservationsHomeSubClusterAvgTime", expectAvgTime); + MetricsRecords.assertMetric(record, "DeleteReservationHomeSubClusterAvgTime", expectAvgTime); + MetricsRecords.assertMetric(record, "UpdateReservationHomeSubClusterAvgTime", expectAvgTime); + + long expectOps = 1; + MetricsRecords.assertMetric(record, "AddAppHomeSubClusterNumOps", expectOps); + MetricsRecords.assertMetric(record, "UpdateAppHomeSubClusterNumOps", expectOps); + MetricsRecords.assertMetric(record, "GetAppHomeSubClusterNumOps", expectOps); + MetricsRecords.assertMetric(record, "GetAppsHomeSubClusterNumOps", expectOps); + MetricsRecords.assertMetric(record, "DeleteAppHomeSubClusterNumOps", expectOps); + MetricsRecords.assertMetric(record, "RegisterSubClusterNumOps", expectOps); + MetricsRecords.assertMetric(record, "DeregisterSubClusterNumOps", expectOps); + MetricsRecords.assertMetric(record, "SubClusterHeartbeatNumOps", expectOps); + MetricsRecords.assertMetric(record, "GetSubClusterNumOps", expectOps); + MetricsRecords.assertMetric(record, "GetSubClustersNumOps", expectOps); + MetricsRecords.assertMetric(record, "GetPolicyConfigurationNumOps", expectOps); + MetricsRecords.assertMetric(record, "SetPolicyConfigurationNumOps", expectOps); + MetricsRecords.assertMetric(record, "GetPoliciesConfigurationsNumOps", expectOps); + MetricsRecords.assertMetric(record, "AddReservationHomeSubClusterNumOps", expectOps); + MetricsRecords.assertMetric(record, "GetReservationHomeSubClusterNumOps", expectOps); + MetricsRecords.assertMetric(record, "GetReservationsHomeSubClusterNumOps", expectOps); + MetricsRecords.assertMetric(record, "DeleteReservationHomeSubClusterNumOps", expectOps); + MetricsRecords.assertMetric(record, "UpdateReservationHomeSubClusterNumOps", expectOps); + } } \ No newline at end of file diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/utils/FederationPoliciesTestUtil.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/utils/FederationPoliciesTestUtil.java index a9b9029b25732..6ae64d555b7e2 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/utils/FederationPoliciesTestUtil.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/utils/FederationPoliciesTestUtil.java @@ -159,6 +159,51 @@ public static void initializePolicyContext( new Configuration()); } + public static FederationPolicyInitializationContext initializePolicyContext2( + ConfigurableFederationPolicy policy, WeightedPolicyInfo policyInfo, + Map activeSubClusters, + FederationStateStoreFacade facade) throws YarnException { + FederationPolicyInitializationContext context = + new FederationPolicyInitializationContext(null, initResolver(), facade, + SubClusterId.newInstance("homesubcluster")); + return initializePolicyContext2(context, policy, policyInfo, activeSubClusters); + } + + public static FederationPolicyInitializationContext initializePolicyContext2( + ConfigurableFederationPolicy policy, WeightedPolicyInfo policyInfo, + Map activeSubClusters) + throws YarnException { + return initializePolicyContext2(policy, policyInfo, activeSubClusters, initFacade()); + } + + public static FederationPolicyInitializationContext initializePolicyContext2( + FederationPolicyInitializationContext fpc, + ConfigurableFederationPolicy policy, WeightedPolicyInfo policyInfo, + Map activeSubClusters) + throws YarnException { + ByteBuffer buf = policyInfo.toByteBuffer(); + fpc.setSubClusterPolicyConfiguration(SubClusterPolicyConfiguration + .newInstance("queue1", policy.getClass().getCanonicalName(), buf)); + + if (fpc.getFederationStateStoreFacade() == null) { + FederationStateStoreFacade facade = FederationStateStoreFacade.getInstance(); + FederationStateStore fss = mock(FederationStateStore.class); + + if (activeSubClusters == null) { + activeSubClusters = new HashMap<>(); + } + + GetSubClustersInfoResponse response = GetSubClustersInfoResponse.newInstance( + activeSubClusters.values()); + + when(fss.getSubClusters(any())).thenReturn(response); + facade.reinitialize(fss, new Configuration()); + fpc.setFederationStateStoreFacade(facade); + } + policy.reinitialize(fpc); + return fpc; + } + /** * Initialize a {@link SubClusterResolver}. * diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/utils/FederationStateStoreTestUtil.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/utils/FederationStateStoreTestUtil.java index 46cf034468313..daf885a2171d2 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/utils/FederationStateStoreTestUtil.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/utils/FederationStateStoreTestUtil.java @@ -22,6 +22,7 @@ import java.util.List; import org.apache.hadoop.yarn.api.records.ApplicationId; +import org.apache.hadoop.yarn.api.records.ReservationId; import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.server.federation.store.FederationStateStore; import org.apache.hadoop.yarn.server.federation.store.records.AddApplicationHomeSubClusterRequest; @@ -38,6 +39,8 @@ import org.apache.hadoop.yarn.server.federation.store.records.SubClusterInfo; import org.apache.hadoop.yarn.server.federation.store.records.SubClusterPolicyConfiguration; import org.apache.hadoop.yarn.server.federation.store.records.SubClusterRegisterRequest; +import org.apache.hadoop.yarn.server.federation.store.records.GetReservationHomeSubClusterResponse; +import org.apache.hadoop.yarn.server.federation.store.records.GetReservationHomeSubClusterRequest; import org.apache.hadoop.yarn.server.federation.store.records.SubClusterState; import org.apache.hadoop.yarn.util.MonotonicClock; @@ -178,4 +181,12 @@ public void deRegisterSubCluster(SubClusterId subClusterId) .newInstance(subClusterId, SubClusterState.SC_UNREGISTERED)); } + public SubClusterId queryReservationHomeSC(ReservationId reservationId) + throws YarnException { + GetReservationHomeSubClusterRequest request = + GetReservationHomeSubClusterRequest.newInstance(reservationId); + GetReservationHomeSubClusterResponse response = + stateStore.getReservationHomeSubCluster(request); + return response.getReservationHomeSubCluster().getHomeSubCluster(); + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/utils/TestFederationStateStoreFacade.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/utils/TestFederationStateStoreFacade.java index 56fa0524a39dc..0606f5c454081 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/utils/TestFederationStateStoreFacade.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/utils/TestFederationStateStoreFacade.java @@ -41,6 +41,8 @@ import org.junit.runners.Parameterized; import org.junit.runners.Parameterized.Parameters; +import javax.cache.Cache; + /** * Unit tests for FederationStateStoreFacade. */ @@ -64,12 +66,14 @@ public static Collection getParameters() { private FederationStateStoreTestUtil stateStoreTestUtil; private FederationStateStoreFacade facade = FederationStateStoreFacade.getInstance(); + private Boolean isCachingEnabled; public TestFederationStateStoreFacade(Boolean isCachingEnabled) { conf = new Configuration(); if (!(isCachingEnabled.booleanValue())) { conf.setInt(YarnConfiguration.FEDERATION_CACHE_TIME_TO_LIVE_SECS, 0); } + this.isCachingEnabled = isCachingEnabled; } @Before @@ -206,4 +210,26 @@ public void testAddApplicationHomeSubCluster() throws YarnException { Assert.assertEquals(subClusterId1, result); } + @Test + public void testGetApplicationHomeSubClusterCache() throws YarnException { + ApplicationId appId = ApplicationId.newInstance(clusterTs, numApps + 1); + SubClusterId subClusterId1 = SubClusterId.newInstance("Home1"); + + ApplicationHomeSubCluster appHomeSubCluster = + ApplicationHomeSubCluster.newInstance(appId, subClusterId1); + SubClusterId subClusterIdAdd = facade.addApplicationHomeSubCluster(appHomeSubCluster); + + SubClusterId subClusterIdByFacade = facade.getApplicationHomeSubCluster(appId); + Assert.assertEquals(subClusterIdByFacade, subClusterIdAdd); + Assert.assertEquals(subClusterId1, subClusterIdAdd); + + if (isCachingEnabled.booleanValue()) { + Cache cache = facade.getCache(); + Object cacheKey = facade.getAppHomeSubClusterCacheRequest(appId); + Object subClusterIdByCache = cache.get(cacheKey); + Assert.assertEquals(subClusterIdByFacade, subClusterIdByCache); + Assert.assertEquals(subClusterId1, subClusterIdByCache); + } + } + } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/uam/TestUnmanagedApplicationManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/uam/TestUnmanagedApplicationManager.java index abb1d93c3399e..ddc85bf3e6fac 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/uam/TestUnmanagedApplicationManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/uam/TestUnmanagedApplicationManager.java @@ -20,10 +20,16 @@ import java.io.IOException; import java.security.PrivilegedExceptionAction; +import java.util.Map; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.security.token.Token; +import org.apache.hadoop.service.Service; +import org.apache.hadoop.test.GenericTestUtils; +import org.apache.hadoop.util.Time; import org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest; import org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse; import org.apache.hadoop.yarn.api.protocolrecords.FinishApplicationMasterRequest; @@ -39,6 +45,7 @@ import org.apache.hadoop.yarn.server.AMRMClientRelayer; import org.apache.hadoop.yarn.server.MockResourceManagerFacade; import org.apache.hadoop.yarn.util.AsyncCallback; +import org.junit.After; import org.junit.Assert; import org.junit.Before; import org.junit.Test; @@ -58,6 +65,9 @@ public class TestUnmanagedApplicationManager { private ApplicationAttemptId attemptId; + private UnmanagedAMPoolManager uamPool; + private ExecutorService threadpool; + @Before public void setup() { conf.set(YarnConfiguration.RM_CLUSTER_ID, "subclusterId"); @@ -69,6 +79,29 @@ public void setup() { uam = new TestableUnmanagedApplicationManager(conf, attemptId.getApplicationId(), null, "submitter", "appNameSuffix", true, "rm"); + + threadpool = Executors.newCachedThreadPool(); + uamPool = new TestableUnmanagedAMPoolManager(this.threadpool); + uamPool.init(conf); + uamPool.start(); + } + + @After + public void tearDown() throws IOException, InterruptedException { + if (uam != null) { + uam.shutDownConnections(); + uam = null; + } + if (uamPool != null) { + if (uamPool.isInState(Service.STATE.STARTED)) { + uamPool.stop(); + } + uamPool = null; + } + if (threadpool != null) { + threadpool.shutdownNow(); + threadpool = null; + } } protected void waitForCallBackCountAndCheckZeroPending( @@ -464,4 +497,49 @@ public Object run() { } } + protected class TestableUnmanagedAMPoolManager extends UnmanagedAMPoolManager { + public TestableUnmanagedAMPoolManager(ExecutorService threadpool) { + super(threadpool); + } + + @Override + public UnmanagedApplicationManager createUAM(Configuration configuration, + ApplicationId appId, String queueName, String submitter, String appNameSuffix, + boolean keepContainersAcrossApplicationAttempts, String rmId) { + return new TestableUnmanagedApplicationManager(configuration, appId, queueName, submitter, + appNameSuffix, keepContainersAcrossApplicationAttempts, rmId); + } + } + + @Test + public void testSeparateThreadWithoutBlockServiceStop() throws Exception { + ApplicationAttemptId attemptId1 = + ApplicationAttemptId.newInstance(ApplicationId.newInstance(Time.now(), 1), 1); + Token token1 = uamPool.launchUAM("SC-1", this.conf, + attemptId1.getApplicationId(), "default", "test-user", "SC-HOME", true, "SC-1"); + Assert.assertNotNull(token1); + + ApplicationAttemptId attemptId2 = + ApplicationAttemptId.newInstance(ApplicationId.newInstance(Time.now(), 2), 1); + Token token2 = uamPool.launchUAM("SC-2", this.conf, + attemptId2.getApplicationId(), "default", "test-user", "SC-HOME", true, "SC-2"); + Assert.assertNotNull(token2); + + Map unmanagedAppMasterMap = + uamPool.getUnmanagedAppMasterMap(); + Assert.assertNotNull(unmanagedAppMasterMap); + Assert.assertEquals(2, unmanagedAppMasterMap.size()); + + // try to stop uamPool + uamPool.stop(); + Assert.assertTrue(uamPool.waitForServiceToStop(2000)); + // process force finish Application in a separate thread, not blocking the main thread + Assert.assertEquals(Service.STATE.STOPPED, uamPool.getServiceState()); + + // Wait for the thread to terminate, check if uamPool#unmanagedAppMasterMap is 0 + Thread finishApplicationThread = uamPool.getFinishApplicationThread(); + GenericTestUtils.waitFor(() -> !finishApplicationThread.isAlive(), + 100, 2000); + Assert.assertEquals(0, unmanagedAppMasterMap.size()); + } } \ No newline at end of file diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/pom.xml index b8697635205df..24be1ebd2a96e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/pom.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/pom.xml @@ -200,10 +200,10 @@ test-jar test - - javax.ws.rs - javax.ws.rs-api + org.apache.hadoop + hadoop-minikdc + test diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DefaultContainerExecutor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DefaultContainerExecutor.java index 7d18016d95ade..705ef88dee357 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DefaultContainerExecutor.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DefaultContainerExecutor.java @@ -20,6 +20,7 @@ import static org.apache.hadoop.fs.CreateFlag.CREATE; import static org.apache.hadoop.fs.CreateFlag.OVERWRITE; +import static org.apache.hadoop.yarn.conf.YarnConfiguration.numaAwarenessEnabled; import org.apache.hadoop.classification.VisibleForTesting; import java.io.DataOutputStream; @@ -28,12 +29,13 @@ import java.io.IOException; import java.io.PrintStream; import java.net.InetSocketAddress; -import java.util.ArrayList; import java.util.Arrays; +import java.util.ArrayList; import java.util.EnumSet; import java.util.List; import java.util.Map; import java.util.Optional; + import org.apache.commons.lang3.RandomUtils; import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.fs.FileContext; @@ -51,14 +53,19 @@ import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.exceptions.ConfigurationException; +import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider; import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container; import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerDiagnosticsUpdateEvent; import org.apache.hadoop.yarn.server.nodemanager.containermanager.launcher.ContainerLaunch; +import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources.ResourceHandlerException; +import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources.numa.NumaResourceAllocation; +import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources.numa.NumaResourceAllocator; import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.ContainerLocalizer; import org.apache.hadoop.yarn.server.nodemanager.containermanager.runtime.ContainerExecutionException; import org.apache.hadoop.yarn.server.nodemanager.executor.ContainerExecContext; import org.apache.hadoop.yarn.server.nodemanager.executor.ContainerLivenessContext; +import org.apache.hadoop.yarn.server.nodemanager.executor.ContainerReacquisitionContext; import org.apache.hadoop.yarn.server.nodemanager.executor.ContainerReapContext; import org.apache.hadoop.yarn.server.nodemanager.executor.ContainerSignalContext; import org.apache.hadoop.yarn.server.nodemanager.executor.ContainerStartContext; @@ -86,6 +93,10 @@ public class DefaultContainerExecutor extends ContainerExecutor { private String logDirPermissions = null; + private NumaResourceAllocator numaResourceAllocator; + + + private String numactl; /** * Default constructor for use in testing. */ @@ -137,7 +148,19 @@ protected void setScriptExecutable(Path script, String owner) @Override public void init(Context nmContext) throws IOException { - // nothing to do or verify here + if(numaAwarenessEnabled(getConf())) { + numaResourceAllocator = new NumaResourceAllocator(nmContext); + numactl = this.getConf().get(YarnConfiguration.NM_NUMA_AWARENESS_NUMACTL_CMD, + YarnConfiguration.DEFAULT_NM_NUMA_AWARENESS_NUMACTL_CMD); + try { + numaResourceAllocator.init(this.getConf()); + LOG.info("NUMA resources allocation is enabled in DefaultContainer Executor," + + " Successfully initialized NUMA resources allocator."); + } catch (YarnException e) { + LOG.warn("Improper NUMA configuration provided.", e); + throw new IOException("Failed to initialize configured numa subsystem!"); + } + } } @Override @@ -300,11 +323,28 @@ public int launchContainer(ContainerStartContext ctx) setScriptExecutable(launchDst, user); setScriptExecutable(sb.getWrapperScriptPath(), user); + // adding numa commands based on configuration + String[] numaCommands = new String[]{}; + + if (numaResourceAllocator != null) { + try { + NumaResourceAllocation numaResourceAllocation = + numaResourceAllocator.allocateNumaNodes(container); + if (numaResourceAllocation != null) { + numaCommands = getNumaCommands(numaResourceAllocation); + } + } catch (ResourceHandlerException e) { + LOG.error("NumaResource Allocation failed!", e); + throw new IOException("NumaResource Allocation Error!", e); + } + } + shExec = buildCommandExecutor(sb.getWrapperScriptPath().toString(), - containerIdStr, user, pidFile, container.getResource(), - new File(containerWorkDir.toUri().getPath()), - container.getLaunchContext().getEnvironment()); - + containerIdStr, user, pidFile, container.getResource(), + new File(containerWorkDir.toUri().getPath()), + container.getLaunchContext().getEnvironment(), + numaCommands); + if (isContainerActive(containerId)) { shExec.execute(); } else { @@ -350,6 +390,7 @@ public int launchContainer(ContainerStartContext ctx) return exitCode; } finally { if (shExec != null) shExec.close(); + postComplete(containerId); } return 0; } @@ -372,16 +413,22 @@ public int relaunchContainer(ContainerStartContext ctx) * as the current working directory for the command. If null, * the current working directory is not modified. * @param environment the container environment + * @param numaCommands list of prefix numa commands * @return the new {@link ShellCommandExecutor} * @see ShellCommandExecutor */ - protected CommandExecutor buildCommandExecutor(String wrapperScriptPath, - String containerIdStr, String user, Path pidFile, Resource resource, - File workDir, Map environment) { - + protected CommandExecutor buildCommandExecutor(String wrapperScriptPath, + String containerIdStr, String user, Path pidFile, Resource resource, + File workDir, Map environment, String[] numaCommands) { + String[] command = getRunCommand(wrapperScriptPath, containerIdStr, user, pidFile, this.getConf(), resource); + // check if numa commands are passed and append it as prefix commands + if(numaCommands != null && numaCommands.length!=0) { + command = concatStringCommands(numaCommands, command); + } + LOG.info("launchContainer: {}", Arrays.toString(command)); return new ShellCommandExecutor( command, @@ -1040,4 +1087,92 @@ public void updateYarnSysFS(Context ctx, String user, String appId, String spec) throws IOException { throw new ServiceStateException("Implementation unavailable"); } + + @Override + public int reacquireContainer(ContainerReacquisitionContext ctx) + throws IOException, InterruptedException { + try { + if (numaResourceAllocator != null) { + numaResourceAllocator.recoverNumaResource(ctx.getContainerId()); + } + return super.reacquireContainer(ctx); + } finally { + postComplete(ctx.getContainerId()); + } + } + + /** + * clean up and release of resources. + * + * @param containerId containerId of running container + */ + public void postComplete(final ContainerId containerId) { + if (numaResourceAllocator != null) { + try { + numaResourceAllocator.releaseNumaResource(containerId); + } catch (ResourceHandlerException e) { + LOG.warn("NumaResource release failed for " + + "containerId: {}. Exception: ", containerId, e); + } + } + } + + /** + * @param resourceAllocation NonNull NumaResourceAllocation object reference + * @return Array of numa specific commands + */ + String[] getNumaCommands(NumaResourceAllocation resourceAllocation) { + String[] numaCommand = new String[3]; + numaCommand[0] = numactl; + numaCommand[1] = "--interleave=" + String.join(",", resourceAllocation.getMemNodes()); + numaCommand[2] = "--cpunodebind=" + String.join(",", resourceAllocation.getCpuNodes()); + return numaCommand; + + } + + /** + * @param firstStringArray Array of String + * @param secondStringArray Array of String + * @return combined array of string where first elements are from firstStringArray + * and later are the elements from secondStringArray + */ + String[] concatStringCommands(String[] firstStringArray, String[] secondStringArray) { + + if(firstStringArray == null && secondStringArray == null) { + return secondStringArray; + } + + else if(firstStringArray == null || firstStringArray.length == 0) { + return secondStringArray; + } + + else if(secondStringArray == null || secondStringArray.length == 0){ + return firstStringArray; + } + + int len = firstStringArray.length + secondStringArray.length; + + String[] ret = new String[len]; + int idx = 0; + for (String s : firstStringArray) { + ret[idx] = s; + idx++; + } + for (String s : secondStringArray) { + ret[idx] = s; + idx++; + } + return ret; + } + + @VisibleForTesting + public void setNumaResourceAllocator(NumaResourceAllocator numaResourceAllocator) { + this.numaResourceAllocator = numaResourceAllocator; + } + + @VisibleForTesting + public void setNumactl(String numactl) { + this.numactl = numactl; + } + } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java index 4dadf9c62e4c5..be7d7e52ba9da 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java @@ -201,7 +201,7 @@ protected void serviceInit(Configuration conf) throws Exception { // Update configured resources via plugins. updateConfiguredResourcesViaPlugins(totalResource); - LOG.info("Nodemanager resources is set to: " + totalResource); + LOG.info("Nodemanager resources is set to: {}.", totalResource); metrics.addResource(totalResource); @@ -247,9 +247,8 @@ protected void serviceInit(Configuration conf) throws Exception { LOG.debug("{} :{}", YARN_NODEMANAGER_DURATION_TO_TRACK_STOPPED_CONTAINERS, durationToTrackStoppedContainers); super.serviceInit(conf); - LOG.info("Initialized nodemanager with :" + - " physical-memory=" + memoryMb + " virtual-memory=" + virtualMemoryMb + - " virtual-cores=" + virtualCores); + LOG.info("Initialized nodemanager with : physical-memory={} virtual-memory={} " + + "virtual-cores={}.", memoryMb, virtualMemoryMb, virtualCores); this.logAggregationEnabled = conf.getBoolean(YarnConfiguration.LOG_AGGREGATION_ENABLED, @@ -264,7 +263,7 @@ protected void serviceStart() throws Exception { // NodeManager is the last service to start, so NodeId is available. this.nodeId = this.context.getNodeId(); - LOG.info("Node ID assigned is : " + this.nodeId); + LOG.info("Node ID assigned is : {}.", this.nodeId); this.httpPort = this.context.getHttpPort(); this.nodeManagerVersionId = YarnVersionInfo.getVersion(); try { @@ -312,10 +311,9 @@ private void unRegisterNM() { request.setNodeId(this.nodeId); try { resourceTracker.unRegisterNodeManager(request); - LOG.info("Successfully Unregistered the Node " + this.nodeId - + " with ResourceManager."); + LOG.info("Successfully Unregistered the Node {} with ResourceManager.", this.nodeId); } catch (Exception e) { - LOG.warn("Unregistration of the Node " + this.nodeId + " failed.", e); + LOG.warn("Unregistration of the Node {} failed.", this.nodeId, e); } } @@ -399,7 +397,7 @@ nodeManagerVersionId, containerReports, getRunningApplications(), nodeLabels, physicalResource, nodeAttributes, nodeStatus); if (containerReports != null && !containerReports.isEmpty()) { - LOG.info("Registering with RM using containers :" + containerReports); + LOG.info("Registering with RM using containers.size : {}." + containerReports.size()); } if (logAggregationEnabled) { // pull log aggregation status for application running in this NM @@ -641,6 +639,7 @@ private List getRunningApplications() { runningApplications.add(appEntry.getKey()); } } + LOG.info("Running Applications Size : {}.", runningApplications.size()); return runningApplications; } @@ -667,8 +666,8 @@ private List getNMContainerStatuses() throws IOException { } } if (!containerStatuses.isEmpty()) { - LOG.info("Sending out " + containerStatuses.size() - + " NM container statuses: " + containerStatuses); + LOG.info("Sending out {} container NM container statuses: {}.", + containerStatuses.size(), containerStatuses); } return containerStatuses; } @@ -724,8 +723,7 @@ public void removeOrTrackCompletedContainersFromContext( } if (!removedContainers.isEmpty()) { - LOG.info("Removed completed containers from NM context: " - + removedContainers); + LOG.info("Removed completed containers from NM context: {}.", removedContainers); } } @@ -792,7 +790,7 @@ public void removeVeryOldStoppedContainersFromCache() { try { context.getNMStateStore().removeContainer(cid); } catch (IOException e) { - LOG.error("Unable to remove container " + cid + " in store", e); + LOG.error("Unable to remove container {} in store.", cid, e); } } } @@ -839,18 +837,15 @@ private boolean handleShutdownOrResyncCommand( if (response.getNodeAction() == NodeAction.SHUTDOWN) { LOG.warn("Received SHUTDOWN signal from Resourcemanager as part of" + " heartbeat, hence shutting down."); - LOG.warn("Message from ResourceManager: " - + response.getDiagnosticsMessage()); + LOG.warn("Message from ResourceManager: {}.", response.getDiagnosticsMessage()); context.setDecommissioned(true); dispatcher.getEventHandler().handle( new NodeManagerEvent(NodeManagerEventType.SHUTDOWN)); return true; } if (response.getNodeAction() == NodeAction.RESYNC) { - LOG.warn("Node is out of sync with ResourceManager," - + " hence resyncing."); - LOG.warn("Message from ResourceManager: " - + response.getDiagnosticsMessage()); + LOG.warn("Node is out of sync with ResourceManager, hence resyncing."); + LOG.warn("Message from ResourceManager: {}.", response.getDiagnosticsMessage()); // Invalidate the RMIdentifier while resync NodeStatusUpdaterImpl.this.rmIdentifier = ResourceManagerConstants.RM_INVALID_IDENTIFIER; @@ -1095,8 +1090,7 @@ protected void validate(Set nodeAttributes) try { NodeLabelUtil.validateNodeAttributes(nodeAttributes); } catch (IOException e) { - LOG.error( - "Invalid node attribute(s) from Provider : " + e.getMessage()); + LOG.error("Invalid node attribute(s) from Provider : {}.", e.getMessage()); throw e; } } @@ -1136,9 +1130,8 @@ public void verifyRMHeartbeatResponseForNodeAttributes( } else { // case where updated node attributes from NodeAttributesProvider // is sent to RM and RM rejected the attributes - LOG.error("NM node attributes {" + getPreviousValue() - + "} were not accepted by RM and message from RM : " + response - .getDiagnosticsMessage()); + LOG.error("NM node attributes [{}] were not accepted by RM and message from RM : {}.", + getPreviousValue(), response.getDiagnosticsMessage()); } } } @@ -1262,7 +1255,7 @@ protected void validate(Set nodeLabels) } } if (hasInvalidLabel) { - LOG.error("Invalid Node Label(s) from Provider : " + errorMsg); + LOG.error("Invalid Node Label(s) from Provider : {}.", errorMsg); throw new IOException(errorMsg.toString()); } } @@ -1287,10 +1280,8 @@ public void verifyRMHeartbeatResponseForNodeLabels( } else { // case where updated labels from NodeLabelsProvider is sent to RM and // RM rejected the labels - LOG.error( - "NM node labels {" + StringUtils.join(",", getPreviousValue()) - + "} were not accepted by RM and message from RM : " - + response.getDiagnosticsMessage()); + LOG.error("NM node labels [{}] were not accepted by RM and message from RM : {}.", + StringUtils.join(",", getPreviousValue()), response.getDiagnosticsMessage()); } } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/WindowsSecureContainerExecutor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/WindowsSecureContainerExecutor.java index 6132e579ef982..9d57f8fff4fe5 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/WindowsSecureContainerExecutor.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/WindowsSecureContainerExecutor.java @@ -718,10 +718,10 @@ public void startLocalizer(LocalizerStartContext ctx) throws IOException, @Override protected CommandExecutor buildCommandExecutor(String wrapperScriptPath, String containerIdStr, String userName, Path pidFile, Resource resource, - File wordDir, Map environment) { + File wordDir, Map environment, String[] numaCommands) { return new WintuilsProcessStubExecutor( wordDir.toString(), - containerIdStr, userName, pidFile.toString(), + containerIdStr, userName, pidFile.toString(), "cmd /c " + wrapperScriptPath); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/AMRMProxyMetrics.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/AMRMProxyMetrics.java index 241eeeb62a85a..f5a30828a4dd2 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/AMRMProxyMetrics.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/AMRMProxyMetrics.java @@ -45,6 +45,16 @@ public final class AMRMProxyMetrics { private MutableGaugeLong failedAllocateRequests; @Metric("# of failed application recoveries") private MutableGaugeLong failedAppRecoveryCount; + @Metric("# of failed application stop") + private MutableGaugeLong failedAppStopRequests; + @Metric("# of failed update token") + private MutableGaugeLong failedUpdateAMRMTokenRequests; + @Metric("# all allocate requests count") + private MutableGaugeLong allocateCount; + @Metric("# all requests count") + private MutableGaugeLong requestCount; + + // Aggregate metrics are shared, and don't have to be looked up per call @Metric("Application start request latency(ms)") private MutableRate totalSucceededAppStartRequests; @@ -54,11 +64,22 @@ public final class AMRMProxyMetrics { private MutableRate totalSucceededFinishAMRequests; @Metric("Allocate latency(ms)") private MutableRate totalSucceededAllocateRequests; + @Metric("Application stop request latency(ms)") + private MutableRate totalSucceededAppStopRequests; + @Metric("Recover latency(ms)") + private MutableRate totalSucceededRecoverRequests; + @Metric("UpdateAMRMToken latency(ms)") + private MutableRate totalSucceededUpdateAMRMTokenRequests; + // Quantile latency in ms - this is needed for SLA (95%, 99%, etc) private MutableQuantiles applicationStartLatency; private MutableQuantiles registerAMLatency; private MutableQuantiles finishAMLatency; private MutableQuantiles allocateLatency; + private MutableQuantiles recoverLatency; + private MutableQuantiles applicationStopLatency; + private MutableQuantiles updateAMRMTokenLatency; + private static volatile AMRMProxyMetrics instance = null; private MetricsRegistry registry; @@ -78,6 +99,15 @@ private AMRMProxyMetrics() { allocateLatency = registry .newQuantiles("allocateLatency", "latency of allocate", "ops", "latency", 10); + applicationStopLatency = registry + .newQuantiles("applicationStopLatency", "latency of app stop", "ops", + "latency", 10); + recoverLatency = registry + .newQuantiles("recoverLatency", "latency of recover", "ops", + "latency", 10); + updateAMRMTokenLatency = registry + .newQuantiles("updateAMRMTokenLatency", "latency of update amrm token", "ops", + "latency", 10); } /** @@ -146,16 +176,57 @@ long getNumSucceededAllocateRequests() { return totalSucceededAllocateRequests.lastStat().numSamples(); } + @VisibleForTesting + long getNumSucceededAppStopRequests() { + return totalSucceededAppStopRequests.lastStat().numSamples(); + } + + @VisibleForTesting + long getNumSucceededRecoverRequests() { + return totalSucceededRecoverRequests.lastStat().numSamples(); + } + + @VisibleForTesting + long getNumSucceededUpdateAMRMTokenRequests() { + return totalSucceededUpdateAMRMTokenRequests.lastStat().numSamples(); + } + + @VisibleForTesting double getLatencySucceededAllocateRequests() { return totalSucceededAllocateRequests.lastStat().mean(); } + @VisibleForTesting + double getLatencySucceededAppStopRequests() { + return totalSucceededAppStopRequests.lastStat().mean(); + } + + @VisibleForTesting + double getLatencySucceededRecoverRequests() { + return totalSucceededRecoverRequests.lastStat().mean(); + } + public void succeededAllocateRequests(long duration) { totalSucceededAllocateRequests.add(duration); allocateLatency.add(duration); } + public void succeededAppStopRequests(long duration) { + totalSucceededAppStopRequests.add(duration); + applicationStopLatency.add(duration); + } + + public void succeededRecoverRequests(long duration) { + totalSucceededRecoverRequests.add(duration); + recoverLatency.add(duration); + } + + public void succeededUpdateTokenRequests(long duration) { + totalSucceededUpdateAMRMTokenRequests.add(duration); + updateAMRMTokenLatency.add(duration); + } + long getFailedAppStartRequests() { return failedAppStartRequests.value(); } @@ -195,4 +266,36 @@ long getFailedAppRecoveryCount() { public void incrFailedAppRecoveryCount() { failedAppRecoveryCount.incr(); } + + long getFailedAppStopRequests() { + return failedAppStopRequests.value(); + } + + public void incrFailedAppStopRequests() { + failedAppStopRequests.incr(); + } + + long getFailedUpdateAMRMTokenRequests() { + return failedUpdateAMRMTokenRequests.value(); + } + + public void incrFailedUpdateAMRMTokenRequests() { + failedUpdateAMRMTokenRequests.incr(); + } + + public void incrAllocateCount() { + allocateCount.incr(); + } + + public void incrRequestCount() { + requestCount.incr(); + } + + long getAllocateCount() { + return allocateCount.value(); + } + + long getRequestCount() { + return requestCount.value(); + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/AMRMProxyService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/AMRMProxyService.java index b0d66ca027d81..82873e07cdb09 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/AMRMProxyService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/AMRMProxyService.java @@ -21,6 +21,7 @@ import java.io.IOException; import java.net.InetSocketAddress; import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.Collection; import java.util.Iterator; @@ -53,6 +54,7 @@ import org.apache.hadoop.yarn.api.protocolrecords.StartContainerRequest; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ApplicationId; +import org.apache.hadoop.yarn.server.api.records.MasterKey; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.event.AsyncDispatcher; import org.apache.hadoop.yarn.event.EventHandler; @@ -123,12 +125,9 @@ public AMRMProxyService(Context nmContext, AsyncDispatcher dispatcher) { Preconditions.checkArgument(dispatcher != null, "dispatcher is null"); this.nmContext = nmContext; this.dispatcher = dispatcher; - this.applPipelineMap = - new ConcurrentHashMap(); - - this.dispatcher.register(ApplicationEventType.class, - new ApplicationEventHandler()); + this.applPipelineMap = new ConcurrentHashMap<>(); + this.dispatcher.register(ApplicationEventType.class, new ApplicationEventHandler()); metrics = AMRMProxyMetrics.getMetrics(); } @@ -155,7 +154,7 @@ protected void serviceInit(Configuration conf) throws Exception { @Override protected void serviceStart() throws Exception { - LOG.info("Starting AMRMProxyService"); + LOG.info("Starting AMRMProxyService."); Configuration conf = getConfig(); YarnRPC rpc = YarnRPC.create(conf); UserGroupInformation.setConfiguration(conf); @@ -182,27 +181,22 @@ protected void serviceStart() throws Exception { listenerEndpoint, serverConf, this.secretManager, numWorkerThreads); - if (conf - .getBoolean(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHORIZATION, - false)) { - this.server.refreshServiceAcl(conf, NMPolicyProvider.getInstance()); + if (conf.getBoolean(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHORIZATION, false)) { + this.server.refreshServiceAcl(conf, NMPolicyProvider.getInstance()); } this.server.start(); - LOG.info("AMRMProxyService listening on address: " - + this.server.getListenerAddress()); + LOG.info("AMRMProxyService listening on address: {}.", this.server.getListenerAddress()); super.serviceStart(); } @Override protected void serviceStop() throws Exception { - LOG.info("Stopping AMRMProxyService"); + LOG.info("Stopping AMRMProxyService."); if (this.server != null) { this.server.stop(); } - this.secretManager.stop(); - super.serviceStop(); } @@ -212,19 +206,21 @@ protected void serviceStop() throws Exception { * @throws IOException if recover fails */ public void recover() throws IOException { - LOG.info("Recovering AMRMProxyService"); + LOG.info("Recovering AMRMProxyService."); RecoveredAMRMProxyState state = this.nmContext.getNMStateStore().loadAMRMProxyState(); this.secretManager.recover(state); - LOG.info("Recovering {} running applications for AMRMProxy", + LOG.info("Recovering {} running applications for AMRMProxy.", state.getAppContexts().size()); + for (Map.Entry> entry : state .getAppContexts().entrySet()) { ApplicationAttemptId attemptId = entry.getKey(); - LOG.info("Recovering app attempt {}", attemptId); + LOG.info("Recovering app attempt {}.", attemptId); + long startTime = clock.getTime(); // Try recover for the running application attempt try { @@ -233,19 +229,18 @@ public void recover() throws IOException { for (Map.Entry contextEntry : entry.getValue() .entrySet()) { if (contextEntry.getKey().equals(NMSS_USER_KEY)) { - user = new String(contextEntry.getValue(), "UTF-8"); + user = new String(contextEntry.getValue(), StandardCharsets.UTF_8); } else if (contextEntry.getKey().equals(NMSS_AMRMTOKEN_KEY)) { amrmToken = new Token<>(); amrmToken.decodeFromUrlString( - new String(contextEntry.getValue(), "UTF-8")); + new String(contextEntry.getValue(), StandardCharsets.UTF_8)); // Clear the service field, as if RM just issued the token amrmToken.setService(new Text()); } } if (amrmToken == null) { - throw new IOException( - "No amrmToken found for app attempt " + attemptId); + throw new IOException("No amrmToken found for app attempt " + attemptId); } if (user == null) { throw new IOException("No user found for app attempt " + attemptId); @@ -258,14 +253,14 @@ public void recover() throws IOException { // Retrieve the AM container credentials from NM context Credentials amCred = null; for (Container container : this.nmContext.getContainers().values()) { - LOG.debug("From NM Context container {}", container.getContainerId()); + LOG.debug("From NM Context container {}.", container.getContainerId()); if (container.getContainerId().getApplicationAttemptId().equals( attemptId) && container.getContainerTokenIdentifier() != null) { - LOG.debug("Container type {}", + LOG.debug("Container type {}.", container.getContainerTokenIdentifier().getContainerType()); if (container.getContainerTokenIdentifier() .getContainerType() == ContainerType.APPLICATION_MASTER) { - LOG.info("AM container {} found in context, has credentials: {}", + LOG.info("AM container {} found in context, has credentials: {}.", container.getContainerId(), (container.getCredentials() != null)); amCred = container.getCredentials(); @@ -274,15 +269,17 @@ public void recover() throws IOException { } if (amCred == null) { LOG.error("No credentials found for AM container of {}. " - + "Yarn registry access might not work", attemptId); + + "Yarn registry access might not work.", attemptId); } - // Create the intercepter pipeline for the AM + // Create the interceptor pipeline for the AM initializePipeline(attemptId, user, amrmToken, localToken, entry.getValue(), true, amCred); + long endTime = clock.getTime(); + this.metrics.succeededRecoverRequests(endTime - startTime); } catch (Throwable e) { - LOG.error("Exception when recovering " + attemptId - + ", removing it from NMStateStore and move on", e); + LOG.error("Exception when recovering {}, removing it from NMStateStore and move on.", + attemptId, e); this.metrics.incrFailedAppRecoveryCount(); this.nmContext.getNMStateStore().removeAMRMProxyAppContext(attemptId); } @@ -292,26 +289,28 @@ public void recover() throws IOException { /** * This is called by the AMs started on this node to register with the RM. * This method does the initial authorization and then forwards the request to - * the application instance specific intercepter chain. + * the application instance specific interceptor chain. */ @Override public RegisterApplicationMasterResponse registerApplicationMaster( RegisterApplicationMasterRequest request) throws YarnException, IOException { + this.metrics.incrRequestCount(); long startTime = clock.getTime(); try { RequestInterceptorChainWrapper pipeline = authorizeAndGetInterceptorChain(); - LOG.info("Registering application master." + " Host:" + request.getHost() - + " Port:" + request.getRpcPort() + " Tracking Url:" + request - .getTrackingUrl() + " for application " + pipeline - .getApplicationAttemptId()); + + LOG.info("RegisteringAM Host: {}, Port: {}, Tracking Url: {} for application {}. ", + request.getHost(), request.getRpcPort(), request.getTrackingUrl(), + pipeline.getApplicationAttemptId()); + RegisterApplicationMasterResponse response = pipeline.getRootInterceptor().registerApplicationMaster(request); long endTime = clock.getTime(); this.metrics.succeededRegisterAMRequests(endTime - startTime); - LOG.info("RegisterAM processing finished in {} ms for application {}", + LOG.info("RegisterAM processing finished in {} ms for application {}.", endTime - startTime, pipeline.getApplicationAttemptId()); return response; } catch (Throwable t) { @@ -323,24 +322,25 @@ public RegisterApplicationMasterResponse registerApplicationMaster( /** * This is called by the AMs started on this node to unregister from the RM. * This method does the initial authorization and then forwards the request to - * the application instance specific intercepter chain. + * the application instance specific interceptor chain. */ @Override public FinishApplicationMasterResponse finishApplicationMaster( FinishApplicationMasterRequest request) throws YarnException, IOException { + this.metrics.incrRequestCount(); long startTime = clock.getTime(); try { RequestInterceptorChainWrapper pipeline = authorizeAndGetInterceptorChain(); - LOG.info("Finishing application master for {}. Tracking Url: {}", + LOG.info("Finishing application master for {}. Tracking Url: {}.", pipeline.getApplicationAttemptId(), request.getTrackingUrl()); FinishApplicationMasterResponse response = pipeline.getRootInterceptor().finishApplicationMaster(request); long endTime = clock.getTime(); this.metrics.succeededFinishAMRequests(endTime - startTime); - LOG.info("FinishAM finished with isUnregistered = {} in {} ms for {}", + LOG.info("FinishAM finished with isUnregistered = {} in {} ms for {}.", response.getIsUnregistered(), endTime - startTime, pipeline.getApplicationAttemptId()); return response; @@ -354,12 +354,13 @@ public FinishApplicationMasterResponse finishApplicationMaster( * This is called by the AMs started on this node to send heart beat to RM. * This method does the initial authorization and then forwards the request to * the application instance specific pipeline, which is a chain of request - * intercepter objects. One application request processing pipeline is created + * interceptor objects. One application request processing pipeline is created * per AM instance. */ @Override public AllocateResponse allocate(AllocateRequest request) throws YarnException, IOException { + this.metrics.incrAllocateCount(); long startTime = clock.getTime(); try { AMRMTokenIdentifier amrmTokenIdentifier = @@ -373,7 +374,7 @@ public AllocateResponse allocate(AllocateRequest request) long endTime = clock.getTime(); this.metrics.succeededAllocateRequests(endTime - startTime); - LOG.info("Allocate processing finished in {} ms for application {}", + LOG.info("Allocate processing finished in {} ms for application {}.", endTime - startTime, pipeline.getApplicationAttemptId()); return allocateResponse; } catch (Throwable t) { @@ -392,6 +393,7 @@ public AllocateResponse allocate(AllocateRequest request) */ public void processApplicationStartRequest(StartContainerRequest request) throws IOException, YarnException { + this.metrics.incrRequestCount(); long startTime = clock.getTime(); try { ContainerTokenIdentifier containerTokenIdentifierForKey = @@ -408,8 +410,7 @@ public void processApplicationStartRequest(StartContainerRequest request) if (!checkIfAppExistsInStateStore(applicationID)) { return; } - LOG.info("Callback received for initializing request " - + "processing pipeline for an AM"); + LOG.info("Callback received for initializing request processing pipeline for an AM."); Credentials credentials = YarnServerSecurityUtils .parseCredentials(request.getContainerLaunchContext()); @@ -417,8 +418,7 @@ public void processApplicationStartRequest(StartContainerRequest request) getFirstAMRMToken(credentials.getAllTokens()); if (amrmToken == null) { throw new YarnRuntimeException( - "AMRMToken not found in the start container request for " - + "application:" + appAttemptId.toString()); + "AMRMToken not found in the start container request for application:" + appAttemptId); } // Substitute the existing AMRM Token with a local one. Keep the rest of @@ -445,7 +445,7 @@ public void processApplicationStartRequest(StartContainerRequest request) } /** - * Initializes the request intercepter pipeline for the specified application. + * Initializes the request interceptor pipeline for the specified application. * * @param applicationAttemptId attempt id * @param user user name @@ -465,8 +465,7 @@ protected void initializePipeline(ApplicationAttemptId applicationAttemptId, .containsKey(applicationAttemptId.getApplicationId())) { LOG.warn("Request to start an already existing appId was received. " + " This can happen if an application failed and a new attempt " - + "was created on this machine. ApplicationId: " - + applicationAttemptId.toString()); + + "was created on this machine. ApplicationId: {}.", applicationAttemptId); RequestInterceptorChainWrapper chainWrapperBackup = this.applPipelineMap.get(applicationAttemptId.getApplicationId()); @@ -476,8 +475,7 @@ protected void initializePipeline(ApplicationAttemptId applicationAttemptId, .equals(applicationAttemptId)) { // TODO: revisit in AMRMProxy HA in YARN-6128 // Remove the existing pipeline - LOG.info("Remove the previous pipeline for ApplicationId: " - + applicationAttemptId.toString()); + LOG.info("Remove the previous pipeline for ApplicationId: {}.", applicationAttemptId); RequestInterceptorChainWrapper pipeline = applPipelineMap.remove(applicationAttemptId.getApplicationId()); @@ -485,19 +483,17 @@ protected void initializePipeline(ApplicationAttemptId applicationAttemptId, try { this.nmContext.getNMStateStore() .removeAMRMProxyAppContext(applicationAttemptId); - } catch (IOException e) { - LOG.error("Error removing AMRMProxy application context for " - + applicationAttemptId, e); + } catch (IOException ioe) { + LOG.error("Error removing AMRMProxy application context for {}.", + applicationAttemptId, ioe); } } try { pipeline.getRootInterceptor().shutdown(); } catch (Throwable ex) { - LOG.warn( - "Failed to shutdown the request processing pipeline for app:" - + applicationAttemptId.getApplicationId(), - ex); + LOG.warn("Failed to shutdown the request processing pipeline for app: {}.", + applicationAttemptId.getApplicationId(), ex); } } else { return; @@ -510,12 +506,11 @@ protected void initializePipeline(ApplicationAttemptId applicationAttemptId, } // We register the pipeline instance in the map first and then initialize it - // later because chain initialization can be expensive and we would like to + // later because chain initialization can be expensive, and we would like to // release the lock as soon as possible to prevent other applications from // blocking when one application's chain is initializing LOG.info("Initializing request processing pipeline for application. " - + " ApplicationId:" + applicationAttemptId + " for the user: " - + user); + + " ApplicationId: {} for the user: {}.", applicationAttemptId, user); try { RequestInterceptor interceptorChain = @@ -525,8 +520,7 @@ protected void initializePipeline(ApplicationAttemptId applicationAttemptId, user, amrmToken, localToken, credentials, this.registry)); if (isRecovery) { if (recoveredDataMap == null) { - throw new YarnRuntimeException( - "null recoveredDataMap recieved for recover"); + throw new YarnRuntimeException("null recoveredDataMap received for recover"); } interceptorChain.recover(recoveredDataMap); } @@ -535,13 +529,13 @@ protected void initializePipeline(ApplicationAttemptId applicationAttemptId, if (!isRecovery && this.nmContext.getNMStateStore() != null) { try { this.nmContext.getNMStateStore().storeAMRMProxyAppContextEntry( - applicationAttemptId, NMSS_USER_KEY, user.getBytes("UTF-8")); + applicationAttemptId, NMSS_USER_KEY, user.getBytes(StandardCharsets.UTF_8)); this.nmContext.getNMStateStore().storeAMRMProxyAppContextEntry( applicationAttemptId, NMSS_AMRMTOKEN_KEY, - amrmToken.encodeToUrlString().getBytes("UTF-8")); + amrmToken.encodeToUrlString().getBytes(StandardCharsets.UTF_8)); } catch (IOException e) { - LOG.error("Error storing AMRMProxy application context entry for " - + applicationAttemptId, e); + LOG.error("Error storing AMRMProxy application context entry for {}.", + applicationAttemptId, e); } } } catch (Exception e) { @@ -557,29 +551,27 @@ protected void initializePipeline(ApplicationAttemptId applicationAttemptId, * @param applicationId application id */ protected void stopApplication(ApplicationId applicationId) { - Preconditions.checkArgument(applicationId != null, - "applicationId is null"); + this.metrics.incrRequestCount(); + Preconditions.checkArgument(applicationId != null, "applicationId is null"); RequestInterceptorChainWrapper pipeline = this.applPipelineMap.remove(applicationId); + boolean isStopSuccess = true; + long startTime = clock.getTime(); if (pipeline == null) { - LOG.info( - "No interceptor pipeline for application {}," - + " likely because its AM is not run in this node.", - applicationId); + LOG.info("No interceptor pipeline for application {}," + + " likely because its AM is not run in this node.", applicationId); + isStopSuccess = false; } else { // Remove the appAttempt in AMRMTokenSecretManager - this.secretManager - .applicationMasterFinished(pipeline.getApplicationAttemptId()); - - LOG.info("Stopping the request processing pipeline for application: " - + applicationId); + this.secretManager.applicationMasterFinished(pipeline.getApplicationAttemptId()); + LOG.info("Stopping the request processing pipeline for application: {}.", applicationId); try { pipeline.getRootInterceptor().shutdown(); } catch (Throwable ex) { - LOG.warn( - "Failed to shutdown the request processing pipeline for app:" - + applicationId, ex); + LOG.warn("Failed to shutdown the request processing pipeline for app: {}.", + applicationId, ex); + isStopSuccess = false; } // Remove the app context from NMSS after the interceptors are shutdown @@ -588,74 +580,83 @@ protected void stopApplication(ApplicationId applicationId) { this.nmContext.getNMStateStore() .removeAMRMProxyAppContext(pipeline.getApplicationAttemptId()); } catch (IOException e) { - LOG.error("Error removing AMRMProxy application context for " - + applicationId, e); + LOG.error("Error removing AMRMProxy application context for {}.", + applicationId, e); + isStopSuccess = false; } } } + + if (isStopSuccess) { + long endTime = clock.getTime(); + this.metrics.succeededAppStopRequests(endTime - startTime); + } else { + this.metrics.incrFailedAppStopRequests(); + } } private void updateAMRMTokens(AMRMTokenIdentifier amrmTokenIdentifier, RequestInterceptorChainWrapper pipeline, AllocateResponse allocateResponse) { + AMRMProxyApplicationContextImpl context = - (AMRMProxyApplicationContextImpl) pipeline.getRootInterceptor() - .getApplicationContext(); + (AMRMProxyApplicationContextImpl) pipeline.getRootInterceptor().getApplicationContext(); + + try { + long startTime = clock.getTime(); - // check to see if the RM has issued a new AMRMToken & accordingly update - // the real ARMRMToken in the current context - if (allocateResponse.getAMRMToken() != null) { - LOG.info("RM rolled master-key for amrm-tokens"); + // check to see if the RM has issued a new AMRMToken & accordingly update + // the real ARMRMToken in the current context + if (allocateResponse.getAMRMToken() != null) { + LOG.info("RM rolled master-key for amrm-tokens."); - org.apache.hadoop.yarn.api.records.Token token = - allocateResponse.getAMRMToken(); + org.apache.hadoop.yarn.api.records.Token token = allocateResponse.getAMRMToken(); - // Do not propagate this info back to AM - allocateResponse.setAMRMToken(null); + // Do not propagate this info back to AM + allocateResponse.setAMRMToken(null); - org.apache.hadoop.security.token.Token newToken = - ConverterUtils.convertFromYarn(token, (Text) null); + org.apache.hadoop.security.token.Token newToken = + ConverterUtils.convertFromYarn(token, (Text) null); - // Update the AMRMToken in context map, and in NM state store if it is - // different - if (context.setAMRMToken(newToken) - && this.nmContext.getNMStateStore() != null) { - try { + // Update the AMRMToken in context map, and in NM state store if it is + // different + if (context.setAMRMToken(newToken) && this.nmContext.getNMStateStore() != null) { this.nmContext.getNMStateStore().storeAMRMProxyAppContextEntry( context.getApplicationAttemptId(), NMSS_AMRMTOKEN_KEY, - newToken.encodeToUrlString().getBytes("UTF-8")); - } catch (IOException e) { - LOG.error("Error storing AMRMProxy application context entry for " - + context.getApplicationAttemptId(), e); + newToken.encodeToUrlString().getBytes(StandardCharsets.UTF_8)); } } - } - // Check if the local AMRMToken is rolled up and update the context and - // response accordingly - MasterKeyData nextMasterKey = - this.secretManager.getNextMasterKeyData(); - - if (nextMasterKey != null - && nextMasterKey.getMasterKey().getKeyId() != amrmTokenIdentifier - .getKeyId()) { - Token localToken = context.getLocalAMRMToken(); - if (nextMasterKey.getMasterKey().getKeyId() != context - .getLocalAMRMTokenKeyId()) { - LOG.info("The local AMRMToken has been rolled-over." - + " Send new local AMRMToken back to application: " - + pipeline.getApplicationId()); - localToken = - this.secretManager.createAndGetAMRMToken(pipeline - .getApplicationAttemptId()); - context.setLocalAMRMToken(localToken); - } + // Check if the local AMRMToken is rolled up and update the context and + // response accordingly + MasterKeyData nextMasterKey = this.secretManager.getNextMasterKeyData(); + + if (nextMasterKey != null) { + MasterKey masterKey = nextMasterKey.getMasterKey(); + if (masterKey.getKeyId() != amrmTokenIdentifier.getKeyId()) { + Token localToken = context.getLocalAMRMToken(); + if (masterKey.getKeyId() != context.getLocalAMRMTokenKeyId()) { + LOG.info("The local AMRMToken has been rolled-over." + + " Send new local AMRMToken back to application: {}", + pipeline.getApplicationId()); + localToken = this.secretManager.createAndGetAMRMToken( + pipeline.getApplicationAttemptId()); + context.setLocalAMRMToken(localToken); + } - allocateResponse - .setAMRMToken(org.apache.hadoop.yarn.api.records.Token - .newInstance(localToken.getIdentifier(), localToken - .getKind().toString(), localToken.getPassword(), - localToken.getService().toString())); + allocateResponse + .setAMRMToken(org.apache.hadoop.yarn.api.records.Token + .newInstance(localToken.getIdentifier(), localToken + .getKind().toString(), localToken.getPassword(), + localToken.getService().toString())); + } + } + long endTime = clock.getTime(); + this.metrics.succeededUpdateTokenRequests(endTime - startTime); + } catch (IOException e) { + LOG.error("Error storing AMRMProxy application context entry for {}.", + context.getApplicationAttemptId(), e); + this.metrics.incrFailedUpdateAMRMTokenRequests(); } } @@ -672,19 +673,19 @@ private AMRMProxyApplicationContext createApplicationMasterContext( } /** - * Gets the Request intercepter chains for all the applications. + * Gets the Request interceptor chains for all the applications. * - * @return the request intercepter chains. + * @return the request interceptor chains. */ protected Map getPipelines() { return this.applPipelineMap; } /** - * This method creates and returns reference of the first intercepter in the - * chain of request intercepter instances. + * This method creates and returns reference of the first interceptor in the + * chain of request interceptor instances. * - * @return the reference of the first intercepter in the chain + * @return the reference of the first interceptor in the chain */ protected RequestInterceptor createRequestInterceptorChain() { Configuration conf = getConfig(); @@ -717,7 +718,7 @@ protected RequestInterceptor createRequestInterceptorChain() { } catch (ClassNotFoundException e) { throw new YarnRuntimeException( "Could not instantiate ApplicationMasterRequestInterceptor: " - + interceptorClassName, e); + + interceptorClassName, e); } } @@ -729,10 +730,10 @@ protected RequestInterceptor createRequestInterceptorChain() { } /** - * Returns the comma separated intercepter class names from the configuration. + * Returns the comma separated interceptor class names from the configuration. * * @param conf configuration - * @return the intercepter class names as an instance of ArrayList + * @return the interceptor class names as an instance of ArrayList */ private List getInterceptorClassNames(Configuration conf) { String configuredInterceptorClassNames = @@ -759,7 +760,7 @@ private List getInterceptorClassNames(Configuration conf) { * Authorizes the request and returns the application specific request * processing pipeline. * - * @return the the intercepter wrapper instance + * @return the interceptor wrapper instance * @throws YarnException if fails */ private RequestInterceptorChainWrapper authorizeAndGetInterceptorChain() @@ -775,11 +776,10 @@ private RequestInterceptorChainWrapper getInterceptorChain( tokenIdentifier.getApplicationAttemptId(); synchronized (this.applPipelineMap) { - if (!this.applPipelineMap.containsKey(appAttemptId - .getApplicationId())) { + if (!this.applPipelineMap.containsKey(appAttemptId.getApplicationId())) { throw new YarnException( "The AM request processing pipeline is not initialized for app: " - + appAttemptId.getApplicationId().toString()); + + appAttemptId.getApplicationId()); } return this.applPipelineMap.get(appAttemptId.getApplicationId()); @@ -827,29 +827,26 @@ public AMRMProxyTokenSecretManager getSecretManager() { /** * Private class for handling application stop events. - * */ class ApplicationEventHandler implements EventHandler { @Override public void handle(ApplicationEvent event) { Application app = - AMRMProxyService.this.nmContext.getApplications().get( - event.getApplicationID()); + AMRMProxyService.this.nmContext.getApplications().get(event.getApplicationID()); if (app != null) { switch (event.getType()) { case APPLICATION_RESOURCES_CLEANEDUP: - LOG.info("Application stop event received for stopping AppId:" - + event.getApplicationID().toString()); + LOG.info("Application stop event received for stopping AppId: {}.", + event.getApplicationID().toString()); AMRMProxyService.this.stopApplication(event.getApplicationID()); break; default: - LOG.debug("AMRMProxy is ignoring event: {}", event.getType()); + LOG.debug("AMRMProxy is ignoring event: {}.", event.getType()); break; } } else { - LOG.warn("Event " + event + " sent to absent application " - + event.getApplicationID()); + LOG.warn("Event {} sent to absent application {}.", event, event.getApplicationID()); } } } @@ -866,20 +863,20 @@ public static class RequestInterceptorChainWrapper { /** * Initializes the wrapper with the specified parameters. - * - * @param rootInterceptor the root request intercepter - * @param applicationAttemptId attempt id + * + * @param interceptor the root request interceptor + * @param appAttemptId attempt id */ - public synchronized void init(RequestInterceptor rootInterceptor, - ApplicationAttemptId applicationAttemptId) { - this.rootInterceptor = rootInterceptor; - this.applicationAttemptId = applicationAttemptId; + public synchronized void init(RequestInterceptor interceptor, + ApplicationAttemptId appAttemptId) { + rootInterceptor = interceptor; + applicationAttemptId = appAttemptId; } /** - * Gets the root request intercepter. + * Gets the root request interceptor. * - * @return the root request intercepter + * @return the root request interceptor */ public synchronized RequestInterceptor getRootInterceptor() { return rootInterceptor; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/FederationInterceptor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/FederationInterceptor.java index f46a31dc723a7..bf18561096f77 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/FederationInterceptor.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/FederationInterceptor.java @@ -66,6 +66,7 @@ import org.apache.hadoop.yarn.api.records.ResourceRequest; import org.apache.hadoop.yarn.api.records.StrictPreemptionContract; import org.apache.hadoop.yarn.api.records.UpdateContainerRequest; +import org.apache.hadoop.yarn.api.records.FinalApplicationStatus; import org.apache.hadoop.yarn.client.AMRMClientUtils; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.exceptions.ApplicationMasterNotRegisteredException; @@ -85,6 +86,7 @@ import org.apache.hadoop.yarn.server.federation.policies.exceptions.FederationPolicyInitializationException; import org.apache.hadoop.yarn.server.federation.resolver.SubClusterResolver; import org.apache.hadoop.yarn.server.federation.store.records.SubClusterId; +import org.apache.hadoop.yarn.server.federation.store.records.SubClusterInfo; import org.apache.hadoop.yarn.server.federation.utils.FederationRegistryClient; import org.apache.hadoop.yarn.server.federation.utils.FederationStateStoreFacade; import org.apache.hadoop.yarn.server.uam.UnmanagedAMPoolManager; @@ -459,8 +461,13 @@ public void recover(Map recoveredDataMap) { // Get the running containers from home RM, note that we will also get the // AM container itself from here. We don't need it, but no harm to put the // map as well. - UserGroupInformation appSubmitter = UserGroupInformation - .createRemoteUser(getApplicationContext().getUser()); + UserGroupInformation appSubmitter; + if (UserGroupInformation.isSecurityEnabled()) { + appSubmitter = UserGroupInformation.createProxyUser(getApplicationContext().getUser(), + UserGroupInformation.getLoginUser()); + } else { + appSubmitter = UserGroupInformation.createRemoteUser(getApplicationContext().getUser()); + } ApplicationClientProtocol rmClient = createHomeRMProxy(getApplicationContext(), ApplicationClientProtocol.class, appSubmitter); @@ -731,50 +738,26 @@ public FinishApplicationMasterResponse finishApplicationMaster( this.finishAMCalled = true; - // TODO: consider adding batchFinishApplicationMaster in UAMPoolManager boolean failedToUnRegister = false; - ExecutorCompletionService compSvc = - null; // Application master is completing operation. Send the finish // application master request to all the registered sub-cluster resource // managers in parallel, wait for the responses and aggregate the results. - Set subClusterIds = this.uamPool.getAllUAMIds(); - if (subClusterIds.size() > 0) { - final FinishApplicationMasterRequest finishRequest = request; - compSvc = - new ExecutorCompletionService( - this.threadpool); - - LOG.info("Sending finish application request to {} sub-cluster RMs", - subClusterIds.size()); - for (final String subClusterId : subClusterIds) { - compSvc.submit(new Callable() { - @Override - public FinishApplicationMasterResponseInfo call() throws Exception { - LOG.info("Sending finish application request to RM {}", - subClusterId); - FinishApplicationMasterResponse uamResponse = null; - try { - uamResponse = - uamPool.finishApplicationMaster(subClusterId, finishRequest); - - if (uamResponse.getIsUnregistered()) { - secondaryRelayers.remove(subClusterId); - if (getNMStateStore() != null) { - getNMStateStore().removeAMRMProxyAppContextEntry(attemptId, - NMSS_SECONDARY_SC_PREFIX + subClusterId); - } - } - } catch (Throwable e) { - LOG.warn("Failed to finish unmanaged application master: " - + "RM address: " + subClusterId + " ApplicationId: " - + attemptId, e); - } - return new FinishApplicationMasterResponseInfo(uamResponse, - subClusterId); - } - }); + Map responseMap = + this.uamPool.batchFinishApplicationMaster(request, attemptId.toString()); + + for (Map.Entry entry : responseMap.entrySet()) { + String subClusterId = entry.getKey(); + FinishApplicationMasterResponse response = entry.getValue(); + if (response != null && response.getIsUnregistered()) { + secondaryRelayers.remove(subClusterId); + if (getNMStateStore() != null) { + getNMStateStore().removeAMRMProxyAppContextEntry(attemptId, + NMSS_SECONDARY_SC_PREFIX + subClusterId); + } + } else { + // response is null or response.getIsUnregistered() == false + failedToUnRegister = true; } } @@ -787,44 +770,27 @@ public FinishApplicationMasterResponseInfo call() throws Exception { // Stop the home heartbeat thread this.homeHeartbeartHandler.shutdown(); - if (subClusterIds.size() > 0) { - // Wait for other sub-cluster resource managers to return the - // response and merge it with the home response - LOG.info( - "Waiting for finish application response from {} sub-cluster RMs", - subClusterIds.size()); - for (int i = 0; i < subClusterIds.size(); ++i) { - try { - Future future = compSvc.take(); - FinishApplicationMasterResponseInfo uamResponse = future.get(); - LOG.debug("Received finish application response from RM: {}", - uamResponse.getSubClusterId()); - if (uamResponse.getResponse() == null - || !uamResponse.getResponse().getIsUnregistered()) { - failedToUnRegister = true; - } - } catch (Throwable e) { - failedToUnRegister = true; - LOG.warn("Failed to finish unmanaged application master: " - + " ApplicationId: " + this.attemptId, e); - } - } - } - if (failedToUnRegister) { homeResponse.setIsUnregistered(false); - } else { + } else if (checkRequestFinalApplicationStatusSuccess(request)) { // Clean up UAMs only when the app finishes successfully, so that no more // attempt will be launched. this.uamPool.stop(); - if (this.registryClient != null) { - this.registryClient - .removeAppFromRegistry(this.attemptId.getApplicationId()); - } + removeAppFromRegistry(); } return homeResponse; } + private boolean checkRequestFinalApplicationStatusSuccess( + FinishApplicationMasterRequest request) { + if (request != null && request.getFinalApplicationStatus() != null) { + if (request.getFinalApplicationStatus().equals(FinalApplicationStatus.SUCCEEDED)) { + return true; + } + } + return false; + } + @Override public void setNextInterceptor(RequestInterceptor next) { throw new YarnRuntimeException( @@ -860,9 +826,21 @@ public void shutdown() { this.homeHeartbeartHandler.shutdown(); this.homeRMRelayer.shutdown(); + // Shutdown needs to clean up app + removeAppFromRegistry(); + super.shutdown(); } + private void removeAppFromRegistry() { + if (this.registryClient != null && this.attemptId != null) { + ApplicationId applicationId = this.attemptId.getApplicationId(); + if (applicationId != null) { + this.registryClient.removeAppFromRegistry(applicationId); + } + } + } + /** * Only for unit test cleanup. */ @@ -1518,6 +1496,7 @@ protected void mergeAllocateResponse(AllocateResponse homeResponse, private void cacheAllocatedContainers(List containers, SubClusterId subClusterId) { for (Container container : containers) { + SubClusterId chooseSubClusterId = SubClusterId.newInstance(subClusterId.toString()); LOG.debug("Adding container {}", container); if (this.containerIdToSubClusterIdMap.containsKey(container.getId())) { @@ -1540,22 +1519,53 @@ private void cacheAllocatedContainers(List containers, + " from same sub-cluster: {}, so ignoring.", container.getId(), subClusterId); } else { + + LOG.info("Duplicate containerID found in the allocated containers. " + + "try to re-pick the sub-cluster."); + // The same container allocation from different sub-clusters, // something is wrong. - // TODO: YARN-6667 if some subcluster RM is configured wrong, we - // should not fail the entire heartbeat. - throw new YarnRuntimeException( - "Duplicate containerID found in the allocated containers. This" - + " can happen if the RM epoch is not configured properly." - + " ContainerId: " + container.getId().toString() - + " ApplicationId: " + this.attemptId + " From RM: " - + subClusterId - + " . Previous container was from sub-cluster: " - + existingSubClusterId); + try { + + boolean existAllocatedScHealth = isSCHealth(existingSubClusterId); + boolean newAllocatedScHealth = isSCHealth(subClusterId); + + if (existAllocatedScHealth) { + // If the previous RM which allocated Container is normal, + // the previous RM will be used first + LOG.info("Use Previous Allocated Container's subCluster. " + + "ContainerId: {} ApplicationId: {} From RM: {}.", this.attemptId, + container.getId(), existingSubClusterId); + chooseSubClusterId = existingSubClusterId; + } else if (newAllocatedScHealth) { + // If the previous RM which allocated Container is abnormal, + // but the RM of the newly allocated Container is normal, use the new RM + LOG.info("Use Newly Allocated Container's subCluster. " + + "ApplicationId: {} ContainerId: {} From RM: {}.", this.attemptId, + container.getId(), subClusterId); + chooseSubClusterId = subClusterId; + } else { + // There is a very small probability that an exception will be thrown. + // The RM of the previously allocated Container + // and the RM of the newly allocated Container are not normal. + throw new YarnRuntimeException( + " Can't use any subCluster because an exception occurred" + + " ContainerId: " + container.getId() + " ApplicationId: " + this.attemptId + + " From RM: " + subClusterId + ". " + + " Previous Container was From subCluster: " + existingSubClusterId); + } + } catch (Exception ex) { + // An exception occurred + throw new YarnRuntimeException( + " Can't use any subCluster because an exception occurred" + + " ContainerId: " + container.getId() + " ApplicationId: " + this.attemptId + + " From RM: " + subClusterId + ". " + + " Previous Container was From subCluster: " + existingSubClusterId, ex); + } } } - this.containerIdToSubClusterIdMap.put(container.getId(), subClusterId); + this.containerIdToSubClusterIdMap.put(container.getId(), chooseSubClusterId); } } @@ -1804,4 +1814,25 @@ public static boolean isNullOrEmpty(Collection c) { public static boolean isNullOrEmpty(Map c) { return (c == null || c.size() == 0); } + + @VisibleForTesting + protected void cacheAllocatedContainersForSubClusterId( + List containers, SubClusterId subClusterId) { + cacheAllocatedContainers(containers, subClusterId); + } + + @VisibleForTesting + protected Map getContainerIdToSubClusterIdMap() { + return containerIdToSubClusterIdMap; + } + + private boolean isSCHealth(SubClusterId subClusterId) throws YarnException { + Set timeOutScs = getTimedOutSCs(true); + SubClusterInfo subClusterInfo = federationFacade.getSubCluster(subClusterId); + if (timeOutScs.contains(subClusterId) || + subClusterInfo == null || subClusterInfo.getState().isUnusable()) { + return false; + } + return true; + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/package-info.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/package-info.java new file mode 100644 index 0000000000000..2f20d98426fef --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/package-info.java @@ -0,0 +1,28 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * Package org.apache.hadoop.yarn.server.nodemanager.amrmproxy contains + * classes for handling federation amrm information. + */ +@InterfaceAudience.Private +@InterfaceStability.Unstable +package org.apache.hadoop.yarn.server.nodemanager.amrmproxy; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/numa/NumaResourceAllocator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/numa/NumaResourceAllocator.java index 6bfa0a931d4bf..2deaf16880a0e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/numa/NumaResourceAllocator.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/numa/NumaResourceAllocator.java @@ -143,7 +143,7 @@ public void init(Configuration conf) throws YarnException { } @VisibleForTesting - String executeNGetCmdOutput(Configuration conf) throws YarnException { + public String executeNGetCmdOutput(Configuration conf) throws YarnException { String numaCtlCmd = conf.get( YarnConfiguration.NM_NUMA_AWARENESS_NUMACTL_CMD, YarnConfiguration.DEFAULT_NM_NUMA_AWARENESS_NUMACTL_CMD); @@ -231,7 +231,7 @@ public synchronized NumaResourceAllocation allocateNumaNodes( } private NumaResourceAllocation allocate(ContainerId containerId, - Resource resource) { + Resource resource) throws ResourceHandlerException { for (int index = 0; index < numaNodesList.size(); index++) { NumaNodeResource numaNode = numaNodesList .get((currentAssignNode + index) % numaNodesList.size()); @@ -306,12 +306,20 @@ private NumaResourceAllocation allocate(ContainerId containerId, * Release assigned NUMA resources for the container. * * @param containerId the container ID + * @throws ResourceHandlerException when failed to release numa resource */ - public synchronized void releaseNumaResource(ContainerId containerId) { + public synchronized void releaseNumaResource(ContainerId containerId) + throws ResourceHandlerException { LOG.info("Releasing the assigned NUMA resources for " + containerId); for (NumaNodeResource numaNode : numaNodesList) { numaNode.releaseResources(containerId); } + // delete from NM State store + try { + context.getNMStateStore().releaseAssignedResources(containerId, NUMA_RESOURCE_TYPE); + } catch (IOException e){ + throw new ResourceHandlerException(e); + } } /** diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/AppLogAggregatorImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/AppLogAggregatorImpl.java index 96e08c5ff38ee..1ba7353a1eef3 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/AppLogAggregatorImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/AppLogAggregatorImpl.java @@ -86,6 +86,7 @@ public class AppLogAggregatorImpl implements AppLogAggregator { private final Dispatcher dispatcher; private final ApplicationId appId; private final String applicationId; + private final boolean enableLocalCleanup; private boolean logAggregationDisabled = false; private final Configuration conf; private final DeletionService delService; @@ -172,6 +173,13 @@ public AppLogAggregatorImpl(Dispatcher dispatcher, this.logAggregationContext = logAggregationContext; this.context = context; this.nodeId = nodeId; + this.enableLocalCleanup = + conf.getBoolean(YarnConfiguration.LOG_AGGREGATION_ENABLE_LOCAL_CLEANUP, + YarnConfiguration.DEFAULT_LOG_AGGREGATION_ENABLE_LOCAL_CLEANUP); + if (!this.enableLocalCleanup) { + LOG.warn("{} is only for testing and not for any production system ", + YarnConfiguration.LOG_AGGREGATION_ENABLE_LOCAL_CLEANUP); + } this.logAggPolicy = getLogAggPolicy(conf); this.recoveredLogInitedTime = recoveredLogInitedTime; this.logFileSizeThreshold = @@ -337,26 +345,26 @@ private void uploadLogsForContainers(boolean appFinished) appFinished, finishedContainers.contains(container)); if (uploadedFilePathsInThisCycle.size() > 0) { uploadedLogsInThisCycle = true; - LOG.trace("Uploaded the following files for {}: {}", - container, uploadedFilePathsInThisCycle.toString()); - List uploadedFilePathsInThisCycleList = new ArrayList<>(); - uploadedFilePathsInThisCycleList.addAll(uploadedFilePathsInThisCycle); - if (LOG.isDebugEnabled()) { - for (Path uploadedFilePath : uploadedFilePathsInThisCycleList) { - try { - long fileSize = lfs.getFileStatus(uploadedFilePath).getLen(); - if (fileSize >= logFileSizeThreshold) { - LOG.debug("Log File " + uploadedFilePath - + " size is " + fileSize + " bytes"); + if (enableLocalCleanup) { + LOG.trace("Uploaded the following files for {}: {}", container, + uploadedFilePathsInThisCycle.toString()); + List uploadedFilePathsInThisCycleList = new ArrayList<>(); + uploadedFilePathsInThisCycleList.addAll(uploadedFilePathsInThisCycle); + if (LOG.isDebugEnabled()) { + for (Path uploadedFilePath : uploadedFilePathsInThisCycleList) { + try { + long fileSize = lfs.getFileStatus(uploadedFilePath).getLen(); + if (fileSize >= logFileSizeThreshold) { + LOG.debug("Log File " + uploadedFilePath + " size is " + fileSize + " bytes"); + } + } catch (Exception e1) { + LOG.error("Failed to get log file size " + e1); } - } catch (Exception e1) { - LOG.error("Failed to get log file size " + e1); } } + deletionTask = new FileDeletionTask(delService, this.userUgi.getShortUserName(), null, + uploadedFilePathsInThisCycleList); } - deletionTask = new FileDeletionTask(delService, - this.userUgi.getShortUserName(), null, - uploadedFilePathsInThisCycleList); } // This container is finished, and all its logs have been uploaded, @@ -528,6 +536,9 @@ private void doAppLogAggregation() throws LogAggregationDFSException { } private void doAppLogAggregationPostCleanUp() { + if (!enableLocalCleanup) { + return; + } // Remove the local app-log-dirs List localAppLogDirs = new ArrayList(); for (String rootLogDir : dirsHandler.getLogDirsForCleanup()) { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/recovery/NMLeveldbStateStoreService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/recovery/NMLeveldbStateStoreService.java index cbdb424fc824b..81cfb2e743b7a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/recovery/NMLeveldbStateStoreService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/recovery/NMLeveldbStateStoreService.java @@ -1813,4 +1813,21 @@ protected void checkVersion() throws IOException { + getCurrentVersion() + ", but loading version " + loadedVersion); } } + @Override + public void releaseAssignedResources(ContainerId containerId, String resourceType) + throws IOException { + LOG.debug("releaseAssignedResources: containerId=" + containerId + " resourceType=" + + resourceType); + try { + try (WriteBatch batch = db.createWriteBatch()) { + String key = CONTAINERS_KEY_PREFIX + containerId + + CONTAINER_ASSIGNED_RESOURCES_KEY_SUFFIX + resourceType; + batch.delete(bytes(key)); + db.write(batch); + } + }catch (DBException e){ + markStoreUnHealthy(e); + throw new IOException(e); + } + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/recovery/NMStateStoreService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/recovery/NMStateStoreService.java index c6d8a1368aa33..aa56c6f247475 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/recovery/NMStateStoreService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/recovery/NMStateStoreService.java @@ -786,6 +786,15 @@ public abstract void storeAssignedResources(Container container, String resourceType, List assignedResources) throws IOException; + /** + * Delete the assigned resources of a container of specific resourceType. + * @param containerId Container Id + * @param resourceType resource Type + * @throws IOException while releasing resources + */ + public void releaseAssignedResources(ContainerId containerId, String resourceType) + throws IOException {} + protected abstract void initStorage(Configuration conf) throws IOException; protected abstract void startStorage() throws IOException; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestDefaultContainerExecutor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestDefaultContainerExecutor.java index a5c115283a021..473292f35f749 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestDefaultContainerExecutor.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestDefaultContainerExecutor.java @@ -20,8 +20,8 @@ import static org.apache.hadoop.fs.CreateFlag.CREATE; import static org.apache.hadoop.fs.CreateFlag.OVERWRITE; -import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; +import static org.junit.jupiter.api.Assertions.assertEquals; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyBoolean; import static org.mockito.ArgumentMatchers.isA; @@ -39,10 +39,12 @@ import java.io.FileWriter; import java.io.IOException; import java.util.ArrayList; +import java.util.Arrays; import java.util.EnumSet; import java.util.HashMap; import java.util.List; import java.util.Random; +import java.util.concurrent.ConcurrentHashMap; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.CommonConfigurationKeys; @@ -60,23 +62,32 @@ import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.ContainerLaunchContext; +import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.exceptions.ConfigurationException; import org.apache.hadoop.yarn.exceptions.YarnException; +import org.apache.hadoop.yarn.exceptions.YarnRuntimeException; import org.apache.hadoop.yarn.server.nodemanager.api.LocalizationProtocol; import org.apache.hadoop.yarn.server.nodemanager.api.ResourceLocalizationSpec; import org.apache.hadoop.yarn.server.nodemanager.api.protocolrecords.LocalizerAction; import org.apache.hadoop.yarn.server.nodemanager.api.protocolrecords.LocalizerStatus; import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container; import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerDiagnosticsUpdateEvent; +import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ResourceMappings; +import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ResourceMappings.AssignedResources; import org.apache.hadoop.yarn.server.nodemanager.containermanager.launcher.ContainerLaunch; +import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources.numa.NumaResourceAllocation; +import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources.numa.NumaResourceAllocator; import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.ContainerLocalizer; import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.MockLocalizerHeartbeatResponse; +import org.apache.hadoop.yarn.server.nodemanager.executor.ContainerReacquisitionContext; import org.apache.hadoop.yarn.server.nodemanager.executor.ContainerStartContext; import org.apache.hadoop.yarn.server.nodemanager.executor.DeletionAsUserContext; import org.apache.hadoop.yarn.server.nodemanager.executor.LocalizerStartContext; +import org.apache.hadoop.yarn.server.nodemanager.recovery.NMStateStoreService; import org.junit.AfterClass; import org.junit.Assert; +import org.junit.Before; import org.junit.Test; import org.mockito.invocation.InvocationOnMock; import org.mockito.stubbing.Answer; @@ -86,6 +97,14 @@ public class TestDefaultContainerExecutor { private static Path BASE_TMP_PATH = new Path("target", TestDefaultContainerExecutor.class.getSimpleName()); + private YarnConfiguration yarnConfiguration; + + private DefaultContainerExecutor containerExecutor; + + private Container mockContainer; + + private NumaResourceAllocator numaResourceAllocator; + @AfterClass public static void deleteTmpFiles() throws IOException { FileContext lfs = FileContext.getLocalFSFileContext(); @@ -736,4 +755,204 @@ public void testPickDirectory() throws Exception { // new FsPermission(ApplicationLocalizer.LOGDIR_PERM), true); // } + @Before + public void setUp() throws IOException, YarnException { + yarnConfiguration = new YarnConfiguration(); + setNumaConfig(); + Context mockContext = createAndGetMockContext(); + NMStateStoreService nmStateStoreService = + mock(NMStateStoreService.class); + when(mockContext.getNMStateStore()).thenReturn(nmStateStoreService); + numaResourceAllocator = new NumaResourceAllocator(mockContext) { + @Override + public String executeNGetCmdOutput(Configuration config) + throws YarnRuntimeException { + return getNumaCmdOutput(); + } + }; + + numaResourceAllocator.init(yarnConfiguration); + FileContext lfs = FileContext.getLocalFSFileContext(); + containerExecutor = new DefaultContainerExecutor(lfs) { + @Override + public Configuration getConf() { + return yarnConfiguration; + } + }; + containerExecutor.setNumaResourceAllocator(numaResourceAllocator); + mockContainer = mock(Container.class); + } + + private void setNumaConfig() { + yarnConfiguration.set(YarnConfiguration.NM_NUMA_AWARENESS_ENABLED, "true"); + yarnConfiguration.set(YarnConfiguration.NM_NUMA_AWARENESS_READ_TOPOLOGY, "true"); + yarnConfiguration.set(YarnConfiguration.NM_NUMA_AWARENESS_NUMACTL_CMD, "/usr/bin/numactl"); + } + + + private String getNumaCmdOutput() { + // architecture of 8 cpu cores + // randomly picked size of memory + return "available: 2 nodes (0-1)\n\t" + + "node 0 cpus: 0 2 4 6\n\t" + + "node 0 size: 73717 MB\n\t" + + "node 0 free: 73717 MB\n\t" + + "node 1 cpus: 1 3 5 7\n\t" + + "node 1 size: 73717 MB\n\t" + + "node 1 free: 73717 MB\n\t" + + "node distances:\n\t" + + "node 0 1\n\t" + + "0: 10 20\n\t" + + "1: 20 10"; + } + + private Context createAndGetMockContext() { + Context mockContext = mock(Context.class); + @SuppressWarnings("unchecked") + ConcurrentHashMap mockContainers = mock( + ConcurrentHashMap.class); + mockContainer = mock(Container.class); + when(mockContainer.getResourceMappings()) + .thenReturn(new ResourceMappings()); + when(mockContainers.get(any())).thenReturn(mockContainer); + when(mockContext.getContainers()).thenReturn(mockContainers); + when(mockContainer.getResource()).thenReturn(Resource.newInstance(2048, 2)); + return mockContext; + } + + private void testAllocateNumaResource(String containerId, Resource resource, + String memNodes, String cpuNodes) throws Exception { + when(mockContainer.getContainerId()) + .thenReturn(ContainerId.fromString(containerId)); + when(mockContainer.getResource()).thenReturn(resource); + NumaResourceAllocation numaResourceAllocation = + numaResourceAllocator.allocateNumaNodes(mockContainer); + containerExecutor.setNumactl(containerExecutor.getConf().get(YarnConfiguration.NM_NUMA_AWARENESS_NUMACTL_CMD, + YarnConfiguration.DEFAULT_NM_NUMA_AWARENESS_NUMACTL_CMD)); + String[] commands = containerExecutor.getNumaCommands(numaResourceAllocation); + assertEquals(Arrays.asList(commands), Arrays.asList("/usr/bin/numactl", + "--interleave=" + memNodes, "--cpunodebind=" + cpuNodes)); + } + + @Test + public void testAllocateNumaMemoryResource() throws Exception { + // keeping cores constant for testing memory resources + + // allocates node 0 for memory and cpu + testAllocateNumaResource("container_1481156246874_0001_01_000001", + Resource.newInstance(2048, 2), "0", "0"); + + // allocates node 1 for memory and cpu since allocator uses round robin assignment + testAllocateNumaResource("container_1481156246874_0001_01_000002", + Resource.newInstance(60000, 2), "1", "1"); + + // allocates node 0,1 for memory since there is no sufficient memory in any one node + testAllocateNumaResource("container_1481156246874_0001_01_000003", + Resource.newInstance(80000, 2), "0,1", "0"); + + // returns null since there are no sufficient resources available for the request + when(mockContainer.getContainerId()).thenReturn( + ContainerId.fromString("container_1481156246874_0001_01_000004")); + when(mockContainer.getResource()) + .thenReturn(Resource.newInstance(80000, 2)); + Assert.assertNull(numaResourceAllocator.allocateNumaNodes(mockContainer)); + + // allocates node 1 for memory and cpu + testAllocateNumaResource("container_1481156246874_0001_01_000005", + Resource.newInstance(1024, 2), "1", "1"); + } + + @Test + public void testAllocateNumaCpusResource() throws Exception { + // keeping memory constant + + // allocates node 0 for memory and cpu + testAllocateNumaResource("container_1481156246874_0001_01_000001", + Resource.newInstance(2048, 2), "0", "0"); + + // allocates node 1 for memory and cpu since allocator uses round robin assignment + testAllocateNumaResource("container_1481156246874_0001_01_000002", + Resource.newInstance(2048, 2), "1", "1"); + + // allocates node 0,1 for cpus since there is are no sufficient cpus available in any one node + testAllocateNumaResource("container_1481156246874_0001_01_000003", + Resource.newInstance(2048, 3), "0", "0,1"); + + // returns null since there are no sufficient resources available for the request + when(mockContainer.getContainerId()).thenReturn( + ContainerId.fromString("container_1481156246874_0001_01_000004")); + when(mockContainer.getResource()).thenReturn(Resource.newInstance(2048, 2)); + Assert.assertNull(numaResourceAllocator.allocateNumaNodes(mockContainer)); + + // allocates node 1 for memory and cpu + testAllocateNumaResource("container_1481156246874_0001_01_000005", + Resource.newInstance(2048, 1), "1", "1"); + } + + @Test + public void testReacquireContainer() throws Exception { + @SuppressWarnings("unchecked") + ConcurrentHashMap mockContainers = mock( + ConcurrentHashMap.class); + Context mockContext = mock(Context.class); + NMStateStoreService mock = mock(NMStateStoreService.class); + when(mockContext.getNMStateStore()).thenReturn(mock); + ResourceMappings resourceMappings = new ResourceMappings(); + AssignedResources assignedRscs = new AssignedResources(); + when(mockContainer.getResource()) + .thenReturn(Resource.newInstance(147434, 2)); + ContainerId cid = ContainerId.fromString("container_1481156246874_0001_01_000001"); + when(mockContainer.getContainerId()).thenReturn(cid); + NumaResourceAllocation numaResourceAllocation = + numaResourceAllocator.allocateNumaNodes(mockContainer); + assignedRscs.updateAssignedResources(Arrays.asList(numaResourceAllocation)); + resourceMappings.addAssignedResources("numa", assignedRscs); + when(mockContainer.getResourceMappings()).thenReturn(resourceMappings); + when(mockContainers.get(any())).thenReturn(mockContainer); + when(mockContext.getContainers()).thenReturn(mockContainers); + + // recovered numa resources should be added to the used resources and + // remaining will be available for further allocation. + + ContainerReacquisitionContext containerReacquisitionContext = + new ContainerReacquisitionContext.Builder() + .setContainerId(cid) + .setUser("user") + .setContainer(mockContainer) + .build(); + + containerExecutor.reacquireContainer(containerReacquisitionContext); + + // reacquireContainer recovers all the numa resources , + // that should be free to use next + testAllocateNumaResource("container_1481156246874_0001_01_000001", + Resource.newInstance(147434, 2), "0,1", "1"); + when(mockContainer.getContainerId()).thenReturn( + ContainerId.fromString("container_1481156246874_0001_01_000004")); + when(mockContainer.getResource()) + .thenReturn(Resource.newInstance(1024, 2)); + + // returns null since there are no sufficient resources available for the request + Assert.assertNull(numaResourceAllocator.allocateNumaNodes(mockContainer)); + } + + @Test + public void testConcatStringCommands() { + // test one array of string as null + assertEquals(containerExecutor.concatStringCommands(null, new String[]{"hello"})[0], + new String[]{"hello"}[0]); + // test both array of string as null + Assert.assertNull(containerExecutor.concatStringCommands(null, null)); + // test case when both arrays are not null and of equal length + String[] res = containerExecutor.concatStringCommands(new String[]{"one"}, + new String[]{"two"}); + assertEquals(res[0]+res[1], "one" + "two"); + // test both array of different length + res = containerExecutor.concatStringCommands(new String[]{"one"}, + new String[]{"two", "three"}); + assertEquals(res[0] + res[1] + res[2], "one" + "two" + "three"); + + } + + } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeStatusUpdater.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeStatusUpdater.java index b3c401453fd24..511013cad19ae 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeStatusUpdater.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeStatusUpdater.java @@ -181,7 +181,7 @@ public RegisterNodeManagerResponse registerNodeManager( IOException { NodeId nodeId = request.getNodeId(); Resource resource = request.getResource(); - LOG.info("Registering " + nodeId.toString()); + LOG.info("Registering {}.", nodeId.toString()); // NOTE: this really should be checking against the config value InetSocketAddress expected = NetUtils.getConnectAddress( conf.getSocketAddr(YarnConfiguration.NM_ADDRESS, null, -1)); @@ -217,7 +217,7 @@ private Map> getAppToContainerStatusMap( public NodeHeartbeatResponse nodeHeartbeat(NodeHeartbeatRequest request) throws YarnException, IOException { NodeStatus nodeStatus = request.getNodeStatus(); - LOG.info("Got heartbeat number " + heartBeatID); + LOG.info("Got heartbeat number {}.", heartBeatID); NodeManagerMetrics mockMetrics = mock(NodeManagerMetrics.class); Dispatcher mockDispatcher = mock(Dispatcher.class); @SuppressWarnings("unchecked") @@ -625,7 +625,7 @@ public RegisterNodeManagerResponse registerNodeManager( @Override public NodeHeartbeatResponse nodeHeartbeat(NodeHeartbeatRequest request) throws YarnException, IOException { - LOG.info("Got heartBeatId: [" + heartBeatID +"]"); + LOG.info("Got heartBeatId: [{}]", heartBeatID); NodeStatus nodeStatus = request.getNodeStatus(); nodeStatus.setResponseId(heartBeatID.getAndIncrement()); NodeHeartbeatResponse nhResponse = YarnServerBuilderUtils. @@ -644,7 +644,7 @@ public NodeHeartbeatResponse nodeHeartbeat(NodeHeartbeatRequest request) } } if (heartBeatID.get() == 2) { - LOG.info("Sending FINISH_APP for application: [" + appId + "]"); + LOG.info("Sending FINISH_APP for application: [{}]", appId); this.context.getApplications().put(appId, mock(Application.class)); nhResponse.addAllApplicationsToCleanup(Collections.singletonList(appId)); } @@ -1528,7 +1528,7 @@ public void testApplicationKeepAlive() throws Exception { rt.context.getApplications().remove(rt.appId); Assert.assertEquals(1, rt.keepAliveRequests.size()); int numKeepAliveRequests = rt.keepAliveRequests.get(rt.appId).size(); - LOG.info("Number of Keep Alive Requests: [" + numKeepAliveRequests + "]"); + LOG.info("Number of Keep Alive Requests: [{}]", numKeepAliveRequests); Assert.assertTrue(numKeepAliveRequests == 2 || numKeepAliveRequests == 3); GenericTestUtils.waitFor( () -> nm.getServiceState() != STATE.STARTED diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/TestAMRMProxyMetrics.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/TestAMRMProxyMetrics.java index 4621c4dce32ca..6219641c6eefd 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/TestAMRMProxyMetrics.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/TestAMRMProxyMetrics.java @@ -42,11 +42,19 @@ public static void init() { Assert.assertEquals(0, metrics.getFailedRegisterAMRequests()); Assert.assertEquals(0, metrics.getFailedFinishAMRequests()); Assert.assertEquals(0, metrics.getFailedAllocateRequests()); + Assert.assertEquals(0, metrics.getFailedAppRecoveryCount()); + Assert.assertEquals(0, metrics.getFailedAppStopRequests()); + Assert.assertEquals(0, metrics.getFailedUpdateAMRMTokenRequests()); + Assert.assertEquals(0, metrics.getAllocateCount()); + Assert.assertEquals(0, metrics.getRequestCount()); Assert.assertEquals(0, metrics.getNumSucceededAppStartRequests()); Assert.assertEquals(0, metrics.getNumSucceededRegisterAMRequests()); Assert.assertEquals(0, metrics.getNumSucceededFinishAMRequests()); Assert.assertEquals(0, metrics.getNumSucceededAllocateRequests()); + Assert.assertEquals(0, metrics.getNumSucceededRecoverRequests()); + Assert.assertEquals(0, metrics.getNumSucceededAppStopRequests()); + Assert.assertEquals(0, metrics.getNumSucceededUpdateAMRMTokenRequests()); LOG.info("Test: aggregate metrics are updated correctly"); } @@ -57,19 +65,19 @@ public void testAllocateRequestWithNullValues() throws Exception { long failedRegisterAMRequests = metrics.getFailedRegisterAMRequests(); long failedFinishAMRequests = metrics.getFailedFinishAMRequests(); long failedAllocateRequests = metrics.getFailedAllocateRequests(); + long failedAppRecoveryRequests = metrics.getFailedAppRecoveryCount(); + long failedAppStopRequests = metrics.getFailedAppStopRequests(); + long failedUpdateAMRMTokenRequests = metrics.getFailedUpdateAMRMTokenRequests(); long succeededAppStartRequests = metrics.getNumSucceededAppStartRequests(); - long succeededRegisterAMRequests = - metrics.getNumSucceededRegisterAMRequests(); + long succeededRegisterAMRequests = metrics.getNumSucceededRegisterAMRequests(); long succeededFinishAMRequests = metrics.getNumSucceededFinishAMRequests(); long succeededAllocateRequests = metrics.getNumSucceededAllocateRequests(); int testAppId = 1; - RegisterApplicationMasterResponse registerResponse = - registerApplicationMaster(testAppId); + RegisterApplicationMasterResponse registerResponse = registerApplicationMaster(testAppId); Assert.assertNotNull(registerResponse); - Assert - .assertEquals(Integer.toString(testAppId), registerResponse.getQueue()); + Assert.assertEquals(Integer.toString(testAppId), registerResponse.getQueue()); AllocateResponse allocateResponse = allocate(testAppId); Assert.assertNotNull(allocateResponse); @@ -80,14 +88,13 @@ public void testAllocateRequestWithNullValues() throws Exception { Assert.assertNotNull(finshResponse); Assert.assertEquals(true, finshResponse.getIsUnregistered()); - Assert.assertEquals(failedAppStartRequests, - metrics.getFailedAppStartRequests()); - Assert.assertEquals(failedRegisterAMRequests, - metrics.getFailedRegisterAMRequests()); - Assert.assertEquals(failedFinishAMRequests, - metrics.getFailedFinishAMRequests()); - Assert.assertEquals(failedAllocateRequests, - metrics.getFailedAllocateRequests()); + Assert.assertEquals(failedAppStartRequests, metrics.getFailedAppStartRequests()); + Assert.assertEquals(failedRegisterAMRequests, metrics.getFailedRegisterAMRequests()); + Assert.assertEquals(failedFinishAMRequests, metrics.getFailedFinishAMRequests()); + Assert.assertEquals(failedAllocateRequests, metrics.getFailedAllocateRequests()); + Assert.assertEquals(failedAppRecoveryRequests, metrics.getFailedAppRecoveryCount()); + Assert.assertEquals(failedAppStopRequests, metrics.getFailedAppStopRequests()); + Assert.assertEquals(failedUpdateAMRMTokenRequests, metrics.getFailedUpdateAMRMTokenRequests()); Assert.assertEquals(succeededAppStartRequests, metrics.getNumSucceededAppStartRequests()); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/TestFederationInterceptor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/TestFederationInterceptor.java index 3288382b26de0..f0659a9d46ef3 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/TestFederationInterceptor.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/TestFederationInterceptor.java @@ -36,6 +36,7 @@ import org.apache.hadoop.registry.client.api.RegistryOperations; import org.apache.hadoop.registry.client.impl.FSRegistryOperationsService; import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.test.LambdaTestUtils; import org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest; import org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse; import org.apache.hadoop.yarn.api.protocolrecords.FinishApplicationMasterRequest; @@ -57,10 +58,12 @@ import org.apache.hadoop.yarn.api.records.Token; import org.apache.hadoop.yarn.api.records.UpdateContainerError; import org.apache.hadoop.yarn.api.records.UpdatedContainer; +import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.exceptions.ApplicationMasterNotRegisteredException; import org.apache.hadoop.yarn.exceptions.InvalidApplicationMasterRequestException; import org.apache.hadoop.yarn.exceptions.YarnException; +import org.apache.hadoop.yarn.exceptions.YarnRuntimeException; import org.apache.hadoop.yarn.server.MockResourceManagerFacade; import org.apache.hadoop.yarn.server.federation.policies.manager.UniformBroadcastPolicyManager; import org.apache.hadoop.yarn.server.federation.store.impl.MemoryFederationStateStore; @@ -69,6 +72,7 @@ import org.apache.hadoop.yarn.server.federation.store.records.SubClusterInfo; import org.apache.hadoop.yarn.server.federation.store.records.SubClusterRegisterRequest; import org.apache.hadoop.yarn.server.federation.store.records.SubClusterState; +import org.apache.hadoop.yarn.server.federation.utils.FederationRegistryClient; import org.apache.hadoop.yarn.server.federation.utils.FederationStateStoreFacade; import org.apache.hadoop.yarn.server.nodemanager.Context; import org.apache.hadoop.yarn.server.nodemanager.NodeManager.NMContext; @@ -969,4 +973,273 @@ private PreemptionMessage createDummyPreemptionMessage( preemptionMessage.setContract(contract); return preemptionMessage; } + + @Test + public void testSameContainerFromDiffRM() throws IOException, InterruptedException { + + UserGroupInformation ugi = + interceptor.getUGIWithToken(interceptor.getAttemptId()); + + ugi.doAs((PrivilegedExceptionAction) () -> { + + // Register the application + RegisterApplicationMasterRequest registerReq = + Records.newRecord(RegisterApplicationMasterRequest.class); + registerReq.setHost(Integer.toString(testAppId)); + registerReq.setRpcPort(0); + registerReq.setTrackingUrl(""); + + RegisterApplicationMasterResponse registerResponse = + interceptor.registerApplicationMaster(registerReq); + Assert.assertNotNull(registerResponse); + lastResponseId = 0; + + Assert.assertEquals(0, interceptor.getUnmanagedAMPoolSize()); + + // Allocate the first batch of containers, with sc1 active + SubClusterId subClusterId1 = SubClusterId.newInstance("SC-1"); + registerSubCluster(subClusterId1); + + int numberOfContainers = 3; + List containers = + getContainersAndAssert(numberOfContainers, numberOfContainers); + Assert.assertNotNull(containers); + Assert.assertEquals(3, containers.size()); + + // with sc2 active + SubClusterId subClusterId2 = SubClusterId.newInstance("SC-2"); + registerSubCluster(subClusterId2); + + // 1.Container has been registered to SubCluster1, try to register the same Container + // to SubCluster2. + // Because SubCluster1 is in normal state at this time, + // So the SubCluster corresponding to Container should be SubCluster1 + interceptor.cacheAllocatedContainersForSubClusterId(containers, subClusterId2); + Map cIdToSCMap = interceptor.getContainerIdToSubClusterIdMap(); + for (SubClusterId subClusterId : cIdToSCMap.values()) { + Assert.assertNotNull(subClusterId); + Assert.assertEquals(subClusterId1, subClusterId); + } + + // 2.Deregister SubCluster1, Register the same Containers to SubCluster2 + // So the SubCluster corresponding to Container should be SubCluster2 + deRegisterSubCluster(subClusterId1); + interceptor.cacheAllocatedContainersForSubClusterId(containers, subClusterId2); + Map cIdToSCMap2 = interceptor.getContainerIdToSubClusterIdMap(); + for (SubClusterId subClusterId : cIdToSCMap2.values()) { + Assert.assertNotNull(subClusterId); + Assert.assertEquals(subClusterId2, subClusterId); + } + + // 3.Deregister subClusterId2, Register the same Containers to SubCluster1 + // Because both SubCluster1 and SubCluster2 are abnormal at this time, + // an exception will be thrown when registering the first Container. + deRegisterSubCluster(subClusterId2); + Container container1 = containers.get(0); + Assert.assertNotNull(container1); + String errMsg = + " Can't use any subCluster because an exception occurred" + + " ContainerId: " + container1.getId() + + " ApplicationId: " + interceptor.getAttemptId() + + " From RM: " + subClusterId1 + ". " + + " Previous Container was From subCluster: " + subClusterId2; + + LambdaTestUtils.intercept(YarnRuntimeException.class, errMsg, + () -> interceptor.cacheAllocatedContainersForSubClusterId(containers, subClusterId1)); + + // 4. register SubCluster1, re-register the Container, + // and try to finish application + registerSubCluster(subClusterId1); + interceptor.cacheAllocatedContainersForSubClusterId(containers, subClusterId1); + releaseContainersAndAssert(containers); + + // Finish the application + FinishApplicationMasterRequest finishReq = + Records.newRecord(FinishApplicationMasterRequest.class); + finishReq.setDiagnostics(""); + finishReq.setTrackingUrl(""); + finishReq.setFinalApplicationStatus(FinalApplicationStatus.SUCCEEDED); + + FinishApplicationMasterResponse finishResponse = + interceptor.finishApplicationMaster(finishReq); + Assert.assertNotNull(finishResponse); + Assert.assertTrue(finishResponse.getIsUnregistered()); + + return null; + }); + } + + @Test + public void testBatchFinishApplicationMaster() throws IOException, InterruptedException { + + final RegisterApplicationMasterRequest registerReq = + Records.newRecord(RegisterApplicationMasterRequest.class); + registerReq.setHost(Integer.toString(testAppId)); + registerReq.setRpcPort(testAppId); + registerReq.setTrackingUrl(""); + + UserGroupInformation ugi = interceptor.getUGIWithToken(interceptor.getAttemptId()); + + ugi.doAs((PrivilegedExceptionAction) () -> { + + // Register the application + RegisterApplicationMasterRequest registerReq1 = + Records.newRecord(RegisterApplicationMasterRequest.class); + registerReq1.setHost(Integer.toString(testAppId)); + registerReq1.setRpcPort(0); + registerReq1.setTrackingUrl(""); + + // Register ApplicationMaster + RegisterApplicationMasterResponse registerResponse = + interceptor.registerApplicationMaster(registerReq1); + Assert.assertNotNull(registerResponse); + lastResponseId = 0; + + Assert.assertEquals(0, interceptor.getUnmanagedAMPoolSize()); + + // Allocate the first batch of containers, with sc1 and sc2 active + registerSubCluster(SubClusterId.newInstance("SC-1")); + registerSubCluster(SubClusterId.newInstance("SC-2")); + + int numberOfContainers = 3; + List containers = + getContainersAndAssert(numberOfContainers, numberOfContainers * 2); + Assert.assertEquals(2, interceptor.getUnmanagedAMPoolSize()); + Assert.assertEquals(numberOfContainers * 2, containers.size()); + + // Finish the application + FinishApplicationMasterRequest finishReq = + Records.newRecord(FinishApplicationMasterRequest.class); + finishReq.setDiagnostics(""); + finishReq.setTrackingUrl(""); + finishReq.setFinalApplicationStatus(FinalApplicationStatus.SUCCEEDED); + + FinishApplicationMasterResponse finishResp = interceptor.finishApplicationMaster(finishReq); + Assert.assertNotNull(finishResp); + Assert.assertTrue(finishResp.getIsUnregistered()); + + return null; + }); + } + + @Test + public void testRemoveAppFromRegistryApplicationSuccess() + throws IOException, InterruptedException { + + final RegisterApplicationMasterRequest registerReq = + Records.newRecord(RegisterApplicationMasterRequest.class); + registerReq.setHost(Integer.toString(testAppId)); + registerReq.setRpcPort(testAppId); + registerReq.setTrackingUrl(""); + + UserGroupInformation ugi = interceptor.getUGIWithToken(interceptor.getAttemptId()); + + ugi.doAs((PrivilegedExceptionAction) () -> { + + // Register the application + RegisterApplicationMasterRequest registerReq1 = + Records.newRecord(RegisterApplicationMasterRequest.class); + registerReq1.setHost(Integer.toString(testAppId)); + registerReq1.setRpcPort(0); + registerReq1.setTrackingUrl(""); + + // Register ApplicationMaster + RegisterApplicationMasterResponse registerResponse = + interceptor.registerApplicationMaster(registerReq1); + Assert.assertNotNull(registerResponse); + lastResponseId = 0; + + Assert.assertEquals(0, interceptor.getUnmanagedAMPoolSize()); + + // Allocate the first batch of containers, with sc1 active + registerSubCluster(SubClusterId.newInstance("SC-1")); + + int numberOfContainers = 3; + List containers = + getContainersAndAssert(numberOfContainers, numberOfContainers); + Assert.assertEquals(1, interceptor.getUnmanagedAMPoolSize()); + Assert.assertEquals(numberOfContainers, containers.size()); + + // Finish the application + FinishApplicationMasterRequest finishReq = + Records.newRecord(FinishApplicationMasterRequest.class); + finishReq.setDiagnostics(""); + finishReq.setTrackingUrl(""); + finishReq.setFinalApplicationStatus(FinalApplicationStatus.SUCCEEDED); + + FinishApplicationMasterResponse finishResp = interceptor.finishApplicationMaster(finishReq); + Assert.assertNotNull(finishResp); + Assert.assertTrue(finishResp.getIsUnregistered()); + + FederationRegistryClient client = interceptor.getRegistryClient(); + List applications = client.getAllApplications(); + Assert.assertNotNull(finishResp); + Assert.assertEquals(0, applications.size()); + return null; + }); + } + + @Test + public void testRemoveAppFromRegistryApplicationFailed() + throws IOException, InterruptedException { + + final RegisterApplicationMasterRequest registerReq = + Records.newRecord(RegisterApplicationMasterRequest.class); + + registerReq.setHost(Integer.toString(testAppId)); + registerReq.setRpcPort(testAppId); + registerReq.setTrackingUrl(""); + + UserGroupInformation ugi = interceptor.getUGIWithToken(interceptor.getAttemptId()); + + ugi.doAs((PrivilegedExceptionAction) () -> { + + // Register the application + RegisterApplicationMasterRequest registerReq1 = + Records.newRecord(RegisterApplicationMasterRequest.class); + registerReq1.setHost(Integer.toString(testAppId)); + registerReq1.setRpcPort(0); + registerReq1.setTrackingUrl(""); + + // Register ApplicationMaster + RegisterApplicationMasterResponse registerResponse = + interceptor.registerApplicationMaster(registerReq1); + Assert.assertNotNull(registerResponse); + lastResponseId = 0; + + Assert.assertEquals(0, interceptor.getUnmanagedAMPoolSize()); + + // Allocate the first batch of containers, with sc1 active + registerSubCluster(SubClusterId.newInstance("SC-1")); + + int numberOfContainers = 3; + List containers = + getContainersAndAssert(numberOfContainers, numberOfContainers); + Assert.assertEquals(1, interceptor.getUnmanagedAMPoolSize()); + Assert.assertEquals(numberOfContainers, containers.size()); + + // Finish the application + FinishApplicationMasterRequest finishReq = + Records.newRecord(FinishApplicationMasterRequest.class); + finishReq.setDiagnostics(""); + finishReq.setTrackingUrl(""); + finishReq.setFinalApplicationStatus(FinalApplicationStatus.FAILED); + + // Check Registry Applications + // At this time, the Application should not be cleaned up because the state is not SUCCESS. + FederationRegistryClient client = interceptor.getRegistryClient(); + List applications = client.getAllApplications(); + Assert.assertNotNull(applications); + Assert.assertEquals(1, applications.size()); + + // interceptor cleanupRegistry + ApplicationId applicationId = interceptor.getAttemptId().getApplicationId(); + client.removeAppFromRegistry(applicationId); + applications = client.getAllApplications(); + Assert.assertNotNull(applications); + Assert.assertEquals(0, applications.size()); + + return null; + }); + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/TestFederationInterceptorSecure.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/TestFederationInterceptorSecure.java new file mode 100644 index 0000000000000..bc895f395e71a --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/TestFederationInterceptorSecure.java @@ -0,0 +1,491 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.nodemanager.amrmproxy; + +import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION; +import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHORIZATION; + +import java.io.File; +import java.io.IOException; +import java.security.PrivilegedExceptionAction; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; + +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.ipc.Server; +import org.apache.hadoop.minikdc.MiniKdc; +import org.apache.hadoop.net.NetUtils; +import org.apache.hadoop.registry.client.api.RegistryOperations; +import org.apache.hadoop.registry.client.impl.FSRegistryOperationsService; +import org.apache.hadoop.security.SaslRpcServer.AuthMethod; +import org.apache.hadoop.security.SecurityUtil; +import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.security.authorize.PolicyProvider; +import org.apache.hadoop.security.authorize.ProxyUsers; +import org.apache.hadoop.security.authorize.Service; +import org.apache.hadoop.yarn.api.ApplicationClientProtocol; +import org.apache.hadoop.yarn.api.ApplicationClientProtocolPB; +import org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest; +import org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse; +import org.apache.hadoop.yarn.api.protocolrecords.FinishApplicationMasterRequest; +import org.apache.hadoop.yarn.api.protocolrecords.FinishApplicationMasterResponse; +import org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterRequest; +import org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterResponse; +import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; +import org.apache.hadoop.yarn.api.records.Container; +import org.apache.hadoop.yarn.api.records.ContainerId; +import org.apache.hadoop.yarn.api.records.FinalApplicationStatus; +import org.apache.hadoop.yarn.api.records.ResourceRequest; +import org.apache.hadoop.yarn.api.records.Token; +import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.exceptions.ApplicationMasterNotRegisteredException; +import org.apache.hadoop.yarn.exceptions.YarnException; +import org.apache.hadoop.yarn.ipc.YarnRPC; +import org.apache.hadoop.yarn.security.AMRMTokenIdentifier; +import org.apache.hadoop.yarn.server.MockResourceManagerFacade; +import org.apache.hadoop.yarn.server.federation.policies.manager.UniformBroadcastPolicyManager; +import org.apache.hadoop.yarn.server.federation.store.impl.MemoryFederationStateStore; +import org.apache.hadoop.yarn.server.federation.store.records.SubClusterId; +import org.apache.hadoop.yarn.server.federation.store.records.SubClusterInfo; +import org.apache.hadoop.yarn.server.federation.store.records.SubClusterRegisterRequest; +import org.apache.hadoop.yarn.server.federation.store.records.SubClusterState; +import org.apache.hadoop.yarn.server.federation.utils.FederationStateStoreFacade; +import org.apache.hadoop.yarn.server.nodemanager.Context; +import org.apache.hadoop.yarn.server.nodemanager.NodeManager.NMContext; +import org.apache.hadoop.yarn.server.nodemanager.recovery.NMMemoryStateStoreService; +import org.apache.hadoop.yarn.server.nodemanager.recovery.NMStateStoreService; +import org.apache.hadoop.yarn.util.Records; +import org.junit.AfterClass; +import org.junit.Assert; +import org.junit.BeforeClass; +import org.junit.ClassRule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class TestFederationInterceptorSecure extends BaseAMRMProxyTest { + + private static final Logger LOG = LoggerFactory.getLogger(TestFederationInterceptor.class); + + public static final String HOME_SC_ID = "SC-home"; + public static final String SC_ID1 = "SC-1"; + private static final String HOME_RM_ADDRESS = "127.0.0.1:28032"; + + private YarnConfiguration conf; + private TestableFederationInterceptor interceptor; + private MemoryFederationStateStore stateStore; + private NMStateStoreService nmStateStore; + private RegistryOperations registry; + + private Context nmContext; + private int testAppId; + private ApplicationAttemptId attemptId; + private volatile int lastResponseId; + + private MockResourceManagerFacade mockHomeRm = null; + private Server homeClietRMRpcServer; + + private static File workDir; + private static final File TEST_ROOT_DIR = + new File(System.getProperty("test.build.dir", "target/test-dir"), + TestFederationInterceptorSecure.class.getName() + "-root"); + private static MiniKdc miniKDC; + private static String serverUserName = "yarn"; + private static String serverPrincipal; + private static File serverKeytab; + + @ClassRule + public final static TemporaryFolder FOLDER = new TemporaryFolder(); + + @BeforeClass + public static void setUpCluster() throws Exception { + miniKDC = new MiniKdc(MiniKdc.createConf(), TEST_ROOT_DIR); + miniKDC.start(); + + workDir = FOLDER.getRoot(); + serverKeytab = new File(workDir, "yarn.keytab"); + serverPrincipal = + serverUserName + "/" + (Path.WINDOWS ? "127.0.0.1" : "localhost") + "@EXAMPLE.COM"; + miniKDC.createPrincipal(serverKeytab, serverPrincipal); + } + + @AfterClass + public static void tearDownCluster() { + if (miniKDC != null) { + miniKDC.stop(); + } + if (FOLDER != null) { + FOLDER.delete(); + } + } + + @Override + public void setUp() throws IOException { + super.setUp(); + + stateStore = new MemoryFederationStateStore(); + stateStore.init(conf); + FederationStateStoreFacade.getInstance().reinitialize(stateStore, conf); + + nmStateStore = new NMMemoryStateStoreService(); + nmStateStore.init(conf); + nmStateStore.start(); + + registry = new FSRegistryOperationsService(); + registry.init(conf); + registry.start(); + + testAppId = 1; + attemptId = getApplicationAttemptId(testAppId); + nmContext = new NMContext(null, null, null, null, nmStateStore, false, conf); + lastResponseId = 0; + this.mockHomeRm = new MockResourceManagerFacade(new YarnConfiguration(conf), 0); + + // Login as NodeManager + conf.set(YarnConfiguration.NM_KEYTAB, this.serverKeytab.getAbsolutePath()); + conf.set(YarnConfiguration.NM_PRINCIPAL, this.serverPrincipal); + SecurityUtil.login(conf, YarnConfiguration.NM_KEYTAB, YarnConfiguration.NM_PRINCIPAL); + + // start rpc server + startRpcServer(); + } + + private void startRpcServer() { + YarnRPC rpc = YarnRPC.create(conf); + this.homeClietRMRpcServer = rpc.getServer(ApplicationClientProtocol.class, mockHomeRm, + NetUtils.createSocketAddr(HOME_RM_ADDRESS), conf, null, 2); + this.homeClietRMRpcServer.start(); + this.homeClietRMRpcServer.refreshServiceAcl(conf, new MockRMPolicyProvider()); + } + + private void stopRpcServer() { + if (homeClietRMRpcServer != null) { + homeClietRMRpcServer.stop(); + } + } + + @Override + public void tearDown() { + interceptor.cleanupRegistry(); + interceptor.shutdown(); + registry.stop(); + stopRpcServer(); + super.tearDown(); + } + + @Override + protected YarnConfiguration createConfiguration() { + conf = new YarnConfiguration(); + conf.setBoolean(YarnConfiguration.AMRM_PROXY_ENABLED, true); + conf.setBoolean(YarnConfiguration.FEDERATION_ENABLED, true); + String mockPassThroughInterceptorClass = + PassThroughRequestInterceptor.class.getName(); + + // Create a request intercepter pipeline for testing. The last one in the + // chain is the federation intercepter that calls the mock resource manager. + // The others in the chain will simply forward it to the next one in the + // chain + conf.set(YarnConfiguration.AMRM_PROXY_INTERCEPTOR_CLASS_PIPELINE, + mockPassThroughInterceptorClass + "," + mockPassThroughInterceptorClass + + "," + TestableFederationInterceptor.class.getName()); + + conf.set(YarnConfiguration.FEDERATION_POLICY_MANAGER, + UniformBroadcastPolicyManager.class.getName()); + + conf.set(YarnConfiguration.RM_CLUSTER_ID, HOME_SC_ID); + + // Disable StateStoreFacade cache + conf.setInt(YarnConfiguration.FEDERATION_CACHE_TIME_TO_LIVE_SECS, 0); + + // Set sub-cluster timeout to 500ms + conf.setLong(YarnConfiguration.FEDERATION_AMRMPROXY_SUBCLUSTER_TIMEOUT, 500); + + // Enable security mode + conf.set(HADOOP_SECURITY_AUTHENTICATION, AuthMethod.KERBEROS.toString()); + conf.setBoolean(HADOOP_SECURITY_AUTHORIZATION, true); + UserGroupInformation.setConfiguration(conf); + conf.set("hadoop.proxyuser.yarn.hosts", "*"); + conf.set("hadoop.proxyuser.yarn.groups", "*"); + ProxyUsers.refreshSuperUserGroupsConfiguration(conf, ProxyUsers.CONF_HADOOP_PROXYUSER); + conf.set(YarnConfiguration.RM_PRINCIPAL, serverPrincipal); + + return conf; + } + + private void registerSubCluster(SubClusterId subClusterId) + throws YarnException { + if (subClusterId.getId().equals(HOME_SC_ID)) { + stateStore.registerSubCluster(SubClusterRegisterRequest.newInstance(SubClusterInfo + .newInstance(subClusterId, "1.2.3.4:1", HOME_RM_ADDRESS, "1.2.3.4:3", "1.2.3.4:4", + SubClusterState.SC_RUNNING, 0, "capacity"))); + } else { + stateStore.registerSubCluster(SubClusterRegisterRequest.newInstance(SubClusterInfo + .newInstance(subClusterId, "1.2.3.4:1", "1.2.3.4:2", "1.2.3.4:3", "1.2.3.4:4", + SubClusterState.SC_RUNNING, 0, "capacity"))); + } + } + + private List getContainersAndAssert(int numberOfResourceRequests, + int numberOfAllocationExcepted) throws Exception { + AllocateRequest allocateRequest = Records.newRecord(AllocateRequest.class); + List containers = new ArrayList(numberOfResourceRequests); + List askList = new ArrayList(numberOfResourceRequests); + for (int id = 0; id < numberOfResourceRequests; id++) { + askList.add(createResourceRequest("test-node-" + Integer.toString(id), 6000, 2, id % 5, 1)); + } + + allocateRequest.setAskList(askList); + + allocateRequest.setResponseId(lastResponseId); + AllocateResponse allocateResponse = interceptor.allocate(allocateRequest); + Assert.assertNotNull("allocate() returned null response", allocateResponse); + checkAMRMToken(allocateResponse.getAMRMToken()); + lastResponseId = allocateResponse.getResponseId(); + + containers.addAll(allocateResponse.getAllocatedContainers()); + LOG.info("Number of allocated containers in the original request: {}", + allocateResponse.getAllocatedContainers().size()); + + // Send max 10 heart beats to receive all the containers. If not, we will + // fail the test + int numHeartbeat = 0; + while (containers.size() < numberOfAllocationExcepted && numHeartbeat++ < 10) { + allocateRequest = Records.newRecord(AllocateRequest.class); + allocateRequest.setResponseId(lastResponseId); + allocateResponse = interceptor.allocate(allocateRequest); + Assert.assertNotNull("allocate() returned null response", allocateResponse); + checkAMRMToken(allocateResponse.getAMRMToken()); + lastResponseId = allocateResponse.getResponseId(); + + // Make sure this request is picked up by all async heartbeat handlers + interceptor.drainAllAsyncQueue(false); + + containers.addAll(allocateResponse.getAllocatedContainers()); + LOG.info("Number of allocated containers in this request: " + + Integer.toString(allocateResponse.getAllocatedContainers().size())); + LOG.info("Total number of allocated containers: {}", containers.size()); + Thread.sleep(10); + } + Assert.assertEquals(numberOfAllocationExcepted, containers.size()); + return containers; + } + + private void releaseContainersAndAssert(List containers) + throws Exception { + Assert.assertTrue(containers.size() > 0); + AllocateRequest allocateRequest = Records.newRecord(AllocateRequest.class); + List relList = new ArrayList(containers.size()); + for (Container container : containers) { + relList.add(container.getId()); + } + + allocateRequest.setReleaseList(relList); + + allocateRequest.setResponseId(lastResponseId); + AllocateResponse allocateResponse = interceptor.allocate(allocateRequest); + Assert.assertNotNull(allocateResponse); + checkAMRMToken(allocateResponse.getAMRMToken()); + lastResponseId = allocateResponse.getResponseId(); + + // The release request will be split and handled by the corresponding UAM. + // The release containers returned by the mock resource managers will be + // aggregated and returned back to us and we can check if total request size + // and returned size are the same + List containersForReleasedContainerIds = new ArrayList(); + List newlyFinished = getCompletedContainerIds( + allocateResponse.getCompletedContainersStatuses()); + containersForReleasedContainerIds.addAll(newlyFinished); + LOG.info("Number of containers received in the original request: {}", newlyFinished.size()); + + // Send max 10 heart beats to receive all the containers. If not, we will fail the test + int numHeartbeat = 0; + while (containersForReleasedContainerIds.size() < relList.size() && numHeartbeat++ < 10) { + allocateRequest = Records.newRecord(AllocateRequest.class); + allocateRequest.setResponseId(lastResponseId); + allocateResponse = interceptor.allocate(allocateRequest); + Assert.assertNotNull(allocateResponse); + checkAMRMToken(allocateResponse.getAMRMToken()); + lastResponseId = allocateResponse.getResponseId(); + + // Make sure this request is picked up by all async heartbeat handlers + interceptor.drainAllAsyncQueue(false); + + newlyFinished = getCompletedContainerIds(allocateResponse.getCompletedContainersStatuses()); + containersForReleasedContainerIds.addAll(newlyFinished); + LOG.info("Number of containers received in this request: ", newlyFinished.size()); + LOG.info("Total number of containers received: ", containersForReleasedContainerIds.size()); + Thread.sleep(10); + } + + Assert.assertEquals(relList.size(), containersForReleasedContainerIds.size()); + } + + private void checkAMRMToken(Token amrmToken) { + if (amrmToken != null) { + // The token should be the one issued by home MockRM + Assert.assertTrue(amrmToken.getKind().equals(Integer.toString(0))); + } + } + + @Test + public void testRecoverWithAMRMProxyHA() throws Exception { + testRecoverWithRPCClientRM(registry); + } + + @Test + public void testRecoverWithoutAMRMProxyHA() throws Exception { + testRecoverWithRPCClientRM(null); + } + + protected void testRecoverWithRPCClientRM( + final RegistryOperations registryObj) + throws Exception { + UserGroupInformation ugi = this.getUGIWithToken(attemptId); + ugi.doAs(new PrivilegedExceptionAction() { + @Override + public Object run() throws Exception { + interceptor = new TestableFederationInterceptor(mockHomeRm); + interceptor.init(new AMRMProxyApplicationContextImpl(nmContext, conf, attemptId, + "test-user", null, null, null, registryObj)); + interceptor.cleanupRegistry(); + + // Register the application + RegisterApplicationMasterRequest registerReq = + Records.newRecord(RegisterApplicationMasterRequest.class); + registerReq.setHost(Integer.toString(testAppId)); + registerReq.setRpcPort(testAppId); + registerReq.setTrackingUrl(""); + + RegisterApplicationMasterResponse registerResponse = + interceptor.registerApplicationMaster(registerReq); + Assert.assertNotNull(registerResponse); + lastResponseId = 0; + + Assert.assertEquals(0, interceptor.getUnmanagedAMPoolSize()); + + // Allocate one batch of containers + registerSubCluster(SubClusterId.newInstance(SC_ID1)); + registerSubCluster(SubClusterId.newInstance(HOME_SC_ID)); + + int numberOfContainers = 3; + List containers = + getContainersAndAssert(numberOfContainers, numberOfContainers * 2); + Assert.assertEquals(1, interceptor.getUnmanagedAMPoolSize()); + + // Make sure all async hb threads are done + interceptor.drainAllAsyncQueue(true); + + // Prepare for Federation Interceptor restart and recover + Map recoveredDataMap = recoverDataMapForAppAttempt(nmStateStore, attemptId); + String scEntry = FederationInterceptor.NMSS_SECONDARY_SC_PREFIX + "SC-1"; + if (registryObj == null) { + Assert.assertTrue(recoveredDataMap.containsKey(scEntry)); + } else { + // When AMRMPRoxy HA is enabled, NMSS should not have the UAM token, + // it should be in Registry + Assert.assertFalse(recoveredDataMap.containsKey(scEntry)); + } + + // Preserve the mock RM instances + MockResourceManagerFacade homeRM = interceptor.getHomeRM(); + ConcurrentHashMap secondaries = + interceptor.getSecondaryRMs(); + + // Create a new intercepter instance and recover + interceptor = new TestableFederationInterceptor(homeRM, secondaries); + interceptor.init(new AMRMProxyApplicationContextImpl(nmContext, conf, attemptId, + "test-user", null, null, null, registryObj)); + interceptor.setClientRPC(true); + interceptor.recover(recoveredDataMap); + interceptor.setClientRPC(false); + + Assert.assertEquals(1, interceptor.getUnmanagedAMPoolSize()); + // SC-1 should be initialized to be timed out + Assert.assertEquals(1, interceptor.getTimedOutSCs(true).size()); + + // The first allocate call expects a fail-over exception and re-register + try { + AllocateRequest allocateRequest = Records.newRecord(AllocateRequest.class); + allocateRequest.setResponseId(lastResponseId); + AllocateResponse allocateResponse = interceptor.allocate(allocateRequest); + lastResponseId = allocateResponse.getResponseId(); + Assert.fail("Expecting an ApplicationMasterNotRegisteredException " + + " after FederationInterceptor restarts and recovers"); + } catch (ApplicationMasterNotRegisteredException e) { + } + interceptor.registerApplicationMaster(registerReq); + lastResponseId = 0; + + // Release all containers + releaseContainersAndAssert(containers); + + // Finish the application + FinishApplicationMasterRequest finishReq = + Records.newRecord(FinishApplicationMasterRequest.class); + finishReq.setDiagnostics(""); + finishReq.setTrackingUrl(""); + finishReq.setFinalApplicationStatus(FinalApplicationStatus.SUCCEEDED); + + FinishApplicationMasterResponse finshResponse = + interceptor.finishApplicationMaster(finishReq); + Assert.assertNotNull(finshResponse); + Assert.assertEquals(true, finshResponse.getIsUnregistered()); + + // After the application succeeds, the registry/NMSS entry should be + // cleaned up + if (registryObj != null) { + Assert.assertEquals(0, interceptor.getRegistryClient().getAllApplications().size()); + } else { + recoveredDataMap = recoverDataMapForAppAttempt(nmStateStore, attemptId); + Assert.assertFalse(recoveredDataMap.containsKey(scEntry)); + } + return null; + } + }); + } + + private UserGroupInformation getUGIWithToken( + ApplicationAttemptId appAttemptId) throws IOException { + UserGroupInformation ugi = UserGroupInformation.getLoginUser(); + AMRMTokenIdentifier token = new AMRMTokenIdentifier(appAttemptId, 1); + ugi.addTokenIdentifier(token); + return ugi; + } + + private class MockRMPolicyProvider extends PolicyProvider { + + private MockRMPolicyProvider() { + } + + private final Service[] resourceManagerServices = + new Service[]{ + new Service( + YarnConfiguration.YARN_SECURITY_SERVICE_AUTHORIZATION_APPLICATIONCLIENT_PROTOCOL, + ApplicationClientProtocolPB.class), + }; + + @Override + public Service[] getServices() { + return resourceManagerServices; + } + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/TestableFederationInterceptor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/TestableFederationInterceptor.java index 7c57ab1292068..3b127974cf646 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/TestableFederationInterceptor.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/TestableFederationInterceptor.java @@ -53,9 +53,15 @@ public class TestableFederationInterceptor extends FederationInterceptor { private AtomicInteger runningIndex = new AtomicInteger(0); private MockResourceManagerFacade mockRm; + private boolean isClientRPC = false; + public TestableFederationInterceptor() { } + public TestableFederationInterceptor(MockResourceManagerFacade homeRM) { + mockRm = homeRM; + } + public TestableFederationInterceptor(MockResourceManagerFacade homeRM, ConcurrentHashMap secondaries) { mockRm = homeRM; @@ -79,6 +85,9 @@ protected AMHeartbeatRequestHandler createHomeHeartbeartHandler( @Override protected T createHomeRMProxy(AMRMProxyApplicationContext appContext, Class protocol, UserGroupInformation user) { + if (isClientRPC) { + return super.createHomeRMProxy(appContext, protocol, user); + } synchronized (this) { if (mockRm == null) { mockRm = new MockResourceManagerFacade( @@ -273,4 +282,8 @@ public Object run() { } } } + + public void setClientRPC(boolean clientRPC) { + this.isClientRPC = clientRPC; + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/numa/TestNumaResourceAllocator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/numa/TestNumaResourceAllocator.java index c8072c327a9b6..42c495c88f8f6 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/numa/TestNumaResourceAllocator.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/numa/TestNumaResourceAllocator.java @@ -95,7 +95,7 @@ public void testReadNumaTopologyFromCmdOutput() throws Exception { + "1: 20 10"; numaResourceAllocator = new NumaResourceAllocator(mock(Context.class)) { @Override - String executeNGetCmdOutput(Configuration config) + public String executeNGetCmdOutput(Configuration config) throws YarnRuntimeException { return cmdOutput; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestLogAggregationService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestLogAggregationService.java index 4cc9ac1f3a7a4..8185f5019c798 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestLogAggregationService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestLogAggregationService.java @@ -234,31 +234,47 @@ private void verifyLocalFileDeletion( // ensure filesystems were closed verify(logAggregationService).closeFileSystems( any(UserGroupInformation.class)); - List dirList = new ArrayList<>(); - dirList.add(new Path(app1LogDir.toURI())); - verify(delSrvc, times(2)).delete(argThat(new FileDeletionMatcher( - delSrvc, user, null, dirList))); - - String containerIdStr = container11.toString(); - File containerLogDir = new File(app1LogDir, containerIdStr); - int count = 0; - int maxAttempts = 50; - for (String fileType : new String[] { "stdout", "stderr", "syslog" }) { - File f = new File(containerLogDir, fileType); - count = 0; - while ((f.exists()) && (count < maxAttempts)) { - count++; - Thread.sleep(100); + boolean filesShouldBeDeleted = + this.conf.getBoolean(YarnConfiguration.LOG_AGGREGATION_ENABLE_LOCAL_CLEANUP, + YarnConfiguration.DEFAULT_LOG_AGGREGATION_ENABLE_LOCAL_CLEANUP); + if (filesShouldBeDeleted) { + List dirList = new ArrayList<>(); + dirList.add(new Path(app1LogDir.toURI())); + verify(delSrvc, times(2)).delete(argThat(new FileDeletionMatcher( + delSrvc, user, null, dirList))); + + String containerIdStr = container11.toString(); + File containerLogDir = new File(app1LogDir, containerIdStr); + int count = 0; + int maxAttempts = 50; + for (String fileType : new String[]{"stdout", "stderr", "syslog"}) { + File f = new File(containerLogDir, fileType); + count = 0; + while ((f.exists()) && (count < maxAttempts)) { + count++; + Thread.sleep(100); + } + Assert.assertFalse("File [" + f + "] was not deleted", f.exists()); } - Assert.assertFalse("File [" + f + "] was not deleted", f.exists()); - } - count = 0; - while ((app1LogDir.exists()) && (count < maxAttempts)) { - count++; - Thread.sleep(100); + Assert.assertFalse("Directory [" + app1LogDir + "] was not deleted", + app1LogDir.exists()); + } else { + List dirList = new ArrayList<>(); + dirList.add(new Path(app1LogDir.toURI())); + verify(delSrvc, never()).delete(argThat(new FileDeletionMatcher( + delSrvc, user, null, dirList))); + + String containerIdStr = container11.toString(); + File containerLogDir = new File(app1LogDir, containerIdStr); + Thread.sleep(5000); + for (String fileType : new String[]{"stdout", "stderr", "syslog"}) { + File f = new File(containerLogDir, fileType); + Assert.assertTrue("File [" + f + "] was not deleted", f.exists()); + } + Assert.assertTrue("Directory [" + app1LogDir + "] was not deleted", + app1LogDir.exists()); } - Assert.assertFalse("Directory [" + app1LogDir + "] was not deleted", - app1LogDir.exists()); + delSrvc.stop(); Path logFilePath = logAggregationService .getLogAggregationFileController(conf) @@ -297,6 +313,20 @@ public void testLocalFileDeletionAfterUpload() throws Exception { verifyLocalFileDeletion(logAggregationService); } + @Test + public void testLocalFileRemainsAfterUploadOnCleanupDisable() throws Exception { + this.delSrvc = new DeletionService(createContainerExecutor()); + delSrvc = spy(delSrvc); + this.delSrvc.init(conf); + this.conf.setBoolean(YarnConfiguration.LOG_AGGREGATION_ENABLE_LOCAL_CLEANUP, false); + this.conf.set(YarnConfiguration.NM_LOG_DIRS, localLogDir.getAbsolutePath()); + this.conf.set(YarnConfiguration.NM_REMOTE_APP_LOG_DIR, + this.remoteRootLogDir.getAbsolutePath()); + LogAggregationService logAggregationService = spy( + new LogAggregationService(dispatcher, this.context, this.delSrvc, super.dirsHandler)); + verifyLocalFileDeletion(logAggregationService); + } + @Test public void testLocalFileDeletionOnDiskFull() throws Exception { this.delSrvc = new DeletionService(createContainerExecutor()); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/recovery/TestNMLeveldbStateStoreService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/recovery/TestNMLeveldbStateStoreService.java index 06ad727b2bea2..74968746bc136 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/recovery/TestNMLeveldbStateStoreService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/recovery/TestNMLeveldbStateStoreService.java @@ -1756,6 +1756,18 @@ public void testStateStoreForResourceMapping() throws IOException { resources = rcs.getResourceMappings().getAssignedResources("numa"); Assert.assertEquals(numaRes, resources); Assert.assertEquals(numaRes, resourceMappings.getAssignedResources("numa")); + // test removing numa resources from state store + stateStore.releaseAssignedResources(containerId, "numa"); + recoveredContainers = loadContainersState(stateStore.getContainerStateIterator()); + resourceMappings = recoveredContainers.get(0).getResourceMappings(); + assertTrue(resourceMappings.getAssignedResources("numa").isEmpty()); + + // testing calling deletion of non-existing key doesn't break anything + try { + stateStore.releaseAssignedResources(containerId, "numa"); + }catch (RuntimeException e){ + Assert.fail("Should not throw exception while deleting non existing key from statestore"); + } } @Test diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/pom.xml index 025827c42f5f4..9d096d20c5fdd 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/pom.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/pom.xml @@ -334,10 +334,6 @@ com.fasterxml.jackson.dataformat jackson-dataformat-yaml - - javax.ws.rs - javax.ws.rs-api - diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMSecretManagerService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMSecretManagerService.java index 2933b40f4ac00..32369ba53ed5c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMSecretManagerService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMSecretManagerService.java @@ -30,6 +30,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.security.RMDelegationTokenSecretManager; import java.io.IOException; +import java.util.concurrent.TimeUnit; public class RMSecretManagerService extends AbstractService { @@ -135,9 +136,13 @@ protected RMDelegationTokenSecretManager createRMDelegationTokenSecretManager( long tokenRenewInterval = conf.getLong(YarnConfiguration.RM_DELEGATION_TOKEN_RENEW_INTERVAL_KEY, YarnConfiguration.RM_DELEGATION_TOKEN_RENEW_INTERVAL_DEFAULT); + long removeScanInterval = + conf.getTimeDuration(YarnConfiguration.RM_DELEGATION_TOKEN_REMOVE_SCAN_INTERVAL_KEY, + YarnConfiguration.RM_DELEGATION_TOKEN_REMOVE_SCAN_INTERVAL_DEFAULT, + TimeUnit.MILLISECONDS); return new RMDelegationTokenSecretManager(secretKeyInterval, - tokenMaxLifetime, tokenRenewInterval, 3600000, rmContext); + tokenMaxLifetime, tokenRenewInterval, removeScanInterval, rmContext); } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/federation/FederationStateStoreService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/federation/FederationStateStoreService.java index c4dae7d4f7d3f..a473186ed63df 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/federation/FederationStateStoreService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/federation/FederationStateStoreService.java @@ -33,12 +33,20 @@ import org.apache.hadoop.yarn.server.federation.store.FederationStateStore; import org.apache.hadoop.yarn.server.federation.store.records.AddApplicationHomeSubClusterRequest; import org.apache.hadoop.yarn.server.federation.store.records.AddApplicationHomeSubClusterResponse; +import org.apache.hadoop.yarn.server.federation.store.records.AddReservationHomeSubClusterRequest; +import org.apache.hadoop.yarn.server.federation.store.records.AddReservationHomeSubClusterResponse; import org.apache.hadoop.yarn.server.federation.store.records.DeleteApplicationHomeSubClusterRequest; import org.apache.hadoop.yarn.server.federation.store.records.DeleteApplicationHomeSubClusterResponse; +import org.apache.hadoop.yarn.server.federation.store.records.DeleteReservationHomeSubClusterRequest; +import org.apache.hadoop.yarn.server.federation.store.records.DeleteReservationHomeSubClusterResponse; import org.apache.hadoop.yarn.server.federation.store.records.GetApplicationHomeSubClusterRequest; import org.apache.hadoop.yarn.server.federation.store.records.GetApplicationHomeSubClusterResponse; import org.apache.hadoop.yarn.server.federation.store.records.GetApplicationsHomeSubClusterRequest; import org.apache.hadoop.yarn.server.federation.store.records.GetApplicationsHomeSubClusterResponse; +import org.apache.hadoop.yarn.server.federation.store.records.GetReservationHomeSubClusterRequest; +import org.apache.hadoop.yarn.server.federation.store.records.GetReservationHomeSubClusterResponse; +import org.apache.hadoop.yarn.server.federation.store.records.GetReservationsHomeSubClusterRequest; +import org.apache.hadoop.yarn.server.federation.store.records.GetReservationsHomeSubClusterResponse; import org.apache.hadoop.yarn.server.federation.store.records.GetSubClusterInfoRequest; import org.apache.hadoop.yarn.server.federation.store.records.GetSubClusterInfoResponse; import org.apache.hadoop.yarn.server.federation.store.records.GetSubClusterPoliciesConfigurationsRequest; @@ -60,6 +68,8 @@ import org.apache.hadoop.yarn.server.federation.store.records.SubClusterState; import org.apache.hadoop.yarn.server.federation.store.records.UpdateApplicationHomeSubClusterRequest; import org.apache.hadoop.yarn.server.federation.store.records.UpdateApplicationHomeSubClusterResponse; +import org.apache.hadoop.yarn.server.federation.store.records.UpdateReservationHomeSubClusterRequest; +import org.apache.hadoop.yarn.server.federation.store.records.UpdateReservationHomeSubClusterResponse; import org.apache.hadoop.yarn.server.federation.utils.FederationStateStoreFacade; import org.apache.hadoop.yarn.server.records.Version; import org.apache.hadoop.yarn.server.resourcemanager.RMContext; @@ -86,6 +96,7 @@ public class FederationStateStoreService extends AbstractService private FederationStateStore stateStoreClient = null; private SubClusterId subClusterId; private long heartbeatInterval; + private long heartbeatInitialDelay; private RMContext rmContext; public FederationStateStoreService(RMContext rmContext) { @@ -116,10 +127,24 @@ protected void serviceInit(Configuration conf) throws Exception { heartbeatInterval = conf.getLong( YarnConfiguration.FEDERATION_STATESTORE_HEARTBEAT_INTERVAL_SECS, YarnConfiguration.DEFAULT_FEDERATION_STATESTORE_HEARTBEAT_INTERVAL_SECS); + if (heartbeatInterval <= 0) { heartbeatInterval = YarnConfiguration.DEFAULT_FEDERATION_STATESTORE_HEARTBEAT_INTERVAL_SECS; } + + heartbeatInitialDelay = conf.getTimeDuration( + YarnConfiguration.FEDERATION_STATESTORE_HEARTBEAT_INITIAL_DELAY, + YarnConfiguration.DEFAULT_FEDERATION_STATESTORE_HEARTBEAT_INITIAL_DELAY, + TimeUnit.SECONDS); + + if (heartbeatInitialDelay <= 0) { + LOG.warn("{} configured value is wrong, must be > 0; using default value of {}", + YarnConfiguration.FEDERATION_STATESTORE_HEARTBEAT_INITIAL_DELAY, + YarnConfiguration.DEFAULT_FEDERATION_STATESTORE_HEARTBEAT_INITIAL_DELAY); + heartbeatInitialDelay = + YarnConfiguration.DEFAULT_FEDERATION_STATESTORE_HEARTBEAT_INITIAL_DELAY; + } LOG.info("Initialized federation membership service."); super.serviceInit(conf); @@ -196,9 +221,9 @@ private void registerAndInitializeHeartbeat() { scheduledExecutorService = HadoopExecutors.newSingleThreadScheduledExecutor(); scheduledExecutorService.scheduleWithFixedDelay(stateStoreHeartbeat, - heartbeatInterval, heartbeatInterval, TimeUnit.SECONDS); - LOG.info("Started federation membership heartbeat with interval: {}", - heartbeatInterval); + heartbeatInitialDelay, heartbeatInterval, TimeUnit.SECONDS); + LOG.info("Started federation membership heartbeat with interval: {} and initial delay: {}", + heartbeatInterval, heartbeatInitialDelay); } @VisibleForTesting @@ -301,4 +326,34 @@ public DeleteApplicationHomeSubClusterResponse deleteApplicationHomeSubCluster( DeleteApplicationHomeSubClusterRequest request) throws YarnException { return stateStoreClient.deleteApplicationHomeSubCluster(request); } + + @Override + public AddReservationHomeSubClusterResponse addReservationHomeSubCluster( + AddReservationHomeSubClusterRequest request) throws YarnException { + return stateStoreClient.addReservationHomeSubCluster(request); + } + + @Override + public GetReservationHomeSubClusterResponse getReservationHomeSubCluster( + GetReservationHomeSubClusterRequest request) throws YarnException { + return stateStoreClient.getReservationHomeSubCluster(request); + } + + @Override + public GetReservationsHomeSubClusterResponse getReservationsHomeSubCluster( + GetReservationsHomeSubClusterRequest request) throws YarnException { + return stateStoreClient.getReservationsHomeSubCluster(request); + } + + @Override + public UpdateReservationHomeSubClusterResponse updateReservationHomeSubCluster( + UpdateReservationHomeSubClusterRequest request) throws YarnException { + return stateStoreClient.updateReservationHomeSubCluster(request); + } + + @Override + public DeleteReservationHomeSubClusterResponse deleteReservationHomeSubCluster( + DeleteReservationHomeSubClusterRequest request) throws YarnException { + return stateStoreClient.deleteReservationHomeSubCluster(request); + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/PartitionQueueMetrics.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/PartitionQueueMetrics.java index 02eaa7bd9b7de..150abdf51d06d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/PartitionQueueMetrics.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/PartitionQueueMetrics.java @@ -78,7 +78,7 @@ public synchronized QueueMetrics getUserMetrics(String userName) { metrics = new PartitionQueueMetrics(this.metricsSystem, this.queueName, null, false, this.conf, this.partition); users.put(userName, metrics); - metricsSystem.register( + registerMetrics( pSourceName(partitionJMXStr).append(qSourceName(queueName)) .append(",user=").append(userName).toString(), "Metrics for user '" + userName + "' in queue '" + queueName + "'", diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/QueueMetrics.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/QueueMetrics.java index d41cbed0b107a..0bfee4d33500e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/QueueMetrics.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/QueueMetrics.java @@ -289,7 +289,7 @@ public synchronized QueueMetrics getUserMetrics(String userName) { metrics = new QueueMetrics(metricsSystem, queueName, null, false, conf); users.put(userName, metrics); - metricsSystem.register( + registerMetrics( sourceName(queueName).append(",user=").append(userName).toString(), "Metrics for user '"+ userName +"' in queue '"+ queueName +"'", metrics.tag(QUEUE_INFO, queueName).tag(USER_INFO, userName)); @@ -334,7 +334,7 @@ public synchronized QueueMetrics getPartitionQueueMetrics(String partition) { QueueMetrics queueMetrics = new PartitionQueueMetrics(metricsSystem, this.queueName, parentQueue, this.enableUserMetrics, this.conf, partition); - metricsSystem.register( + registerMetrics( pSourceName(partitionJMXStr).append(qSourceName(this.queueName)) .toString(), "Metrics for queue: " + this.queueName, @@ -378,7 +378,7 @@ private QueueMetrics getPartitionMetrics(String partition) { // Register with the MetricsSystems if (metricsSystem != null) { - metricsSystem.register(pSourceName(partitionJMXStr).toString(), + registerMetrics(pSourceName(partitionJMXStr).toString(), "Metrics for partition: " + partitionJMXStr, (PartitionQueueMetrics) metrics.tag(PARTITION_INFO, partitionJMXStr)); @@ -1359,4 +1359,15 @@ public void setParentQueue(Queue parentQueue) { } } } + + protected void registerMetrics(String sourceName, String desc, QueueMetrics metrics) { + MetricsSource source = metricsSystem.getSource(sourceName); + // Unregister metrics if a source is already present + if (source != null) { + LOG.info("Unregistering source " + sourceName); + metricsSystem.unregisterSource(sourceName); + } + + metricsSystem.register(sourceName, desc, metrics); + } } \ No newline at end of file diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AutoCreatedQueueTemplate.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AutoCreatedQueueTemplate.java index 8fa1e130d4f5e..bbeca4b072450 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AutoCreatedQueueTemplate.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AutoCreatedQueueTemplate.java @@ -42,7 +42,6 @@ public class AutoCreatedQueueTemplate { AUTO_QUEUE_CREATION_V2_PREFIX + "parent-template."; private static final String WILDCARD_QUEUE = "*"; - private static final int MAX_WILDCARD_LEVEL = 1; private final Map templateProperties = new HashMap<>(); private final Map leafOnlyProperties = new HashMap<>(); @@ -86,7 +85,7 @@ public Map getParentOnlyProperties() { /** * Sets the common template properties and parent specific template * properties of a child queue based on its parent template settings. - * @param conf configuration to set + * @param conf configuration to set * @param childQueuePath child queue path used for prefixing the properties */ public void setTemplateEntriesForChild(CapacitySchedulerConfiguration conf, @@ -169,8 +168,8 @@ private void setTemplateConfigEntries(CapacitySchedulerConfiguration configurati // start with the most explicit format (without wildcard) int wildcardLevel = 0; // root can not be wildcarded - // MAX_WILDCARD_LEVEL will be configurable in the future - int supportedWildcardLevel = Math.min(queuePathMaxIndex, MAX_WILDCARD_LEVEL); + int supportedWildcardLevel = Math.min(queuePathMaxIndex, + configuration.getMaximumAutoCreatedQueueDepth(queuePath.getFullPath())); // Allow root to have template properties if (queuePath.isRoot()) { supportedWildcardLevel = 0; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueMetrics.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueMetrics.java index 586f837f147b2..16ebc15512847 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueMetrics.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueMetrics.java @@ -21,6 +21,7 @@ import java.util.Map; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.metrics2.MetricsSource; import org.apache.hadoop.metrics2.MetricsSystem; import org.apache.hadoop.metrics2.annotation.Metric; import org.apache.hadoop.metrics2.annotation.Metrics; @@ -240,6 +241,10 @@ public synchronized static CSQueueMetrics forQueue(String queueName, // Register with the MetricsSystems if (ms != null) { + MetricsSource source = ms.getSource(sourceName(queueName).toString()); + if (source != null) { + ms.unregisterSource(sourceName(queueName).toString()); + } metrics = ms.register(sourceName(queueName).toString(), "Metrics for queue: " + queueName, metrics); @@ -260,7 +265,7 @@ public synchronized QueueMetrics getUserMetrics(String userName) { metrics = new CSQueueMetrics(metricsSystem, queueName, null, false, conf); users.put(userName, metrics); - metricsSystem.register( + registerMetrics( sourceName(queueName).append(",user=").append(userName).toString(), "Metrics for user '" + userName + "' in queue '" + queueName + "'", ((CSQueueMetrics) metrics.tag(QUEUE_INFO, queueName)).tag(USER_INFO, diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfigValidator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfigValidator.java index 147f392ad9280..5ddc709b57c9c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfigValidator.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfigValidator.java @@ -23,6 +23,7 @@ import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.exceptions.YarnRuntimeException; import org.apache.hadoop.yarn.server.resourcemanager.RMContext; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueMetrics; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -55,6 +56,8 @@ public static boolean validateCSConfiguration( return true; } finally { newCs.stop(); + QueueMetrics.clearQueueMetrics(); + liveScheduler.resetSchedulerMetrics(); } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/NodeIDsInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/NodeIDsInfo.java index b492793927d3e..7bf8a63786c62 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/NodeIDsInfo.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/NodeIDsInfo.java @@ -24,6 +24,9 @@ import java.util.ArrayList; import java.util.List; +import java.util.Set; +import java.util.Collection; +import java.util.HashSet; import javax.xml.bind.annotation.XmlAccessType; import javax.xml.bind.annotation.XmlAccessorType; @@ -58,6 +61,11 @@ public NodeIDsInfo(List nodeIdsList, Resource resource) { this.partitionInfo = new PartitionInfo(new ResourceInfo(resource)); } + public NodeIDsInfo(Collection nodeIdsList, PartitionInfo partitionInfo) { + this.nodeIDsList.addAll(nodeIdsList); + this.partitionInfo = partitionInfo; + } + public ArrayList getNodeIDs() { return nodeIDsList; } @@ -65,4 +73,36 @@ public ArrayList getNodeIDs() { public PartitionInfo getPartitionInfo() { return partitionInfo; } + + /** + * This method will generate a new NodeIDsInfo object based on the two NodeIDsInfo objects. + * The information to be combined includes the node list (removed duplicate node) + * and partitionInfo object. + * + * @param left left NodeIDsInfo Object. + * @param right right NodeIDsInfo Object. + * @return new NodeIDsInfo Object. + */ + public static NodeIDsInfo add(NodeIDsInfo left, NodeIDsInfo right) { + Set nodes = new HashSet<>(); + if (left != null && left.nodeIDsList != null) { + nodes.addAll(left.nodeIDsList); + } + if (right != null && right.nodeIDsList != null) { + nodes.addAll(right.nodeIDsList); + } + + PartitionInfo leftPartitionInfo = null; + if (left != null) { + leftPartitionInfo = left.getPartitionInfo(); + } + + PartitionInfo rightPartitionInfo = null; + if (right != null) { + rightPartitionInfo = right.getPartitionInfo(); + } + + PartitionInfo info = PartitionInfo.addTo(leftPartitionInfo, rightPartitionInfo); + return new NodeIDsInfo(nodes, info); + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServiceProtocol.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServiceProtocol.java index 41fc4ea870985..6c0309969eebb 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServiceProtocol.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServiceProtocol.java @@ -62,6 +62,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ResourceOptionInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.BulkActivitiesInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.SchedulerTypeInfo; +import org.apache.hadoop.yarn.webapp.dao.SchedConfUpdateInfo; /** *

    @@ -745,4 +746,31 @@ RMQueueAclInfo checkUserAccessToQueue(String queue, String username, */ Response signalToContainer(String containerId, String command, HttpServletRequest req) throws AuthorizationException; + + /** + * This method updates the Scheduler configuration, and it is reachable by + * using {@link RMWSConsts#SCHEDULER_CONF}. + * + * @param mutationInfo th information for making scheduler configuration + * changes (supports adding, removing, or updating a queue, as well + * as global scheduler conf changes) + * @param hsr the servlet request + * @return Response containing the status code + * @throws AuthorizationException if the user is not authorized to invoke this + * method + * @throws InterruptedException if interrupted + */ + Response updateSchedulerConfiguration(SchedConfUpdateInfo mutationInfo, + HttpServletRequest hsr) throws AuthorizationException, InterruptedException; + + /** + * This method retrieves all the Scheduler configuration, and it is reachable + * by using {@link RMWSConsts#SCHEDULER_CONF}. + * + * @param hsr the servlet request + * @return Response containing the status code + * @throws AuthorizationException if the user is not authorized to invoke this + * method. + */ + Response getSchedulerConfiguration(HttpServletRequest hsr) throws AuthorizationException; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java index 26c921796b3c4..6a6a2e1de1e81 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java @@ -2748,26 +2748,26 @@ public synchronized Response updateSchedulerConfiguration(SchedConfUpdateInfo initForWritableEndpoints(callerUGI, true); ResourceScheduler scheduler = rm.getResourceScheduler(); - if (isConfigurationMutable(scheduler)) { + if (!(scheduler instanceof MutableConfScheduler)) { + return Response.status(Status.BAD_REQUEST) + .entity("Configuration change only supported by MutableConfScheduler.").build(); + } else if (!((MutableConfScheduler) scheduler).isConfigurationMutable()) { + return Response.status(Status.BAD_REQUEST) + .entity("Configuration change only supported by mutable configuration store.").build(); + } else { try { callerUGI.doAs((PrivilegedExceptionAction) () -> { - MutableConfigurationProvider provider = ((MutableConfScheduler) - scheduler).getMutableConfProvider(); + MutableConfigurationProvider provider = + ((MutableConfScheduler) scheduler).getMutableConfProvider(); LogMutation logMutation = applyMutation(provider, callerUGI, mutationInfo); return refreshQueues(provider, logMutation); }); } catch (IOException e) { LOG.error("Exception thrown when modifying configuration.", e); - return Response.status(Status.BAD_REQUEST).entity(e.getMessage()) - .build(); + return Response.status(Status.BAD_REQUEST).entity(e.getMessage()).build(); } return Response.status(Status.OK).entity("Configuration change successfully applied.") .build(); - } else { - return Response.status(Status.BAD_REQUEST) - .entity(String.format("Configuration change only supported by " + - "%s.", MutableConfScheduler.class.getSimpleName())) - .build(); } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java index 6ebaa06174eff..481544e477c72 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java @@ -27,6 +27,8 @@ import javax.xml.bind.annotation.XmlRootElement; import javax.xml.bind.annotation.XmlTransient; +import org.apache.hadoop.HadoopIllegalArgumentException; +import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ApplicationResourceUsageReport; import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext; @@ -38,6 +40,8 @@ import org.apache.hadoop.yarn.api.records.ResourceRequest; import org.apache.hadoop.yarn.api.records.SchedulingRequest; import org.apache.hadoop.yarn.api.records.YarnApplicationState; +import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.server.federation.store.records.SubClusterIdInfo; import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppMetrics; @@ -69,6 +73,8 @@ public class AppInfo { protected ApplicationId applicationId; @XmlTransient private String schemePrefix; + @XmlTransient + private SubClusterIdInfo subClusterId; // these are ok for any user to see protected String id; @@ -82,6 +88,7 @@ public class AppInfo { protected String trackingUrl; protected String diagnostics; protected long clusterId; + protected String rmClusterId; protected String applicationType; protected String applicationTags = ""; protected int priority; @@ -182,6 +189,16 @@ public AppInfo(ResourceManager rm, RMApp app, Boolean hasAccess, this.finalStatus = app.getFinalApplicationStatus(); this.clusterId = ResourceManager.getClusterTimeStamp(); if (hasAccess) { + if (rm != null && rm.getConfig() != null) { + try { + Configuration yarnConfig = rm.getConfig(); + subClusterId = new SubClusterIdInfo(YarnConfiguration.getClusterId(yarnConfig)); + rmClusterId = this.subClusterId.toId().toString(); + } catch (HadoopIllegalArgumentException e) { + subClusterId = null; + rmClusterId = null; + } + } this.startedTime = app.getStartTime(); this.launchTime = app.getLaunchTime(); this.finishedTime = app.getFinishTime(); @@ -454,6 +471,14 @@ public long getClusterId() { return this.clusterId; } + public SubClusterIdInfo getSubClusterIdInfo() { + return this.subClusterId; + } + + public String getRmClusterId() { + return this.rmClusterId; + } + public String getApplicationType() { return this.applicationType; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppTimeoutInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppTimeoutInfo.java index 140857e4063df..460309bc297f6 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppTimeoutInfo.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppTimeoutInfo.java @@ -22,6 +22,7 @@ import javax.xml.bind.annotation.XmlElement; import javax.xml.bind.annotation.XmlRootElement; +import org.apache.hadoop.yarn.api.records.ApplicationTimeout; import org.apache.hadoop.yarn.api.records.ApplicationTimeoutType; /** @@ -45,6 +46,12 @@ public AppTimeoutInfo() { remainingTimeInSec = -1; } + public AppTimeoutInfo(ApplicationTimeout applicationTimeout) { + this.expiryTime = applicationTimeout.getExpiryTime(); + this.remainingTimeInSec = applicationTimeout.getRemainingTime(); + this.timeoutType = applicationTimeout.getTimeoutType(); + } + public ApplicationTimeoutType getTimeoutType() { return timeoutType; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ApplicationStatisticsInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ApplicationStatisticsInfo.java index b77ffb4ed60f1..7b7066a731855 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ApplicationStatisticsInfo.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ApplicationStatisticsInfo.java @@ -18,6 +18,7 @@ package org.apache.hadoop.yarn.server.resourcemanager.webapp.dao; import java.util.ArrayList; +import java.util.Collection; import javax.xml.bind.annotation.XmlAccessType; import javax.xml.bind.annotation.XmlAccessorType; @@ -33,6 +34,10 @@ public class ApplicationStatisticsInfo { public ApplicationStatisticsInfo() { } // JAXB needs this + public ApplicationStatisticsInfo(Collection items) { + statItem.addAll(items); + } + public void add(StatisticsItemInfo statItem) { this.statItem.add(statItem); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/LabelsToNodesInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/LabelsToNodesInfo.java index e842d4245080d..d8849f0d105f4 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/LabelsToNodesInfo.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/LabelsToNodesInfo.java @@ -31,8 +31,7 @@ @XmlAccessorType(XmlAccessType.FIELD) public class LabelsToNodesInfo { - protected Map labelsToNodes = - new HashMap(); + private Map labelsToNodes = new HashMap<>(); public LabelsToNodesInfo() { } // JAXB needs this @@ -44,4 +43,8 @@ public LabelsToNodesInfo(Map labelsToNodes) { public Map getLabelsToNodes() { return labelsToNodes; } + + public void setLabelsToNodes(Map labelsToNodes) { + this.labelsToNodes = labelsToNodes; + } } \ No newline at end of file diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/NodeLabelsInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/NodeLabelsInfo.java index 2c3a8a507ab57..c9809b6d2e3cd 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/NodeLabelsInfo.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/NodeLabelsInfo.java @@ -32,8 +32,7 @@ public class NodeLabelsInfo { @XmlElement(name = "nodeLabelInfo") - private ArrayList nodeLabelsInfo = - new ArrayList(); + private ArrayList nodeLabelsInfo = new ArrayList<>(); public NodeLabelsInfo() { // JAXB needs this @@ -44,25 +43,32 @@ public NodeLabelsInfo(ArrayList nodeLabels) { } public NodeLabelsInfo(List nodeLabels) { - this.nodeLabelsInfo = new ArrayList(); + this.nodeLabelsInfo = new ArrayList<>(); for (NodeLabel label : nodeLabels) { this.nodeLabelsInfo.add(new NodeLabelInfo(label)); } } public NodeLabelsInfo(Set nodeLabelsName) { - this.nodeLabelsInfo = new ArrayList(); + this.nodeLabelsInfo = new ArrayList<>(); for (String labelName : nodeLabelsName) { this.nodeLabelsInfo.add(new NodeLabelInfo(labelName)); } } + public NodeLabelsInfo(Collection nodeLabels) { + this.nodeLabelsInfo = new ArrayList<>(); + nodeLabels.stream().forEach(nodeLabel -> { + this.nodeLabelsInfo.add(new NodeLabelInfo(nodeLabel)); + }); + } + public ArrayList getNodeLabelsInfo() { return nodeLabelsInfo; } public Set getNodeLabels() { - Set nodeLabels = new HashSet(); + Set nodeLabels = new HashSet<>(); for (NodeLabelInfo label : nodeLabelsInfo) { nodeLabels.add(NodeLabel.newInstance(label.getName(), label.getExclusivity())); @@ -71,7 +77,7 @@ public Set getNodeLabels() { } public List getNodeLabelsName() { - ArrayList nodeLabelsName = new ArrayList(); + ArrayList nodeLabelsName = new ArrayList<>(); for (NodeLabelInfo label : nodeLabelsInfo) { nodeLabelsName.add(label.getName()); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/NodeToLabelsInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/NodeToLabelsInfo.java index 0b6e4bc868c74..e9044b9397cfc 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/NodeToLabelsInfo.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/NodeToLabelsInfo.java @@ -35,7 +35,17 @@ public NodeToLabelsInfo() { // JAXB needs this } + public NodeToLabelsInfo(HashMap nodeToLabels) { + if (nodeToLabels != null) { + this.nodeToLabels.putAll(nodeToLabels); + } + } + public HashMap getNodeToLabels() { return nodeToLabels; } + + public void setNodeToLabels(HashMap nodeToLabels) { + this.nodeToLabels = nodeToLabels; + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/PartitionInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/PartitionInfo.java index 95baf74a18621..c0742b1cfaad4 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/PartitionInfo.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/PartitionInfo.java @@ -18,6 +18,9 @@ package org.apache.hadoop.yarn.server.resourcemanager.webapp.dao; +import org.apache.hadoop.yarn.api.records.Resource; +import org.apache.hadoop.yarn.util.resource.Resources; + import javax.xml.bind.annotation.XmlAccessType; import javax.xml.bind.annotation.XmlAccessorType; import javax.xml.bind.annotation.XmlElement; @@ -43,4 +46,29 @@ public PartitionInfo(ResourceInfo resourceAvailable) { public ResourceInfo getResourceAvailable() { return resourceAvailable; } + + /** + * This method will generate a new PartitionInfo object based on two PartitionInfo objects. + * The combination process is mainly based on the Resources.add method. + * + * @param left left PartitionInfo Object. + * @param right right PartitionInfo Object. + * @return new PartitionInfo Object. + */ + public static PartitionInfo addTo(PartitionInfo left, PartitionInfo right) { + Resource leftResource = Resource.newInstance(0, 0); + if (left != null && left.getResourceAvailable() != null) { + ResourceInfo leftResourceInfo = left.getResourceAvailable(); + leftResource = leftResourceInfo.getResource(); + } + + Resource rightResource = Resource.newInstance(0, 0); + if (right != null && right.getResourceAvailable() != null) { + ResourceInfo rightResourceInfo = right.getResourceAvailable(); + rightResource = rightResourceInfo.getResource(); + } + + Resource resource = Resources.addTo(leftResource, rightResource); + return new PartitionInfo(new ResourceInfo(resource)); + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/StatisticsItemInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/StatisticsItemInfo.java index e12dd5fb63b9e..8f79fef0ce8f8 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/StatisticsItemInfo.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/StatisticsItemInfo.java @@ -41,6 +41,12 @@ public StatisticsItemInfo( this.count = count; } + public StatisticsItemInfo(StatisticsItemInfo info) { + this.state = info.state; + this.type = info.type; + this.count = info.count; + } + public YarnApplicationState getState() { return state; } @@ -53,4 +59,7 @@ public long getCount() { return count; } + public void setCount(long count) { + this.count = count; + } } \ No newline at end of file diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java index 0aac2f1399a94..95db1171b35e3 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java @@ -20,6 +20,7 @@ import static org.apache.hadoop.yarn.server.resourcemanager.MockNM.createMockNodeStatus; +import org.apache.hadoop.classification.VisibleForTesting; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem; import org.apache.hadoop.security.token.Token; @@ -64,6 +65,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.recovery.MemoryRMStateStore; import org.apache.hadoop.yarn.server.resourcemanager.recovery.NullRMStateStore; import org.apache.hadoop.yarn.server.resourcemanager.recovery.RMStateStore; +import org.apache.hadoop.yarn.server.resourcemanager.reservation.ReservationSystem; import org.apache.hadoop.yarn.server.resourcemanager.resource.TestResourceProfiles; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppState; @@ -1035,4 +1037,9 @@ protected void serviceInit(Configuration conf) throws Exception { public RMStateStore getRMStateStore() { return getRMContext().getStateStore(); } + + @VisibleForTesting + public ReservationSystem getReservationSystem(){ + return this.reservationSystem; + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMTokens.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMTokens.java index 556fd5bdf00d8..90ba812632854 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMTokens.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMTokens.java @@ -34,6 +34,7 @@ import java.net.InetSocketAddress; import java.security.PrivilegedAction; import java.security.PrivilegedExceptionAction; +import java.util.concurrent.TimeUnit; import org.apache.hadoop.test.LambdaTestUtils; import org.apache.hadoop.thirdparty.protobuf.InvalidProtocolBufferException; @@ -124,9 +125,13 @@ public void testDelegationToken() throws Exception { long initialInterval = 10000l; long maxLifetime= 20000l; long renewInterval = 10000l; + long delegationTokenRemoverScanInterval = + conf.getTimeDuration(YarnConfiguration.RM_DELEGATION_TOKEN_REMOVE_SCAN_INTERVAL_KEY, + YarnConfiguration.RM_DELEGATION_TOKEN_REMOVE_SCAN_INTERVAL_DEFAULT, + TimeUnit.MILLISECONDS); RMDelegationTokenSecretManager rmDtSecretManager = createRMDelegationTokenSecretManager( - initialInterval, maxLifetime, renewInterval); + initialInterval, maxLifetime, renewInterval, delegationTokenRemoverScanInterval); rmDtSecretManager.startThreads(); LOG.info("Creating DelegationTokenSecretManager with initialInterval: " + initialInterval + ", maxLifetime: " + maxLifetime @@ -574,7 +579,8 @@ private static ResourceScheduler createMockScheduler(Configuration conf) { private static RMDelegationTokenSecretManager createRMDelegationTokenSecretManager(long secretKeyInterval, - long tokenMaxLifetime, long tokenRenewInterval) { + long tokenMaxLifetime, long tokenRenewInterval, + long delegationTokenRemoverScanInterval) { ResourceManager rm = mock(ResourceManager.class); RMContext rmContext = mock(RMContext.class); when(rmContext.getStateStore()).thenReturn(new NullRMStateStore()); @@ -583,7 +589,7 @@ private static ResourceScheduler createMockScheduler(Configuration conf) { RMDelegationTokenSecretManager rmDtSecretManager = new RMDelegationTokenSecretManager(secretKeyInterval, tokenMaxLifetime, - tokenRenewInterval, 3600000, rmContext); + tokenRenewInterval, delegationTokenRemoverScanInterval, rmContext); return rmDtSecretManager; } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMNodeTransitions.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMNodeTransitions.java index db14d422f6eb5..3a0288715075e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMNodeTransitions.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMNodeTransitions.java @@ -55,6 +55,7 @@ import org.apache.hadoop.yarn.nodelabels.CommonNodeLabelsManager; import org.apache.hadoop.yarn.server.api.protocolrecords.NMContainerStatus; import org.apache.hadoop.yarn.server.api.protocolrecords.NodeHeartbeatResponse; +import org.apache.hadoop.yarn.server.api.records.NodeAction; import org.apache.hadoop.yarn.server.api.records.NodeHealthStatus; import org.apache.hadoop.yarn.server.api.records.NodeStatus; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp; @@ -82,6 +83,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.SchedulerEventType; import org.apache.hadoop.yarn.server.resourcemanager.security.DelegationTokenRenewer; import org.apache.hadoop.yarn.server.utils.BuilderUtils; +import org.apache.hadoop.yarn.server.utils.YarnServerBuilderUtils; import org.apache.hadoop.yarn.util.Records; import org.apache.hadoop.yarn.util.resource.Resources; import org.junit.After; @@ -1502,4 +1504,27 @@ public void testCalculateHeartBeatInterval() { calcIntervalTest(rmNode, nodeUtil, hbDefault, hbMin, hbMax, speedup, slowdown, vcoreUnit * 1.0F, hbDefault); // 100% } + + @Test + public void testFinishedContainersPulledByAmOnDecommissioningNode() { + RMNodeImpl rMNodeImpl = getRunningNode(); + rMNodeImpl.handle( + new RMNodeEvent(rMNodeImpl.getNodeID(), RMNodeEventType.GRACEFUL_DECOMMISSION)); + Assert.assertEquals(NodeState.DECOMMISSIONING, rMNodeImpl.getState()); + + ContainerId containerId = BuilderUtils.newContainerId( + BuilderUtils.newApplicationAttemptId(BuilderUtils.newApplicationId(0, 0), 0), 0); + List containerIds = Arrays.asList(containerId); + + rMNodeImpl.handle( + new RMNodeFinishedContainersPulledByAMEvent(rMNodeImpl.getNodeID(), containerIds)); + Assert.assertEquals(NodeState.DECOMMISSIONING, rMNodeImpl.getState()); + + // Verify expected containersToBeRemovedFromNM from NodeHeartbeatResponse. + NodeHeartbeatResponse response = + YarnServerBuilderUtils.newNodeHeartbeatResponse(1, NodeAction.NORMAL, null, null, null, + null, 1000); + rMNodeImpl.setAndUpdateNodeHeartbeatResponse(response); + Assert.assertEquals(1, response.getContainersToBeRemovedFromNM().size()); + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/federation/TestFederationRMStateStoreService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/federation/TestFederationRMStateStoreService.java index e5e156dcf76c6..e8ebdd5bedde5 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/federation/TestFederationRMStateStoreService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/federation/TestFederationRMStateStoreService.java @@ -25,6 +25,7 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.ha.HAServiceProtocol; +import org.apache.hadoop.test.GenericTestUtils; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.server.federation.store.FederationStateStore; @@ -173,4 +174,37 @@ private String checkSubClusterInfo(SubClusterState state) return response.getCapability(); } + @Test + public void testFederationStateStoreServiceInitialHeartbeatDelay() throws Exception { + conf.setBoolean(YarnConfiguration.FEDERATION_ENABLED, true); + conf.setInt(YarnConfiguration.FEDERATION_STATESTORE_HEARTBEAT_INITIAL_DELAY, 10); + conf.set(YarnConfiguration.RM_CLUSTER_ID, subClusterId.getId()); + + GenericTestUtils.LogCapturer logCapture = + GenericTestUtils.LogCapturer.captureLogs(FederationStateStoreService.LOG); + + final MockRM rm = new MockRM(conf); + + // Initially there should be no entry for the sub-cluster + rm.init(conf); + stateStore = rm.getFederationStateStoreService().getStateStoreClient(); + GetSubClusterInfoResponse response = stateStore.getSubCluster(request); + Assert.assertNull(response); + + // Validate if sub-cluster is registered + rm.start(); + String capability = checkSubClusterInfo(SubClusterState.SC_NEW); + Assert.assertTrue(capability.isEmpty()); + + // Heartbeat to see if sub-cluster transitions to running + FederationStateStoreHeartbeat storeHeartbeat = + rm.getFederationStateStoreService().getStateStoreHeartbeatThread(); + storeHeartbeat.run(); + capability = checkSubClusterInfo(SubClusterState.SC_RUNNING); + checkClusterMetricsInfo(capability, 0); + + Assert.assertTrue(logCapture.getOutput().contains( + "Started federation membership heartbeat with interval: 300 and initial delay: 10")); + rm.stop(); + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestAutoCreatedQueueTemplate.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestAutoCreatedQueueTemplate.java index e73001b7e5e14..201214ec2674b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestAutoCreatedQueueTemplate.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestAutoCreatedQueueTemplate.java @@ -66,6 +66,22 @@ public void testOneLevelWildcardTemplate() { } + @Test + public void testTwoLevelWildcardTemplate() { + conf.set(getTemplateKey("root.*", "capacity"), "6w"); + conf.set(getTemplateKey("root.*.*", "capacity"), "5w"); + + new AutoCreatedQueueTemplate(conf, TEST_QUEUE_A) + .setTemplateEntriesForChild(conf, TEST_QUEUE_AB.getFullPath()); + new AutoCreatedQueueTemplate(conf, TEST_QUEUE_AB) + .setTemplateEntriesForChild(conf, TEST_QUEUE_ABC.getFullPath()); + + Assert.assertEquals("weight is not set", 6f, + conf.getNonLabeledQueueWeight(TEST_QUEUE_AB.getFullPath()), 10e-6); + Assert.assertEquals("weight is not set", 5f, + conf.getNonLabeledQueueWeight(TEST_QUEUE_ABC.getFullPath()), 10e-6); + } + @Test public void testIgnoredWhenRootWildcarded() { conf.set(getTemplateKey("*", "capacity"), "6w"); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java index 4c859baf78906..b304b8a7857b0 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java @@ -103,6 +103,7 @@ protected void configureServlets() { YarnConfiguration.DEFAULT_RM_AM_MAX_ATTEMPTS); conf.setClass(YarnConfiguration.RM_SCHEDULER, scheduler, ResourceScheduler.class); + conf.set(YarnConfiguration.RM_CLUSTER_ID, "subCluster1"); rm = new MockRM(conf); bind(ResourceManager.class).toInstance(rm); serve("/*").with(GuiceContainer.class); @@ -1805,7 +1806,7 @@ public void verifyAppsXML(NodeList nodes, RMApp app, boolean hasResourceReq) public void verifyAppInfo(JSONObject info, RMApp app, boolean hasResourceReqs) throws JSONException, Exception { - int expectedNumberOfElements = 40 + (hasResourceReqs ? 2 : 0); + int expectedNumberOfElements = 41 + (hasResourceReqs ? 2 : 0); String appNodeLabelExpression = null; String amNodeLabelExpression = null; if (app.getApplicationSubmissionContext() @@ -1825,6 +1826,7 @@ public void verifyAppInfo(JSONObject info, RMApp app, boolean hasResourceReqs) } assertEquals("incorrect number of elements", expectedNumberOfElements, info.length()); + assertEquals("rmClusterId is incorrect", "subCluster1", info.getString("rmClusterId")); verifyAppInfoGeneric(app, info.getString("id"), info.getString("user"), info.getString("name"), info.getString("applicationType"), info.getString("queue"), info.getInt("priority"), diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/pom.xml index 8ef7b62f24f4f..aa808801df4e8 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/pom.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/pom.xml @@ -117,8 +117,16 @@ - javax.ws.rs - javax.ws.rs-api + org.apache.hadoop + hadoop-minikdc + test + + + + org.apache.hadoop + hadoop-auth + test + test-jar diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/Router.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/Router.java index fc0b9eeaef3cc..e95b25678bf8a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/Router.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/Router.java @@ -19,12 +19,15 @@ package org.apache.hadoop.yarn.server.router; import java.io.IOException; +import java.net.InetAddress; +import java.net.UnknownHostException; import java.util.concurrent.atomic.AtomicBoolean; import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem; import org.apache.hadoop.metrics2.source.JvmMetrics; +import org.apache.hadoop.security.SecurityUtil; import org.apache.hadoop.service.CompositeService; import org.apache.hadoop.util.JvmPauseMonitor; import org.apache.hadoop.util.ShutdownHookManager; @@ -88,7 +91,8 @@ public Router() { } protected void doSecureLogin() throws IOException { - // TODO YARN-6539 Create SecureLogin inside Router + SecurityUtil.login(this.conf, YarnConfiguration.ROUTER_KEYTAB, + YarnConfiguration.ROUTER_PRINCIPAL, getHostName(this.conf)); } @Override @@ -195,4 +199,31 @@ public static void main(String[] argv) { System.exit(-1); } } + + @VisibleForTesting + public RouterClientRMService getClientRMProxyService() { + return clientRMProxyService; + } + + @VisibleForTesting + public RouterRMAdminService getRmAdminProxyService() { + return rmAdminProxyService; + } + + /** + * Returns the hostname for this Router. If the hostname is not + * explicitly configured in the given config, then it is determined. + * + * @param config configuration + * @return the hostname (NB: may not be a FQDN) + * @throws UnknownHostException if the hostname cannot be determined + */ + private String getHostName(Configuration config) + throws UnknownHostException { + String name = config.get(YarnConfiguration.ROUTER_KERBEROS_PRINCIPAL_HOSTNAME_KEY); + if (name == null) { + name = InetAddress.getLocalHost().getHostName(); + } + return name; + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/RouterAuditLogger.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/RouterAuditLogger.java index cd5b0c95d5aaf..a89d0e4462aa2 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/RouterAuditLogger.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/RouterAuditLogger.java @@ -18,11 +18,14 @@ package org.apache.hadoop.yarn.server.router; +import org.apache.hadoop.ipc.Server; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.server.federation.store.records.SubClusterId; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.net.InetAddress; + /** * Manages Router audit logs. * Audit log format is written as key=value pairs. Tab separated. @@ -46,6 +49,8 @@ public static class AuditConstants { public static final String SUBMIT_NEW_APP = "Submit New App"; public static final String FORCE_KILL_APP = "Force Kill App"; public static final String GET_APP_REPORT = "Get Application Report"; + public static final String TARGET_CLIENT_RM_SERVICE = "RouterClientRMService"; + public static final String UNKNOWN = "UNKNOWN"; } /** @@ -111,6 +116,7 @@ private static StringBuilder createStringBuilderForSuccessEvent(String user, String operation, String target) { StringBuilder b = new StringBuilder(); start(Keys.USER, user, b); + addRemoteIP(b); add(Keys.OPERATION, operation, b); add(Keys.TARGET, target, b); add(Keys.RESULT, AuditConstants.SUCCESS, b); @@ -216,6 +222,7 @@ private static StringBuilder createStringBuilderForFailureLog(String user, String operation, String target, String description, String perm) { StringBuilder b = new StringBuilder(); start(Keys.USER, user, b); + addRemoteIP(b); add(Keys.OPERATION, operation, b); add(Keys.TARGET, target, b); add(Keys.RESULT, AuditConstants.FAILURE, b); @@ -240,4 +247,15 @@ static void add(Keys key, String value, StringBuilder b) { b.append(AuditConstants.PAIR_SEPARATOR).append(key.name()) .append(AuditConstants.KEY_VAL_SEPARATOR).append(value); } + + /** + * A helper api to add remote IP address. + */ + static void addRemoteIP(StringBuilder b) { + InetAddress ip = Server.getRemoteIp(); + // ip address can be null for testcases + if (ip != null) { + add(Keys.IP, ip.getHostAddress(), b); + } + } } \ No newline at end of file diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/RouterMetrics.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/RouterMetrics.java index ac37c4ed1b9e6..712072cc27923 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/RouterMetrics.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/RouterMetrics.java @@ -71,7 +71,7 @@ public final class RouterMetrics { private MutableGaugeInt numGetContainerReportFailedRetrieved; @Metric("# of getContainers failed to be retrieved") private MutableGaugeInt numGetContainersFailedRetrieved; - @Metric("# of getContainers failed to be retrieved") + @Metric("# of listReservations failed to be retrieved") private MutableGaugeInt numListReservationsFailedRetrieved; @Metric("# of getResourceTypeInfo failed to be retrieved") private MutableGaugeInt numGetResourceTypeInfo; @@ -83,6 +83,30 @@ public final class RouterMetrics { private MutableGaugeInt numUpdateAppTimeoutsFailedRetrieved; @Metric("# of signalToContainer failed to be retrieved") private MutableGaugeInt numSignalToContainerFailedRetrieved; + @Metric("# of getQueueInfo failed to be retrieved") + private MutableGaugeInt numGetQueueInfoFailedRetrieved; + @Metric("# of moveApplicationAcrossQueues failed to be retrieved") + private MutableGaugeInt numMoveApplicationAcrossQueuesFailedRetrieved; + @Metric("# of getResourceProfiles failed to be retrieved") + private MutableGaugeInt numGetResourceProfilesFailedRetrieved; + @Metric("# of getResourceProfile failed to be retrieved") + private MutableGaugeInt numGetResourceProfileFailedRetrieved; + @Metric("# of getAttributesToNodes failed to be retrieved") + private MutableGaugeInt numGetAttributesToNodesFailedRetrieved; + @Metric("# of getClusterNodeAttributes failed to be retrieved") + private MutableGaugeInt numGetClusterNodeAttributesFailedRetrieved; + @Metric("# of getNodesToAttributes failed to be retrieved") + private MutableGaugeInt numGetNodesToAttributesFailedRetrieved; + @Metric("# of getNewReservation failed to be retrieved") + private MutableGaugeInt numGetNewReservationFailedRetrieved; + @Metric("# of submitReservation failed to be retrieved") + private MutableGaugeInt numSubmitReservationFailedRetrieved; + @Metric("# of submitReservation failed to be retrieved") + private MutableGaugeInt numUpdateReservationFailedRetrieved; + @Metric("# of deleteReservation failed to be retrieved") + private MutableGaugeInt numDeleteReservationFailedRetrieved; + @Metric("# of listReservation failed to be retrieved") + private MutableGaugeInt numListReservationFailedRetrieved; // Aggregate metrics are shared, and don't have to be looked up per call @Metric("Total number of successful Submitted apps and latency(ms)") @@ -93,14 +117,11 @@ public final class RouterMetrics { private MutableRate totalSucceededAppsCreated; @Metric("Total number of successful Retrieved app reports and latency(ms)") private MutableRate totalSucceededAppsRetrieved; - @Metric("Total number of successful Retrieved multiple apps reports and " - + "latency(ms)") + @Metric("Total number of successful Retrieved multiple apps reports and latency(ms)") private MutableRate totalSucceededMultipleAppsRetrieved; - @Metric("Total number of successful Retrieved " + - "appAttempt reports and latency(ms)") + @Metric("Total number of successful Retrieved appAttempt reports and latency(ms)") private MutableRate totalSucceededAppAttemptsRetrieved; - @Metric("Total number of successful Retrieved getClusterMetrics and " - + "latency(ms)") + @Metric("Total number of successful Retrieved getClusterMetrics and latency(ms)") private MutableRate totalSucceededGetClusterMetricsRetrieved; @Metric("Total number of successful Retrieved getClusterNodes and latency(ms)") private MutableRate totalSucceededGetClusterNodesRetrieved; @@ -130,6 +151,30 @@ public final class RouterMetrics { private MutableRate totalSucceededUpdateAppTimeoutsRetrieved; @Metric("Total number of successful Retrieved signalToContainer and latency(ms)") private MutableRate totalSucceededSignalToContainerRetrieved; + @Metric("Total number of successful Retrieved getQueueInfo and latency(ms)") + private MutableRate totalSucceededGetQueueInfoRetrieved; + @Metric("Total number of successful Retrieved moveApplicationAcrossQueues and latency(ms)") + private MutableRate totalSucceededMoveApplicationAcrossQueuesRetrieved; + @Metric("Total number of successful Retrieved getResourceProfiles and latency(ms)") + private MutableRate totalSucceededGetResourceProfilesRetrieved; + @Metric("Total number of successful Retrieved getResourceProfile and latency(ms)") + private MutableRate totalSucceededGetResourceProfileRetrieved; + @Metric("Total number of successful Retrieved getAttributesToNodes and latency(ms)") + private MutableRate totalSucceededGetAttributesToNodesRetrieved; + @Metric("Total number of successful Retrieved getClusterNodeAttributes and latency(ms)") + private MutableRate totalSucceededGetClusterNodeAttributesRetrieved; + @Metric("Total number of successful Retrieved getNodesToAttributes and latency(ms)") + private MutableRate totalSucceededGetNodesToAttributesRetrieved; + @Metric("Total number of successful Retrieved GetNewReservation and latency(ms)") + private MutableRate totalSucceededGetNewReservationRetrieved; + @Metric("Total number of successful Retrieved SubmitReservation and latency(ms)") + private MutableRate totalSucceededSubmitReservationRetrieved; + @Metric("Total number of successful Retrieved UpdateReservation and latency(ms)") + private MutableRate totalSucceededUpdateReservationRetrieved; + @Metric("Total number of successful Retrieved DeleteReservation and latency(ms)") + private MutableRate totalSucceededDeleteReservationRetrieved; + @Metric("Total number of successful Retrieved ListReservation and latency(ms)") + private MutableRate totalSucceededListReservationRetrieved; /** * Provide quantile counters for all latencies. @@ -155,6 +200,18 @@ public final class RouterMetrics { private MutableQuantiles updateAppPriorityLatency; private MutableQuantiles updateAppTimeoutsLatency; private MutableQuantiles signalToContainerLatency; + private MutableQuantiles getQueueInfoLatency; + private MutableQuantiles moveApplicationAcrossQueuesLatency; + private MutableQuantiles getResourceProfilesLatency; + private MutableQuantiles getResourceProfileLatency; + private MutableQuantiles getAttributesToNodesLatency; + private MutableQuantiles getClusterNodeAttributesLatency; + private MutableQuantiles getNodesToAttributesLatency; + private MutableQuantiles getNewReservationLatency; + private MutableQuantiles submitReservationLatency; + private MutableQuantiles updateReservationLatency; + private MutableQuantiles deleteReservationLatency; + private MutableQuantiles listReservationLatency; private static volatile RouterMetrics instance = null; private static MetricsRegistry registry; @@ -237,6 +294,54 @@ private RouterMetrics() { signalToContainerLatency = registry.newQuantiles("signalToContainerLatency", "latency of signal to container timeouts", "ops", "latency", 10); + + getQueueInfoLatency = + registry.newQuantiles("getQueueInfoLatency", + "latency of get queue info timeouts", "ops", "latency", 10); + + moveApplicationAcrossQueuesLatency = + registry.newQuantiles("moveApplicationAcrossQueuesLatency", + "latency of move application across queues timeouts", "ops", "latency", 10); + + getResourceProfilesLatency = + registry.newQuantiles("getResourceProfilesLatency", + "latency of get resource profiles timeouts", "ops", "latency", 10); + + getResourceProfileLatency = + registry.newQuantiles("getResourceProfileLatency", + "latency of get resource profile timeouts", "ops", "latency", 10); + + getAttributesToNodesLatency = + registry.newQuantiles("getAttributesToNodesLatency", + "latency of get attributes to nodes timeouts", "ops", "latency", 10); + + getClusterNodeAttributesLatency = + registry.newQuantiles("getClusterNodeAttributesLatency", + "latency of get cluster node attributes timeouts", "ops", "latency", 10); + + getNodesToAttributesLatency = + registry.newQuantiles("getNodesToAttributesLatency", + "latency of get nodes to attributes timeouts", "ops", "latency", 10); + + getNewReservationLatency = + registry.newQuantiles("getNewReservationLatency", + "latency of get new reservation timeouts", "ops", "latency", 10); + + submitReservationLatency = + registry.newQuantiles("submitReservationLatency", + "latency of submit reservation timeouts", "ops", "latency", 10); + + updateReservationLatency = + registry.newQuantiles("updateReservationLatency", + "latency of update reservation timeouts", "ops", "latency", 10); + + deleteReservationLatency = + registry.newQuantiles("deleteReservationLatency", + "latency of delete reservation timeouts", "ops", "latency", 10); + + listReservationLatency = + registry.newQuantiles("listReservationLatency", + "latency of list reservation timeouts", "ops", "latency", 10); } public static RouterMetrics getMetrics() { @@ -363,6 +468,66 @@ public long getNumSucceededSignalToContainerRetrieved() { return totalSucceededSignalToContainerRetrieved.lastStat().numSamples(); } + @VisibleForTesting + public long getNumSucceededGetQueueInfoRetrieved() { + return totalSucceededGetQueueInfoRetrieved.lastStat().numSamples(); + } + + @VisibleForTesting + public long getNumSucceededMoveApplicationAcrossQueuesRetrieved() { + return totalSucceededMoveApplicationAcrossQueuesRetrieved.lastStat().numSamples(); + } + + @VisibleForTesting + public long getNumSucceededGetResourceProfilesRetrieved() { + return totalSucceededGetResourceProfilesRetrieved.lastStat().numSamples(); + } + + @VisibleForTesting + public long getNumSucceededGetResourceProfileRetrieved() { + return totalSucceededGetResourceProfileRetrieved.lastStat().numSamples(); + } + + @VisibleForTesting + public long getNumSucceededGetAttributesToNodesRetrieved() { + return totalSucceededGetAttributesToNodesRetrieved.lastStat().numSamples(); + } + + @VisibleForTesting + public long getNumSucceededGetClusterNodeAttributesRetrieved() { + return totalSucceededGetClusterNodeAttributesRetrieved.lastStat().numSamples(); + } + + @VisibleForTesting + public long getNumSucceededGetNodesToAttributesRetrieved() { + return totalSucceededGetNodesToAttributesRetrieved.lastStat().numSamples(); + } + + @VisibleForTesting + public long getNumSucceededGetNewReservationRetrieved() { + return totalSucceededGetNewReservationRetrieved.lastStat().numSamples(); + } + + @VisibleForTesting + public long getNumSucceededSubmitReservationRetrieved() { + return totalSucceededSubmitReservationRetrieved.lastStat().numSamples(); + } + + @VisibleForTesting + public long getNumSucceededUpdateReservationRetrieved() { + return totalSucceededUpdateReservationRetrieved.lastStat().numSamples(); + } + + @VisibleForTesting + public long getNumSucceededDeleteReservationRetrieved() { + return totalSucceededDeleteReservationRetrieved.lastStat().numSamples(); + } + + @VisibleForTesting + public long getNumSucceededListReservationRetrieved() { + return totalSucceededListReservationRetrieved.lastStat().numSamples(); + } + @VisibleForTesting public double getLatencySucceededAppsCreated() { return totalSucceededAppsCreated.lastStat().mean(); @@ -468,6 +633,66 @@ public double getLatencySucceededSignalToContainerRetrieved() { return totalSucceededSignalToContainerRetrieved.lastStat().mean(); } + @VisibleForTesting + public double getLatencySucceededGetQueueInfoRetrieved() { + return totalSucceededGetQueueInfoRetrieved.lastStat().mean(); + } + + @VisibleForTesting + public double getLatencySucceededMoveApplicationAcrossQueuesRetrieved() { + return totalSucceededMoveApplicationAcrossQueuesRetrieved.lastStat().mean(); + } + + @VisibleForTesting + public double getLatencySucceededGetResourceProfilesRetrieved() { + return totalSucceededGetResourceProfilesRetrieved.lastStat().mean(); + } + + @VisibleForTesting + public double getLatencySucceededGetResourceProfileRetrieved() { + return totalSucceededGetResourceProfileRetrieved.lastStat().mean(); + } + + @VisibleForTesting + public double getLatencySucceededGetAttributesToNodesRetrieved() { + return totalSucceededGetAttributesToNodesRetrieved.lastStat().mean(); + } + + @VisibleForTesting + public double getLatencySucceededGetClusterNodeAttributesRetrieved() { + return totalSucceededGetClusterNodeAttributesRetrieved.lastStat().mean(); + } + + @VisibleForTesting + public double getLatencySucceededGetNodesToAttributesRetrieved() { + return totalSucceededGetNodesToAttributesRetrieved.lastStat().mean(); + } + + @VisibleForTesting + public double getLatencySucceededGetNewReservationRetrieved() { + return totalSucceededGetNewReservationRetrieved.lastStat().mean(); + } + + @VisibleForTesting + public double getLatencySucceededSubmitReservationRetrieved() { + return totalSucceededSubmitReservationRetrieved.lastStat().mean(); + } + + @VisibleForTesting + public double getLatencySucceededUpdateReservationRetrieved() { + return totalSucceededUpdateReservationRetrieved.lastStat().mean(); + } + + @VisibleForTesting + public double getLatencySucceededDeleteReservationRetrieved() { + return totalSucceededDeleteReservationRetrieved.lastStat().mean(); + } + + @VisibleForTesting + public double getLatencySucceededListReservationRetrieved() { + return totalSucceededListReservationRetrieved.lastStat().mean(); + } + @VisibleForTesting public int getAppsFailedCreated() { return numAppsFailedCreated.value(); @@ -573,6 +798,54 @@ public int getSignalToContainerFailedRetrieved() { return numSignalToContainerFailedRetrieved.value(); } + public int getQueueInfoFailedRetrieved() { + return numGetQueueInfoFailedRetrieved.value(); + } + + public int getMoveApplicationAcrossQueuesFailedRetrieved() { + return numMoveApplicationAcrossQueuesFailedRetrieved.value(); + } + + public int getResourceProfilesFailedRetrieved() { + return numGetResourceProfilesFailedRetrieved.value(); + } + + public int getResourceProfileFailedRetrieved() { + return numGetResourceProfileFailedRetrieved.value(); + } + + public int getAttributesToNodesFailedRetrieved() { + return numGetAttributesToNodesFailedRetrieved.value(); + } + + public int getClusterNodeAttributesFailedRetrieved() { + return numGetClusterNodeAttributesFailedRetrieved.value(); + } + + public int getNodesToAttributesFailedRetrieved() { + return numGetNodesToAttributesFailedRetrieved.value(); + } + + public int getNewReservationFailedRetrieved() { + return numGetNewReservationFailedRetrieved.value(); + } + + public int getSubmitReservationFailedRetrieved() { + return numSubmitReservationFailedRetrieved.value(); + } + + public int getUpdateReservationFailedRetrieved() { + return numUpdateReservationFailedRetrieved.value(); + } + + public int getDeleteReservationFailedRetrieved() { + return numDeleteReservationFailedRetrieved.value(); + } + + public int getListReservationFailedRetrieved() { + return numListReservationFailedRetrieved.value(); + } + public void succeededAppsCreated(long duration) { totalSucceededAppsCreated.add(duration); getNewApplicationLatency.add(duration); @@ -678,6 +951,66 @@ public void succeededSignalToContainerRetrieved(long duration) { signalToContainerLatency.add(duration); } + public void succeededGetQueueInfoRetrieved(long duration) { + totalSucceededGetQueueInfoRetrieved.add(duration); + getQueueInfoLatency.add(duration); + } + + public void succeededMoveApplicationAcrossQueuesRetrieved(long duration) { + totalSucceededMoveApplicationAcrossQueuesRetrieved.add(duration); + moveApplicationAcrossQueuesLatency.add(duration); + } + + public void succeededGetResourceProfilesRetrieved(long duration) { + totalSucceededGetResourceProfilesRetrieved.add(duration); + getResourceProfilesLatency.add(duration); + } + + public void succeededGetResourceProfileRetrieved(long duration) { + totalSucceededGetResourceProfileRetrieved.add(duration); + getResourceProfileLatency.add(duration); + } + + public void succeededGetAttributesToNodesRetrieved(long duration) { + totalSucceededGetAttributesToNodesRetrieved.add(duration); + getAttributesToNodesLatency.add(duration); + } + + public void succeededGetClusterNodeAttributesRetrieved(long duration) { + totalSucceededGetClusterNodeAttributesRetrieved.add(duration); + getClusterNodeAttributesLatency.add(duration); + } + + public void succeededGetNodesToAttributesRetrieved(long duration) { + totalSucceededGetNodesToAttributesRetrieved.add(duration); + getNodesToAttributesLatency.add(duration); + } + + public void succeededGetNewReservationRetrieved(long duration) { + totalSucceededGetNewReservationRetrieved.add(duration); + getNewReservationLatency.add(duration); + } + + public void succeededSubmitReservationRetrieved(long duration) { + totalSucceededSubmitReservationRetrieved.add(duration); + submitReservationLatency.add(duration); + } + + public void succeededUpdateReservationRetrieved(long duration) { + totalSucceededUpdateReservationRetrieved.add(duration); + updateReservationLatency.add(duration); + } + + public void succeededDeleteReservationRetrieved(long duration) { + totalSucceededDeleteReservationRetrieved.add(duration); + deleteReservationLatency.add(duration); + } + + public void succeededListReservationRetrieved(long duration) { + totalSucceededListReservationRetrieved.add(duration); + listReservationLatency.add(duration); + } + public void incrAppsFailedCreated() { numAppsFailedCreated.incr(); } @@ -761,4 +1094,52 @@ public void incrUpdateApplicationTimeoutsRetrieved() { public void incrSignalToContainerFailedRetrieved() { numSignalToContainerFailedRetrieved.incr(); } + + public void incrGetQueueInfoFailedRetrieved() { + numGetQueueInfoFailedRetrieved.incr(); + } + + public void incrMoveApplicationAcrossQueuesFailedRetrieved() { + numMoveApplicationAcrossQueuesFailedRetrieved.incr(); + } + + public void incrGetResourceProfilesFailedRetrieved() { + numGetResourceProfilesFailedRetrieved.incr(); + } + + public void incrGetResourceProfileFailedRetrieved() { + numGetResourceProfileFailedRetrieved.incr(); + } + + public void incrGetAttributesToNodesFailedRetrieved() { + numGetAttributesToNodesFailedRetrieved.incr(); + } + + public void incrGetClusterNodeAttributesFailedRetrieved() { + numGetClusterNodeAttributesFailedRetrieved.incr(); + } + + public void incrGetNodesToAttributesFailedRetrieved() { + numGetNodesToAttributesFailedRetrieved.incr(); + } + + public void incrGetNewReservationFailedRetrieved() { + numGetNewReservationFailedRetrieved.incr(); + } + + public void incrSubmitReservationFailedRetrieved() { + numSubmitReservationFailedRetrieved.incr(); + } + + public void incrUpdateReservationFailedRetrieved() { + numUpdateReservationFailedRetrieved.incr(); + } + + public void incrDeleteReservationFailedRetrieved() { + numDeleteReservationFailedRetrieved.incr(); + } + + public void incrListReservationFailedRetrieved() { + numListReservationFailedRetrieved.incr(); + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/RouterServerUtil.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/RouterServerUtil.java index cc96da62331b9..36f02dd3e8a6d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/RouterServerUtil.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/RouterServerUtil.java @@ -21,10 +21,21 @@ import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.classification.InterfaceAudience.Public; import org.apache.hadoop.classification.InterfaceStability.Unstable; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.util.ReflectionUtils; +import org.apache.hadoop.util.StringUtils; import org.apache.hadoop.yarn.exceptions.YarnException; +import org.apache.hadoop.yarn.exceptions.YarnRuntimeException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Method; +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; +import java.io.IOException; + /** * Common utility methods used by the Router server. * @@ -40,6 +51,28 @@ private RouterServerUtil() { public static final Logger LOG = LoggerFactory.getLogger(RouterServerUtil.class); + /** + * Throws an exception due to an error. + * + * @param t the throwable raised in the called class. + * @param errMsgFormat the error message format string. + * @param args referenced by the format specifiers in the format string. + * @throws YarnException on failure + */ + @Public + @Unstable + public static void logAndThrowException(Throwable t, String errMsgFormat, Object... args) + throws YarnException { + String msg = String.format(errMsgFormat, args); + if (t != null) { + LOG.error(msg, t); + throw new YarnException(msg, t); + } else { + LOG.error(msg); + throw new YarnException(msg); + } + } + /** * Throws an exception due to an error. * @@ -60,4 +93,133 @@ public static void logAndThrowException(String errMsg, Throwable t) } } + public static R createRequestInterceptorChain(Configuration conf, String pipeLineClassName, + String interceptorClassName, Class clazz) { + + List interceptorClassNames = getInterceptorClassNames(conf, + pipeLineClassName, interceptorClassName); + + R pipeline = null; + R current = null; + + for (String className : interceptorClassNames) { + try { + Class interceptorClass = conf.getClassByName(className); + if (clazz.isAssignableFrom(interceptorClass)) { + Object interceptorInstance = ReflectionUtils.newInstance(interceptorClass, conf); + if (pipeline == null) { + pipeline = clazz.cast(interceptorInstance); + current = clazz.cast(interceptorInstance); + continue; + } else { + Method method = clazz.getMethod("setNextInterceptor", clazz); + method.invoke(current, interceptorInstance); + current = clazz.cast(interceptorInstance); + } + } else { + LOG.error("Class: {} not instance of {}.", className, clazz.getCanonicalName()); + throw new YarnRuntimeException("Class: " + className + " not instance of " + + clazz.getCanonicalName()); + } + } catch (ClassNotFoundException e) { + LOG.error("Could not instantiate RequestInterceptor: {}", className, e); + throw new YarnRuntimeException("Could not instantiate RequestInterceptor: " + className, e); + } catch (InvocationTargetException e) { + LOG.error("RequestInterceptor {} call setNextInterceptor error.", className, e); + throw new YarnRuntimeException("RequestInterceptor " + className + + " call setNextInterceptor error.", e); + } catch (NoSuchMethodException e) { + LOG.error("RequestInterceptor {} does not contain the method setNextInterceptor.", + className); + throw new YarnRuntimeException("RequestInterceptor " + className + + " does not contain the method setNextInterceptor.", e); + } catch (IllegalAccessException e) { + LOG.error("RequestInterceptor {} call the method setNextInterceptor " + + "does not have access.", className); + throw new YarnRuntimeException("RequestInterceptor " + + className + " call the method setNextInterceptor does not have access.", e); + } + } + + if (pipeline == null) { + throw new YarnRuntimeException( + "RequestInterceptor pipeline is not configured in the system."); + } + + return pipeline; + } + + private static List getInterceptorClassNames(Configuration conf, + String pipeLineClass, String interceptorClass) { + String configuredInterceptorClassNames = conf.get(pipeLineClass, interceptorClass); + List interceptorClassNames = new ArrayList<>(); + Collection tempList = + StringUtils.getStringCollection(configuredInterceptorClassNames); + for (String item : tempList) { + interceptorClassNames.add(item.trim()); + } + return interceptorClassNames; + } + + /** + * Throws an IOException due to an error. + * + * @param errMsg the error message + * @param t the throwable raised in the called class. + * @throws IOException on failure + */ + @Public + @Unstable + public static void logAndThrowIOException(String errMsg, Throwable t) + throws IOException { + if (t != null) { + LOG.error(errMsg, t); + throw new IOException(errMsg, t); + } else { + LOG.error(errMsg); + throw new IOException(errMsg); + } + } + + /** + * Throws an RunTimeException due to an error. + * + * @param errMsg the error message + * @param t the throwable raised in the called class. + * @throws RuntimeException on failure + */ + @Public + @Unstable + public static void logAndThrowRunTimeException(String errMsg, Throwable t) + throws RuntimeException { + if (t != null) { + LOG.error(errMsg, t); + throw new RuntimeException(errMsg, t); + } else { + LOG.error(errMsg); + throw new RuntimeException(errMsg); + } + } + + /** + * Throws an RunTimeException due to an error. + * + * @param t the throwable raised in the called class. + * @param errMsgFormat the error message format string. + * @param args referenced by the format specifiers in the format string. + * @throws RuntimeException on failure + */ + @Public + @Unstable + public static void logAndThrowRunTimeException(Throwable t, String errMsgFormat, Object... args) + throws RuntimeException { + String msg = String.format(errMsgFormat, args); + if (t != null) { + LOG.error(msg, t); + throw new RuntimeException(msg, t); + } else { + LOG.error(msg); + throw new RuntimeException(msg); + } + } } \ No newline at end of file diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/clientrm/AbstractClientRequestInterceptor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/clientrm/AbstractClientRequestInterceptor.java index 01ba3bdcadf35..961026d014611 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/clientrm/AbstractClientRequestInterceptor.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/clientrm/AbstractClientRequestInterceptor.java @@ -29,7 +29,7 @@ /** * Implements the {@link ClientRequestInterceptor} interface and provides common * functionality which can can be used and/or extended by other concrete - * intercepter classes. + * interceptor classes. * */ public abstract class AbstractClientRequestInterceptor @@ -106,8 +106,9 @@ private void setupUser(String userName) { try { // Do not create a proxy user if user name matches the user name on // current UGI - if (userName.equalsIgnoreCase( - UserGroupInformation.getCurrentUser().getUserName())) { + if (UserGroupInformation.isSecurityEnabled()) { + user = UserGroupInformation.createProxyUser(userName, UserGroupInformation.getLoginUser()); + } else if (userName.equalsIgnoreCase(UserGroupInformation.getCurrentUser().getUserName())) { user = UserGroupInformation.getCurrentUser(); } else { user = UserGroupInformation.createProxyUser(userName, diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/clientrm/ClientRequestInterceptor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/clientrm/ClientRequestInterceptor.java index 2f8fb936345ca..3e3ffce5f4b3f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/clientrm/ClientRequestInterceptor.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/clientrm/ClientRequestInterceptor.java @@ -22,14 +22,14 @@ import org.apache.hadoop.yarn.api.ApplicationClientProtocol; /** - * Defines the contract to be implemented by the request intercepter classes, + * Defines the contract to be implemented by the request interceptor classes, * that can be used to intercept and inspect messages sent from the client to * the resource manager. */ public interface ClientRequestInterceptor extends ApplicationClientProtocol, Configurable { /** - * This method is called for initializing the intercepter. This is guaranteed + * This method is called for initializing the interceptor. This is guaranteed * to be called only once in the lifetime of this instance. * * @param user the name of the client @@ -37,28 +37,28 @@ public interface ClientRequestInterceptor void init(String user); /** - * This method is called to release the resources held by the intercepter. + * This method is called to release the resources held by the interceptor. * This will be called when the application pipeline is being destroyed. The * concrete implementations should dispose the resources and forward the - * request to the next intercepter, if any. + * request to the next interceptor, if any. */ void shutdown(); /** - * Sets the next intercepter in the pipeline. The concrete implementation of + * Sets the next interceptor in the pipeline. The concrete implementation of * this interface should always pass the request to the nextInterceptor after - * inspecting the message. The last intercepter in the chain is responsible to + * inspecting the message. The last interceptor in the chain is responsible to * send the messages to the resource manager service and so the last - * intercepter will not receive this method call. + * interceptor will not receive this method call. * * @param nextInterceptor the ClientRequestInterceptor to set in the pipeline */ void setNextInterceptor(ClientRequestInterceptor nextInterceptor); /** - * Returns the next intercepter in the chain. + * Returns the next interceptor in the chain. * - * @return the next intercepter in the chain + * @return the next interceptor in the chain */ ClientRequestInterceptor getNextInterceptor(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/clientrm/FederationClientInterceptor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/clientrm/FederationClientInterceptor.java index 6cc317242cd73..ec6f5fbb0d998 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/clientrm/FederationClientInterceptor.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/clientrm/FederationClientInterceptor.java @@ -116,6 +116,7 @@ import org.apache.hadoop.yarn.api.protocolrecords.UpdateApplicationTimeoutsRequest; import org.apache.hadoop.yarn.api.protocolrecords.UpdateApplicationTimeoutsResponse; import org.apache.hadoop.yarn.api.records.ApplicationId; +import org.apache.hadoop.yarn.api.records.ReservationId; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.exceptions.YarnRuntimeException; @@ -124,6 +125,7 @@ import org.apache.hadoop.yarn.server.federation.policies.RouterPolicyFacade; import org.apache.hadoop.yarn.server.federation.policies.exceptions.FederationPolicyInitializationException; import org.apache.hadoop.yarn.server.federation.store.records.ApplicationHomeSubCluster; +import org.apache.hadoop.yarn.server.federation.store.records.ReservationHomeSubCluster; import org.apache.hadoop.yarn.server.federation.store.records.SubClusterId; import org.apache.hadoop.yarn.server.federation.store.records.SubClusterInfo; import org.apache.hadoop.yarn.server.federation.utils.FederationStateStoreFacade; @@ -137,11 +139,18 @@ import org.apache.hadoop.classification.VisibleForTesting; +import static org.apache.hadoop.yarn.server.router.RouterAuditLogger.AuditConstants.GET_NEW_APP; +import static org.apache.hadoop.yarn.server.router.RouterAuditLogger.AuditConstants.SUBMIT_NEW_APP; +import static org.apache.hadoop.yarn.server.router.RouterAuditLogger.AuditConstants.GET_APP_REPORT; +import static org.apache.hadoop.yarn.server.router.RouterAuditLogger.AuditConstants.FORCE_KILL_APP; +import static org.apache.hadoop.yarn.server.router.RouterAuditLogger.AuditConstants.TARGET_CLIENT_RM_SERVICE; +import static org.apache.hadoop.yarn.server.router.RouterAuditLogger.AuditConstants.UNKNOWN; + /** * Extends the {@code AbstractRequestInterceptorClient} class and provides an * implementation for federation of YARN RM and scaling an application across * multiple YARN SubClusters. All the federation specific implementation is - * encapsulated in this class. This is always the last intercepter in the chain. + * encapsulated in this class. This is always the last interceptor in the chain. */ public class FederationClientInterceptor extends AbstractClientRequestInterceptor { @@ -175,7 +184,6 @@ public void init(String userName) { federationFacade = FederationStateStoreFacade.getInstance(); rand = new Random(System.currentTimeMillis()); - int numThreads = getConf().getInt( YarnConfiguration.ROUTER_USER_CLIENT_THREADS_SIZE, YarnConfiguration.DEFAULT_ROUTER_USER_CLIENT_THREADS_SIZE); @@ -195,12 +203,11 @@ public void init(String userName) { LOG.error(e.getMessage()); } - numSubmitRetries = - conf.getInt(YarnConfiguration.ROUTER_CLIENTRM_SUBMIT_RETRY, - YarnConfiguration.DEFAULT_ROUTER_CLIENTRM_SUBMIT_RETRY); + numSubmitRetries = conf.getInt( + YarnConfiguration.ROUTER_CLIENTRM_SUBMIT_RETRY, + YarnConfiguration.DEFAULT_ROUTER_CLIENTRM_SUBMIT_RETRY); - clientRMProxies = - new ConcurrentHashMap(); + clientRMProxies = new ConcurrentHashMap<>(); routerMetrics = RouterMetrics.getMetrics(); returnPartialReport = conf.getBoolean( @@ -227,35 +234,29 @@ protected ApplicationClientProtocol getClientRMProxyForSubCluster( ApplicationClientProtocol clientRMProxy = null; try { boolean serviceAuthEnabled = getConf().getBoolean( - CommonConfigurationKeys.HADOOP_SECURITY_AUTHORIZATION, false); + CommonConfigurationKeys.HADOOP_SECURITY_AUTHORIZATION, false); UserGroupInformation realUser = user; if (serviceAuthEnabled) { realUser = UserGroupInformation.createProxyUser( - user.getShortUserName(), UserGroupInformation.getLoginUser()); + user.getShortUserName(), UserGroupInformation.getLoginUser()); } clientRMProxy = FederationProxyProviderUtil.createRMProxy(getConf(), ApplicationClientProtocol.class, subClusterId, realUser); } catch (Exception e) { RouterServerUtil.logAndThrowException( - "Unable to create the interface to reach the SubCluster " - + subClusterId, - e); + "Unable to create the interface to reach the SubCluster " + subClusterId, e); } - clientRMProxies.put(subClusterId, clientRMProxy); return clientRMProxy; } private SubClusterId getRandomActiveSubCluster( - Map activeSubclusters) - throws YarnException { - - if (activeSubclusters == null || activeSubclusters.size() < 1) { + Map activeSubClusters) throws YarnException { + if (activeSubClusters == null || activeSubClusters.isEmpty()) { RouterServerUtil.logAndThrowException( FederationPolicyUtils.NO_ACTIVE_SUBCLUSTER_AVAILABLE, null); } - List list = new ArrayList<>(activeSubclusters.keySet()); - + List list = new ArrayList<>(activeSubClusters.keySet()); return list.get(rand.nextInt(list.size())); } @@ -280,46 +281,43 @@ private SubClusterId getRandomActiveSubCluster( public GetNewApplicationResponse getNewApplication( GetNewApplicationRequest request) throws YarnException, IOException { - long startTime = clock.getTime(); + if (request == null) { + routerMetrics.incrAppsFailedCreated(); + String errMsg = "Missing getNewApplication request."; + RouterAuditLogger.logFailure(user.getShortUserName(), GET_NEW_APP, UNKNOWN, + TARGET_CLIENT_RM_SERVICE, errMsg); + RouterServerUtil.logAndThrowException(errMsg, null); + } + long startTime = clock.getTime(); Map subClustersActive = federationFacade.getSubClusters(true); for (int i = 0; i < numSubmitRetries; ++i) { SubClusterId subClusterId = getRandomActiveSubCluster(subClustersActive); - LOG.debug( - "getNewApplication try #{} on SubCluster {}", i, subClusterId); - ApplicationClientProtocol clientRMProxy = - getClientRMProxyForSubCluster(subClusterId); + LOG.info("getNewApplication try #{} on SubCluster {}.", i, subClusterId); + ApplicationClientProtocol clientRMProxy = getClientRMProxyForSubCluster(subClusterId); GetNewApplicationResponse response = null; try { response = clientRMProxy.getNewApplication(request); } catch (Exception e) { - LOG.warn("Unable to create a new ApplicationId in SubCluster " - + subClusterId.getId(), e); + LOG.warn("Unable to create a new ApplicationId in SubCluster {}.", subClusterId.getId(), e); + subClustersActive.remove(subClusterId); } if (response != null) { - long stopTime = clock.getTime(); routerMetrics.succeededAppsCreated(stopTime - startTime); - RouterAuditLogger.logSuccess(user.getShortUserName(), - RouterAuditLogger.AuditConstants.GET_NEW_APP, - "RouterClientRMService", response.getApplicationId()); + RouterAuditLogger.logSuccess(user.getShortUserName(), GET_NEW_APP, + TARGET_CLIENT_RM_SERVICE, response.getApplicationId()); return response; - } else { - // Empty response from the ResourceManager. - // Blacklist this subcluster for this request. - subClustersActive.remove(subClusterId); } - } routerMetrics.incrAppsFailedCreated(); - String errMsg = "Fail to create a new application."; - RouterAuditLogger.logFailure(user.getShortUserName(), - RouterAuditLogger.AuditConstants.GET_NEW_APP, "UNKNOWN", - "RouterClientRMService", errMsg); + String errMsg = "Failed to create a new application."; + RouterAuditLogger.logFailure(user.getShortUserName(), GET_NEW_APP, UNKNOWN, + TARGET_CLIENT_RM_SERVICE, errMsg); throw new YarnException(errMsg); } @@ -392,32 +390,32 @@ public GetNewApplicationResponse getNewApplication( public SubmitApplicationResponse submitApplication( SubmitApplicationRequest request) throws YarnException, IOException { - long startTime = clock.getTime(); - if (request == null || request.getApplicationSubmissionContext() == null - || request.getApplicationSubmissionContext() - .getApplicationId() == null) { + || request.getApplicationSubmissionContext().getApplicationId() == null) { routerMetrics.incrAppsFailedSubmitted(); String errMsg = - "Missing submitApplication request or applicationSubmissionContext " - + "information."; - RouterAuditLogger.logFailure(user.getShortUserName(), - RouterAuditLogger.AuditConstants.SUBMIT_NEW_APP, "UNKNOWN", - "RouterClientRMService", errMsg); - throw new YarnException(errMsg); + "Missing submitApplication request or applicationSubmissionContext information."; + RouterAuditLogger.logFailure(user.getShortUserName(), SUBMIT_NEW_APP, UNKNOWN, + TARGET_CLIENT_RM_SERVICE, errMsg); + RouterServerUtil.logAndThrowException(errMsg, null); } + SubmitApplicationResponse response = null; + + long startTime = clock.getTime(); + ApplicationId applicationId = request.getApplicationSubmissionContext().getApplicationId(); - List blacklist = new ArrayList(); + List blacklist = new ArrayList<>(); for (int i = 0; i < numSubmitRetries; ++i) { SubClusterId subClusterId = policyFacade.getHomeSubcluster( request.getApplicationSubmissionContext(), blacklist); - LOG.info("submitApplication appId {} try #{} on SubCluster {}.", applicationId, i, - subClusterId); + + LOG.info("submitApplication appId {} try #{} on SubCluster {}.", + applicationId, i, subClusterId); ApplicationHomeSubCluster appHomeSubCluster = ApplicationHomeSubCluster.newInstance(applicationId, subClusterId); @@ -430,12 +428,12 @@ public SubmitApplicationResponse submitApplication( federationFacade.addApplicationHomeSubCluster(appHomeSubCluster); } catch (YarnException e) { routerMetrics.incrAppsFailedSubmitted(); - String message = "Unable to insert the ApplicationId " + applicationId - + " into the FederationStateStore"; - RouterAuditLogger.logFailure(user.getShortUserName(), - RouterAuditLogger.AuditConstants.SUBMIT_NEW_APP, "UNKNOWN", - "RouterClientRMService", message, applicationId, subClusterId); - throw new YarnException(message, e); + String message = + String.format("Unable to insert the ApplicationId %s into the FederationStateStore.", + applicationId); + RouterAuditLogger.logFailure(user.getShortUserName(), SUBMIT_NEW_APP, UNKNOWN, + TARGET_CLIENT_RM_SERVICE, message, applicationId, subClusterId); + RouterServerUtil.logAndThrowException(message, e); } } else { try { @@ -443,19 +441,19 @@ public SubmitApplicationResponse submitApplication( // the new subClusterId we have selected federationFacade.updateApplicationHomeSubCluster(appHomeSubCluster); } catch (YarnException e) { - String message = "Unable to update the ApplicationId " + applicationId - + " into the FederationStateStore"; + String message = + String.format("Unable to update the ApplicationId %s into the FederationStateStore.", + applicationId); SubClusterId subClusterIdInStateStore = federationFacade.getApplicationHomeSubCluster(applicationId); if (subClusterId == subClusterIdInStateStore) { - LOG.info("Application {} already submitted on SubCluster {}.", applicationId, - subClusterId); + LOG.info("Application {} already submitted on SubCluster {}.", + applicationId, subClusterId); } else { routerMetrics.incrAppsFailedSubmitted(); - RouterAuditLogger.logFailure(user.getShortUserName(), - RouterAuditLogger.AuditConstants.SUBMIT_NEW_APP, "UNKNOWN", - "RouterClientRMService", message, applicationId, subClusterId); - throw new YarnException(message, e); + RouterAuditLogger.logFailure(user.getShortUserName(), SUBMIT_NEW_APP, UNKNOWN, + TARGET_CLIENT_RM_SERVICE, message, applicationId, subClusterId); + RouterServerUtil.logAndThrowException(message, e); } } } @@ -463,7 +461,6 @@ public SubmitApplicationResponse submitApplication( ApplicationClientProtocol clientRMProxy = getClientRMProxyForSubCluster(subClusterId); - SubmitApplicationResponse response = null; try { response = clientRMProxy.submitApplication(request); } catch (Exception e) { @@ -477,9 +474,8 @@ public SubmitApplicationResponse submitApplication( applicationId, subClusterId); long stopTime = clock.getTime(); routerMetrics.succeededAppsSubmitted(stopTime - startTime); - RouterAuditLogger.logSuccess(user.getShortUserName(), - RouterAuditLogger.AuditConstants.SUBMIT_NEW_APP, - "RouterClientRMService", applicationId, subClusterId); + RouterAuditLogger.logSuccess(user.getShortUserName(), SUBMIT_NEW_APP, + TARGET_CLIENT_RM_SERVICE, applicationId, subClusterId); return response; } else { // Empty response from the ResourceManager. @@ -489,13 +485,11 @@ public SubmitApplicationResponse submitApplication( } routerMetrics.incrAppsFailedSubmitted(); - String errMsg = "Application " - + request.getApplicationSubmissionContext().getApplicationName() - + " with appId " + applicationId + " failed to be submitted."; - RouterAuditLogger.logFailure(user.getShortUserName(), - RouterAuditLogger.AuditConstants.SUBMIT_NEW_APP, "UNKNOWN", - "RouterClientRMService", errMsg, applicationId); - throw new YarnException(errMsg); + String msg = String.format("Application %s with appId %s failed to be submitted.", + request.getApplicationSubmissionContext().getApplicationName(), applicationId); + RouterAuditLogger.logFailure(user.getShortUserName(), SUBMIT_NEW_APP, UNKNOWN, + TARGET_CLIENT_RM_SERVICE, msg, applicationId); + throw new YarnException(msg); } /** @@ -518,16 +512,16 @@ public SubmitApplicationResponse submitApplication( public KillApplicationResponse forceKillApplication( KillApplicationRequest request) throws YarnException, IOException { - long startTime = clock.getTime(); - if (request == null || request.getApplicationId() == null) { routerMetrics.incrAppsFailedKilled(); - String message = "Missing forceKillApplication request or ApplicationId."; - RouterAuditLogger.logFailure(user.getShortUserName(), - RouterAuditLogger.AuditConstants.FORCE_KILL_APP, "UNKNOWN", - "RouterClientRMService", message); - throw new YarnException(message); + String msg = "Missing forceKillApplication request or ApplicationId."; + RouterAuditLogger.logFailure(user.getShortUserName(), FORCE_KILL_APP, UNKNOWN, + TARGET_CLIENT_RM_SERVICE, msg); + RouterServerUtil.logAndThrowException(msg, null); } + + long startTime = clock.getTime(); + ApplicationId applicationId = request.getApplicationId(); SubClusterId subClusterId = null; @@ -536,12 +530,11 @@ public KillApplicationResponse forceKillApplication( .getApplicationHomeSubCluster(request.getApplicationId()); } catch (YarnException e) { routerMetrics.incrAppsFailedKilled(); - String msg = "Application " + applicationId - + " does not exist in FederationStateStore"; - RouterAuditLogger.logFailure(user.getShortUserName(), - RouterAuditLogger.AuditConstants.FORCE_KILL_APP, "UNKNOWN", - "RouterClientRMService", msg, applicationId); - throw new YarnException(msg, e); + String msg = + String.format("Application %s does not exist in FederationStateStore.", applicationId); + RouterAuditLogger.logFailure(user.getShortUserName(), FORCE_KILL_APP, UNKNOWN, + TARGET_CLIENT_RM_SERVICE, msg, applicationId); + RouterServerUtil.logAndThrowException(msg, e); } ApplicationClientProtocol clientRMProxy = @@ -553,23 +546,21 @@ public KillApplicationResponse forceKillApplication( response = clientRMProxy.forceKillApplication(request); } catch (Exception e) { routerMetrics.incrAppsFailedKilled(); - RouterAuditLogger.logFailure(user.getShortUserName(), - RouterAuditLogger.AuditConstants.FORCE_KILL_APP, "UNKNOWN", - "RouterClientRMService", "Unable to kill the application report", - applicationId, subClusterId); - throw e; + String msg = "Unable to kill the application report."; + RouterAuditLogger.logFailure(user.getShortUserName(), FORCE_KILL_APP, UNKNOWN, + TARGET_CLIENT_RM_SERVICE, msg, applicationId, subClusterId); + RouterServerUtil.logAndThrowException(msg, e); } if (response == null) { - LOG.error("No response when attempting to kill the application " - + applicationId + " to SubCluster " + subClusterId.getId()); + LOG.error("No response when attempting to kill the application {} to SubCluster {}.", + applicationId, subClusterId.getId()); } long stopTime = clock.getTime(); routerMetrics.succeededAppsKilled(stopTime - startTime); - RouterAuditLogger.logSuccess(user.getShortUserName(), - RouterAuditLogger.AuditConstants.FORCE_KILL_APP, - "RouterClientRMService", applicationId); + RouterAuditLogger.logSuccess(user.getShortUserName(), FORCE_KILL_APP, + TARGET_CLIENT_RM_SERVICE, applicationId); return response; } @@ -593,18 +584,15 @@ public KillApplicationResponse forceKillApplication( public GetApplicationReportResponse getApplicationReport( GetApplicationReportRequest request) throws YarnException, IOException { - long startTime = clock.getTime(); - if (request == null || request.getApplicationId() == null) { routerMetrics.incrAppsFailedRetrieved(); - String errMsg = - "Missing getApplicationReport request or applicationId information."; - RouterAuditLogger.logFailure(user.getShortUserName(), - RouterAuditLogger.AuditConstants.GET_APP_REPORT, "UNKNOWN", - "RouterClientRMService", errMsg); - throw new YarnException(errMsg); + String errMsg = "Missing getApplicationReport request or applicationId information."; + RouterAuditLogger.logFailure(user.getShortUserName(), GET_APP_REPORT, UNKNOWN, + TARGET_CLIENT_RM_SERVICE, errMsg); + RouterServerUtil.logAndThrowException(errMsg, null); } + long startTime = clock.getTime(); SubClusterId subClusterId = null; try { @@ -612,29 +600,26 @@ public GetApplicationReportResponse getApplicationReport( .getApplicationHomeSubCluster(request.getApplicationId()); } catch (YarnException e) { routerMetrics.incrAppsFailedRetrieved(); - String errMsg = "Application " + request.getApplicationId() - + " does not exist in FederationStateStore"; - RouterAuditLogger.logFailure(user.getShortUserName(), - RouterAuditLogger.AuditConstants.GET_APP_REPORT, "UNKNOWN", - "RouterClientRMService", errMsg, request.getApplicationId()); - throw new YarnException(errMsg, e); + String errMsg = String.format("Application %s does not exist in FederationStateStore.", + request.getApplicationId()); + RouterAuditLogger.logFailure(user.getShortUserName(), GET_APP_REPORT, UNKNOWN, + TARGET_CLIENT_RM_SERVICE, errMsg, request.getApplicationId()); + RouterServerUtil.logAndThrowException(errMsg, e); } ApplicationClientProtocol clientRMProxy = getClientRMProxyForSubCluster(subClusterId); - GetApplicationReportResponse response = null; + try { response = clientRMProxy.getApplicationReport(request); } catch (Exception e) { routerMetrics.incrAppsFailedRetrieved(); - String errMsg = "Unable to get the application report for " + request - .getApplicationId() + "to SubCluster " + subClusterId.getId(); - RouterAuditLogger.logFailure(user.getShortUserName(), - RouterAuditLogger.AuditConstants.GET_APP_REPORT, "UNKNOWN", - "RouterClientRMService", errMsg, request.getApplicationId(), - subClusterId); - throw e; + String errMsg = String.format("Unable to get the application report for %s to SubCluster %s.", + request.getApplicationId(), subClusterId.getId()); + RouterAuditLogger.logFailure(user.getShortUserName(), GET_APP_REPORT, UNKNOWN, + TARGET_CLIENT_RM_SERVICE, errMsg, request.getApplicationId(), subClusterId); + RouterServerUtil.logAndThrowException(errMsg, e); } if (response == null) { @@ -642,12 +627,10 @@ public GetApplicationReportResponse getApplicationReport( + "the application {} to SubCluster {}.", request.getApplicationId(), subClusterId.getId()); } - long stopTime = clock.getTime(); routerMetrics.succeededAppsRetrieved(stopTime - startTime); - RouterAuditLogger.logSuccess(user.getShortUserName(), - RouterAuditLogger.AuditConstants.GET_APP_REPORT, - "RouterClientRMService", request.getApplicationId()); + RouterAuditLogger.logSuccess(user.getShortUserName(), GET_APP_REPORT, + TARGET_CLIENT_RM_SERVICE, request.getApplicationId()); return response; } @@ -662,7 +645,7 @@ public GetApplicationReportResponse getApplicationReport( * Router: the Client will timeout and resubmit the request. * * ResourceManager: the Router calls each Yarn RM in parallel. In case a - * Yarn RM fails, a single call will timeout. However the Router will + * Yarn RM fails, a single call will timeout. However, the Router will * merge the ApplicationReports it got, and provides a partial list to * the client. * @@ -675,56 +658,48 @@ public GetApplicationsResponse getApplications(GetApplicationsRequest request) throws YarnException, IOException { if (request == null) { routerMetrics.incrMultipleAppsFailedRetrieved(); - RouterServerUtil.logAndThrowException( - "Missing getApplications request.", - null); + RouterServerUtil.logAndThrowException("Missing getApplications request.", null); } long startTime = clock.getTime(); Map subclusters = federationFacade.getSubClusters(true); ClientMethod remoteMethod = new ClientMethod("getApplications", new Class[] {GetApplicationsRequest.class}, new Object[] {request}); - Map applications; - + Map applications = null; try { applications = invokeConcurrent(subclusters.keySet(), remoteMethod, GetApplicationsResponse.class); - } catch (Exception ex) { routerMetrics.incrMultipleAppsFailedRetrieved(); - LOG.error("Unable to get applications due to exception.", ex); - throw ex; + RouterServerUtil.logAndThrowException("Unable to get applications due to exception.", ex); } long stopTime = clock.getTime(); routerMetrics.succeededMultipleAppsRetrieved(stopTime - startTime); // Merge the Application Reports - return RouterYarnClientUtils.mergeApplications(applications.values(), - returnPartialReport); + return RouterYarnClientUtils.mergeApplications(applications.values(), returnPartialReport); } @Override public GetClusterMetricsResponse getClusterMetrics( GetClusterMetricsRequest request) throws YarnException, IOException { + if (request == null) { + routerMetrics.incrGetClusterMetricsFailedRetrieved(); + RouterServerUtil.logAndThrowException("Missing getClusterMetrics request.", null); + } long startTime = clock.getTime(); - Map subclusters = - federationFacade.getSubClusters(true); ClientMethod remoteMethod = new ClientMethod("getClusterMetrics", new Class[] {GetClusterMetricsRequest.class}, new Object[] {request}); - Map clusterMetrics; - + Collection clusterMetrics = null; try { - clusterMetrics = invokeConcurrent(subclusters.keySet(), remoteMethod, - GetClusterMetricsResponse.class); - + clusterMetrics = invokeAppClientProtocolMethod( + true, remoteMethod, GetClusterMetricsResponse.class); } catch (Exception ex) { routerMetrics.incrGetClusterMetricsFailedRetrieved(); - LOG.error("Unable to get cluster metrics due to exception.", ex); - throw ex; + RouterServerUtil.logAndThrowException("Unable to get cluster metrics due to exception.", ex); } - long stopTime = clock.getTime(); routerMetrics.succeededGetClusterMetricsRetrieved(stopTime - startTime); - return RouterYarnClientUtils.merge(clusterMetrics.values()); + return RouterYarnClientUtils.merge(clusterMetrics); } Map invokeConcurrent(ArrayList clusterIds, @@ -809,8 +784,7 @@ public GetClusterNodesResponse getClusterNodes(GetClusterNodesRequest request) clusterNodes.put(subClusterId, response); } catch (Exception ex) { routerMetrics.incrClusterNodesFailedRetrieved(); - LOG.error("Unable to get cluster nodes due to exception.", ex); - throw ex; + RouterServerUtil.logAndThrowException("Unable to get cluster nodes due to exception.", ex); } } long stopTime = clock.getTime(); @@ -822,7 +796,27 @@ public GetClusterNodesResponse getClusterNodes(GetClusterNodesRequest request) @Override public GetQueueInfoResponse getQueueInfo(GetQueueInfoRequest request) throws YarnException, IOException { - throw new NotImplementedException("Code is not implemented"); + if (request == null || request.getQueueName() == null) { + routerMetrics.incrGetQueueInfoFailedRetrieved(); + RouterServerUtil.logAndThrowException("Missing getQueueInfo request or queueName.", null); + } + + long startTime = clock.getTime(); + ClientMethod remoteMethod = new ClientMethod("getQueueInfo", + new Class[]{GetQueueInfoRequest.class}, new Object[]{request}); + Collection queues = null; + try { + queues = invokeAppClientProtocolMethod(true, remoteMethod, + GetQueueInfoResponse.class); + } catch (Exception ex) { + routerMetrics.incrGetQueueInfoFailedRetrieved(); + RouterServerUtil.logAndThrowException("Unable to get queue [" + + request.getQueueName() + "] to exception.", ex); + } + long stopTime = clock.getTime(); + routerMetrics.succeededGetQueueInfoRetrieved(stopTime - startTime); + // Merge the GetQueueInfoResponse + return RouterYarnClientUtils.mergeQueues(queues); } @Override @@ -835,14 +829,13 @@ public GetQueueUserAclsInfoResponse getQueueUserAcls( long startTime = clock.getTime(); ClientMethod remoteMethod = new ClientMethod("getQueueUserAcls", new Class[] {GetQueueUserAclsInfoRequest.class}, new Object[] {request}); - Collection queueUserAcls; + Collection queueUserAcls = null; try { queueUserAcls = invokeAppClientProtocolMethod(true, remoteMethod, GetQueueUserAclsInfoResponse.class); } catch (Exception ex) { routerMetrics.incrQueueUserAclsFailedRetrieved(); - LOG.error("Unable to get queue user Acls due to exception.", ex); - throw ex; + RouterServerUtil.logAndThrowException("Unable to get queue user Acls due to exception.", ex); } long stopTime = clock.getTime(); routerMetrics.succeededGetQueueUserAclsRetrieved(stopTime - startTime); @@ -854,19 +847,137 @@ public GetQueueUserAclsInfoResponse getQueueUserAcls( public MoveApplicationAcrossQueuesResponse moveApplicationAcrossQueues( MoveApplicationAcrossQueuesRequest request) throws YarnException, IOException { - throw new NotImplementedException("Code is not implemented"); + if (request == null || request.getApplicationId() == null || request.getTargetQueue() == null) { + routerMetrics.incrMoveApplicationAcrossQueuesFailedRetrieved(); + RouterServerUtil.logAndThrowException("Missing moveApplicationAcrossQueues request or " + + "applicationId or target queue.", null); + } + + long startTime = clock.getTime(); + SubClusterId subClusterId = null; + + ApplicationId applicationId = request.getApplicationId(); + try { + subClusterId = federationFacade + .getApplicationHomeSubCluster(applicationId); + } catch (YarnException e) { + routerMetrics.incrMoveApplicationAcrossQueuesFailedRetrieved(); + RouterServerUtil.logAndThrowException("Application " + + applicationId + " does not exist in FederationStateStore.", e); + } + + ApplicationClientProtocol clientRMProxy = getClientRMProxyForSubCluster(subClusterId); + MoveApplicationAcrossQueuesResponse response = null; + try { + response = clientRMProxy.moveApplicationAcrossQueues(request); + } catch (Exception e) { + routerMetrics.incrAppAttemptsFailedRetrieved(); + RouterServerUtil.logAndThrowException("Unable to moveApplicationAcrossQueues for " + + applicationId + " to SubCluster " + subClusterId.getId(), e); + } + + if (response == null) { + LOG.error("No response when moveApplicationAcrossQueues " + + "the applicationId {} to Queue {} In SubCluster {}.", + request.getApplicationId(), request.getTargetQueue(), subClusterId.getId()); + } + + long stopTime = clock.getTime(); + routerMetrics.succeededMoveApplicationAcrossQueuesRetrieved(stopTime - startTime); + return response; } @Override public GetNewReservationResponse getNewReservation( GetNewReservationRequest request) throws YarnException, IOException { - throw new NotImplementedException("Code is not implemented"); + + if (request == null) { + routerMetrics.incrGetNewReservationFailedRetrieved(); + String errMsg = "Missing getNewReservation request."; + RouterServerUtil.logAndThrowException(errMsg, null); + } + + long startTime = clock.getTime(); + Map subClustersActive = federationFacade.getSubClusters(true); + + for (int i = 0; i < numSubmitRetries; ++i) { + SubClusterId subClusterId = getRandomActiveSubCluster(subClustersActive); + LOG.info("getNewReservation try #{} on SubCluster {}.", i, subClusterId); + ApplicationClientProtocol clientRMProxy = getClientRMProxyForSubCluster(subClusterId); + try { + GetNewReservationResponse response = clientRMProxy.getNewReservation(request); + if (response != null) { + long stopTime = clock.getTime(); + routerMetrics.succeededGetNewReservationRetrieved(stopTime - startTime); + return response; + } + } catch (Exception e) { + LOG.warn("Unable to create a new Reservation in SubCluster {}.", subClusterId.getId(), e); + subClustersActive.remove(subClusterId); + } + } + + routerMetrics.incrGetNewReservationFailedRetrieved(); + String errMsg = "Failed to create a new reservation."; + throw new YarnException(errMsg); } @Override public ReservationSubmissionResponse submitReservation( ReservationSubmissionRequest request) throws YarnException, IOException { - throw new NotImplementedException("Code is not implemented"); + + if (request == null || request.getReservationId() == null + || request.getReservationDefinition() == null || request.getQueue() == null) { + routerMetrics.incrSubmitReservationFailedRetrieved(); + RouterServerUtil.logAndThrowException( + "Missing submitReservation request or reservationId " + + "or reservation definition or queue.", null); + } + + long startTime = clock.getTime(); + ReservationId reservationId = request.getReservationId(); + + for (int i = 0; i < numSubmitRetries; i++) { + try { + // First, Get SubClusterId according to specific strategy. + SubClusterId subClusterId = policyFacade.getReservationHomeSubCluster(request); + LOG.info("submitReservation ReservationId {} try #{} on SubCluster {}.", + reservationId, i, subClusterId); + ReservationHomeSubCluster reservationHomeSubCluster = + ReservationHomeSubCluster.newInstance(reservationId, subClusterId); + + // Second, determine whether the current ReservationId has a corresponding subCluster. + // If it does not exist, add it. If it exists, update it. + Boolean exists = existsReservationHomeSubCluster(reservationId); + + // We may encounter the situation of repeated submission of Reservation, + // at this time we should try to use the reservation that has been allocated + // !exists indicates that the reservation does not exist and needs to be added + // i==0, mainly to consider repeated submissions, + // so the first time to apply for reservation, try to use the original reservation + if (!exists || i == 0) { + addReservationHomeSubCluster(reservationId, reservationHomeSubCluster); + } else { + updateReservationHomeSubCluster(subClusterId, reservationId, reservationHomeSubCluster); + } + + // Third, Submit a Reservation request to the subCluster + ApplicationClientProtocol clientRMProxy = getClientRMProxyForSubCluster(subClusterId); + ReservationSubmissionResponse response = clientRMProxy.submitReservation(request); + if (response != null) { + LOG.info("Reservation {} submitted on subCluster {}.", reservationId, subClusterId); + long stopTime = clock.getTime(); + routerMetrics.succeededSubmitReservationRetrieved(stopTime - startTime); + return response; + } + } catch (Exception e) { + LOG.warn("Unable to submit(try #{}) the Reservation {}.", i, reservationId, e); + } + } + + routerMetrics.incrSubmitReservationFailedRetrieved(); + String msg = String.format("Reservation %s failed to be submitted.", reservationId); + throw new YarnException(msg); } @Override @@ -879,14 +990,14 @@ public ReservationListResponse listReservations( long startTime = clock.getTime(); ClientMethod remoteMethod = new ClientMethod("listReservations", new Class[] {ReservationListRequest.class}, new Object[] {request}); - Collection listResponses; + Collection listResponses = null; try { listResponses = invokeAppClientProtocolMethod(true, remoteMethod, ReservationListResponse.class); } catch (Exception ex) { routerMetrics.incrListReservationsFailedRetrieved(); - LOG.error("Unable to list reservations node due to exception.", ex); - throw ex; + RouterServerUtil.logAndThrowException( + "Unable to list reservations node due to exception.", ex); } long stopTime = clock.getTime(); routerMetrics.succeededListReservationsRetrieved(stopTime - startTime); @@ -897,13 +1008,68 @@ public ReservationListResponse listReservations( @Override public ReservationUpdateResponse updateReservation( ReservationUpdateRequest request) throws YarnException, IOException { - throw new NotImplementedException("Code is not implemented"); + + if (request == null || request.getReservationId() == null + || request.getReservationDefinition() == null) { + routerMetrics.incrUpdateReservationFailedRetrieved(); + RouterServerUtil.logAndThrowException( + "Missing updateReservation request or reservationId or reservation definition.", null); + } + + long startTime = clock.getTime(); + ReservationId reservationId = request.getReservationId(); + SubClusterId subClusterId = getReservationHomeSubCluster(reservationId); + + try { + ApplicationClientProtocol client = getClientRMProxyForSubCluster(subClusterId); + ReservationUpdateResponse response = client.updateReservation(request); + if (response != null) { + long stopTime = clock.getTime(); + routerMetrics.succeededUpdateReservationRetrieved(stopTime - startTime); + return response; + } + } catch (Exception ex) { + routerMetrics.incrUpdateReservationFailedRetrieved(); + RouterServerUtil.logAndThrowException( + "Unable to reservation update due to exception.", ex); + } + + routerMetrics.incrUpdateReservationFailedRetrieved(); + String msg = String.format("Reservation %s failed to be update.", reservationId); + throw new YarnException(msg); } @Override public ReservationDeleteResponse deleteReservation( ReservationDeleteRequest request) throws YarnException, IOException { - throw new NotImplementedException("Code is not implemented"); + if (request == null || request.getReservationId() == null) { + routerMetrics.incrDeleteReservationFailedRetrieved(); + RouterServerUtil.logAndThrowException( + "Missing deleteReservation request or reservationId.", null); + } + + long startTime = clock.getTime(); + ReservationId reservationId = request.getReservationId(); + SubClusterId subClusterId = getReservationHomeSubCluster(reservationId); + + try { + ApplicationClientProtocol client = getClientRMProxyForSubCluster(subClusterId); + ReservationDeleteResponse response = client.deleteReservation(request); + if (response != null) { + federationFacade.deleteReservationHomeSubCluster(reservationId); + long stopTime = clock.getTime(); + routerMetrics.succeededDeleteReservationRetrieved(stopTime - startTime); + return response; + } + } catch (Exception ex) { + routerMetrics.incrUpdateReservationFailedRetrieved(); + RouterServerUtil.logAndThrowException( + "Unable to reservation delete due to exception.", ex); + } + + routerMetrics.incrDeleteReservationFailedRetrieved(); + String msg = String.format("Reservation %s failed to be delete.", reservationId); + throw new YarnException(msg); } private Collection invokeAppClientProtocolMethod( @@ -934,14 +1100,13 @@ public GetNodesToLabelsResponse getNodeToLabels( long startTime = clock.getTime(); ClientMethod remoteMethod = new ClientMethod("getNodeToLabels", new Class[] {GetNodesToLabelsRequest.class}, new Object[] {request}); - Collection clusterNodes; + Collection clusterNodes = null; try { clusterNodes = invokeAppClientProtocolMethod(true, remoteMethod, GetNodesToLabelsResponse.class); } catch (Exception ex) { routerMetrics.incrNodeToLabelsFailedRetrieved(); - LOG.error("Unable to get label node due to exception.", ex); - throw ex; + RouterServerUtil.logAndThrowException("Unable to get node label due to exception.", ex); } long stopTime = clock.getTime(); routerMetrics.succeededGetNodeToLabelsRetrieved(stopTime - startTime); @@ -959,14 +1124,13 @@ public GetLabelsToNodesResponse getLabelsToNodes( long startTime = clock.getTime(); ClientMethod remoteMethod = new ClientMethod("getLabelsToNodes", new Class[] {GetLabelsToNodesRequest.class}, new Object[] {request}); - Collection labelNodes; + Collection labelNodes = null; try { labelNodes = invokeAppClientProtocolMethod(true, remoteMethod, GetLabelsToNodesResponse.class); } catch (Exception ex) { routerMetrics.incrLabelsToNodesFailedRetrieved(); - LOG.error("Unable to get label node due to exception.", ex); - throw ex; + RouterServerUtil.logAndThrowException("Unable to get label node due to exception.", ex); } long stopTime = clock.getTime(); routerMetrics.succeededGetLabelsToNodesRetrieved(stopTime - startTime); @@ -984,14 +1148,14 @@ public GetClusterNodeLabelsResponse getClusterNodeLabels( long startTime = clock.getTime(); ClientMethod remoteMethod = new ClientMethod("getClusterNodeLabels", new Class[] {GetClusterNodeLabelsRequest.class}, new Object[] {request}); - Collection nodeLabels; + Collection nodeLabels = null; try { nodeLabels = invokeAppClientProtocolMethod(true, remoteMethod, GetClusterNodeLabelsResponse.class); } catch (Exception ex) { routerMetrics.incrClusterNodeLabelsFailedRetrieved(); - LOG.error("Unable to get cluster nodeLabels due to exception.", ex); - throw ex; + RouterServerUtil.logAndThrowException("Unable to get cluster nodeLabels due to exception.", + ex); } long stopTime = clock.getTime(); routerMetrics.succeededGetClusterNodeLabelsRetrieved(stopTime - startTime); @@ -1044,15 +1208,15 @@ public GetApplicationAttemptReportResponse getApplicationAttemptReport( ApplicationClientProtocol clientRMProxy = getClientRMProxyForSubCluster(subClusterId); - GetApplicationAttemptReportResponse response; + GetApplicationAttemptReportResponse response = null; try { response = clientRMProxy.getApplicationAttemptReport(request); } catch (Exception e) { routerMetrics.incrAppAttemptsFailedRetrieved(); - LOG.error("Unable to get the applicationAttempt report for {} " - + "to SubCluster {}, error = {}.", - request.getApplicationAttemptId(), subClusterId.getId(), e); - throw e; + String msg = String.format( + "Unable to get the applicationAttempt report for %s to SubCluster %s.", + request.getApplicationAttemptId(), subClusterId.getId()); + RouterServerUtil.logAndThrowException(msg, e); } if (response == null) { @@ -1276,8 +1440,7 @@ public UpdateApplicationPriorityResponse updateApplicationPriority( } catch (YarnException e) { routerMetrics.incrUpdateAppPriorityFailedRetrieved(); RouterServerUtil.logAndThrowException("Application " + - request.getApplicationId() + - " does not exist in FederationStateStore.", e); + request.getApplicationId() + " does not exist in FederationStateStore.", e); } ApplicationClientProtocol clientRMProxy = getClientRMProxyForSubCluster(subClusterId); @@ -1391,13 +1554,50 @@ public UpdateApplicationTimeoutsResponse updateApplicationTimeouts( @Override public GetAllResourceProfilesResponse getResourceProfiles( GetAllResourceProfilesRequest request) throws YarnException, IOException { - throw new NotImplementedException("Code is not implemented"); + if (request == null) { + routerMetrics.incrGetResourceProfilesFailedRetrieved(); + RouterServerUtil.logAndThrowException("Missing getResourceProfiles request.", null); + } + long startTime = clock.getTime(); + ClientMethod remoteMethod = new ClientMethod("getResourceProfiles", + new Class[] {GetAllResourceProfilesRequest.class}, new Object[] {request}); + Collection resourceProfiles = null; + try { + resourceProfiles = invokeAppClientProtocolMethod(true, remoteMethod, + GetAllResourceProfilesResponse.class); + } catch (Exception ex) { + routerMetrics.incrGetResourceProfilesFailedRetrieved(); + RouterServerUtil.logAndThrowException("Unable to get resource profiles due to exception.", + ex); + } + long stopTime = clock.getTime(); + routerMetrics.succeededGetResourceProfilesRetrieved(stopTime - startTime); + return RouterYarnClientUtils.mergeClusterResourceProfilesResponse(resourceProfiles); } @Override public GetResourceProfileResponse getResourceProfile( GetResourceProfileRequest request) throws YarnException, IOException { - throw new NotImplementedException("Code is not implemented"); + if (request == null || request.getProfileName() == null) { + routerMetrics.incrGetResourceProfileFailedRetrieved(); + RouterServerUtil.logAndThrowException("Missing getResourceProfile request or profileName.", + null); + } + long startTime = clock.getTime(); + ClientMethod remoteMethod = new ClientMethod("getResourceProfile", + new Class[] {GetResourceProfileRequest.class}, new Object[] {request}); + Collection resourceProfile = null; + try { + resourceProfile = invokeAppClientProtocolMethod(true, remoteMethod, + GetResourceProfileResponse.class); + } catch (Exception ex) { + routerMetrics.incrGetResourceProfileFailedRetrieved(); + RouterServerUtil.logAndThrowException("Unable to get resource profile due to exception.", + ex); + } + long stopTime = clock.getTime(); + routerMetrics.succeededGetResourceProfileRetrieved(stopTime - startTime); + return RouterYarnClientUtils.mergeClusterResourceProfileResponse(resourceProfile); } @Override @@ -1434,20 +1634,75 @@ public void shutdown() { @Override public GetAttributesToNodesResponse getAttributesToNodes( GetAttributesToNodesRequest request) throws YarnException, IOException { - throw new NotImplementedException("Code is not implemented"); + if (request == null || request.getNodeAttributes() == null) { + routerMetrics.incrGetAttributesToNodesFailedRetrieved(); + RouterServerUtil.logAndThrowException("Missing getAttributesToNodes request " + + "or nodeAttributes.", null); + } + long startTime = clock.getTime(); + ClientMethod remoteMethod = new ClientMethod("getAttributesToNodes", + new Class[] {GetAttributesToNodesRequest.class}, new Object[] {request}); + Collection attributesToNodesResponses = null; + try { + attributesToNodesResponses = invokeAppClientProtocolMethod(true, remoteMethod, + GetAttributesToNodesResponse.class); + } catch (Exception ex) { + routerMetrics.incrGetAttributesToNodesFailedRetrieved(); + RouterServerUtil.logAndThrowException("Unable to get attributes to nodes due to exception.", + ex); + } + long stopTime = clock.getTime(); + routerMetrics.succeededGetAttributesToNodesRetrieved(stopTime - startTime); + return RouterYarnClientUtils.mergeAttributesToNodesResponse(attributesToNodesResponses); } @Override public GetClusterNodeAttributesResponse getClusterNodeAttributes( - GetClusterNodeAttributesRequest request) - throws YarnException, IOException { - throw new NotImplementedException("Code is not implemented"); + GetClusterNodeAttributesRequest request) throws YarnException, IOException { + if (request == null) { + routerMetrics.incrGetClusterNodeAttributesFailedRetrieved(); + RouterServerUtil.logAndThrowException("Missing getClusterNodeAttributes request.", null); + } + long startTime = clock.getTime(); + ClientMethod remoteMethod = new ClientMethod("getClusterNodeAttributes", + new Class[] {GetClusterNodeAttributesRequest.class}, new Object[] {request}); + Collection clusterNodeAttributesResponses = null; + try { + clusterNodeAttributesResponses = invokeAppClientProtocolMethod(true, remoteMethod, + GetClusterNodeAttributesResponse.class); + } catch (Exception ex) { + routerMetrics.incrGetClusterNodeAttributesFailedRetrieved(); + RouterServerUtil.logAndThrowException("Unable to get cluster node attributes due " + + " to exception.", ex); + } + long stopTime = clock.getTime(); + routerMetrics.succeededGetClusterNodeAttributesRetrieved(stopTime - startTime); + return RouterYarnClientUtils.mergeClusterNodeAttributesResponse(clusterNodeAttributesResponses); } @Override public GetNodesToAttributesResponse getNodesToAttributes( GetNodesToAttributesRequest request) throws YarnException, IOException { - throw new NotImplementedException("Code is not implemented"); + if (request == null || request.getHostNames() == null) { + routerMetrics.incrGetNodesToAttributesFailedRetrieved(); + RouterServerUtil.logAndThrowException("Missing getNodesToAttributes request or " + + "hostNames.", null); + } + long startTime = clock.getTime(); + ClientMethod remoteMethod = new ClientMethod("getNodesToAttributes", + new Class[] {GetNodesToAttributesRequest.class}, new Object[] {request}); + Collection nodesToAttributesResponses = null; + try { + nodesToAttributesResponses = invokeAppClientProtocolMethod(true, remoteMethod, + GetNodesToAttributesResponse.class); + } catch (Exception ex) { + routerMetrics.incrGetNodesToAttributesFailedRetrieved(); + RouterServerUtil.logAndThrowException("Unable to get nodes to attributes due " + + " to exception.", ex); + } + long stopTime = clock.getTime(); + routerMetrics.succeededGetNodesToAttributesRetrieved(stopTime - startTime); + return RouterYarnClientUtils.mergeNodesToAttributesResponse(nodesToAttributesResponses); } protected SubClusterId getApplicationHomeSubCluster( @@ -1465,7 +1720,7 @@ protected SubClusterId getApplicationHomeSubCluster( getApplicationHomeSubCluster(applicationId); } catch (YarnException ex) { if(LOG.isDebugEnabled()){ - LOG.debug("can't find applicationId = {} in home sub cluster, " + + LOG.debug("Can't find applicationId = {} in home sub cluster, " + " try foreach sub clusters.", applicationId); } } @@ -1497,13 +1752,93 @@ protected SubClusterId getApplicationHomeSubCluster( } catch (Exception ex) { if(LOG.isDebugEnabled()){ - LOG.debug("Can't Find ApplicationId = {} in Sub Cluster!", applicationId); + LOG.debug("Can't find applicationId = {} in Sub Cluster!", applicationId); } } } String errorMsg = - String.format("Can't Found applicationId = %s in any sub clusters", applicationId); + String.format("Can't find applicationId = %s in any sub clusters", applicationId); + throw new YarnException(errorMsg); + } + + protected SubClusterId getReservationHomeSubCluster(ReservationId reservationId) + throws YarnException { + + if (reservationId == null) { + LOG.error("ReservationId is Null, Can't find in SubCluster."); + return null; + } + + // try looking for reservation in Home SubCluster + try { + SubClusterId resultSubClusterId = + federationFacade.getReservationHomeSubCluster(reservationId); + if (resultSubClusterId != null) { + return resultSubClusterId; + } + } catch (YarnException e) { + RouterServerUtil.logAndThrowException(e, + "Can't find reservationId = %s in home sub cluster.", reservationId); + } + + String errorMsg = + String.format("Can't find reservationId = %s in home sub cluster.", reservationId); throw new YarnException(errorMsg); } + + @VisibleForTesting + public FederationStateStoreFacade getFederationFacade() { + return federationFacade; + } + + @VisibleForTesting + public Map getClientRMProxies() { + return clientRMProxies; + } + + private Boolean existsReservationHomeSubCluster(ReservationId reservationId) { + try { + SubClusterId subClusterId = federationFacade.getReservationHomeSubCluster(reservationId); + if (subClusterId != null) { + return true; + } + } catch (YarnException e) { + LOG.warn("get homeSubCluster by reservationId = {} error.", reservationId, e); + } + return false; + } + + private void addReservationHomeSubCluster(ReservationId reservationId, + ReservationHomeSubCluster homeSubCluster) throws YarnException { + try { + // persist the mapping of reservationId and the subClusterId which has + // been selected as its home + federationFacade.addReservationHomeSubCluster(homeSubCluster); + } catch (YarnException e) { + RouterServerUtil.logAndThrowException(e, + "Unable to insert the ReservationId %s into the FederationStateStore.", + reservationId); + } + } + + private void updateReservationHomeSubCluster(SubClusterId subClusterId, + ReservationId reservationId, ReservationHomeSubCluster homeSubCluster) throws YarnException { + try { + // update the mapping of reservationId and the home subClusterId to + // the new subClusterId we have selected + federationFacade.updateReservationHomeSubCluster(homeSubCluster); + } catch (YarnException e) { + SubClusterId subClusterIdInStateStore = + federationFacade.getReservationHomeSubCluster(reservationId); + if (subClusterId == subClusterIdInStateStore) { + LOG.info("Reservation {} already submitted on SubCluster {}.", + reservationId, subClusterId); + } else { + RouterServerUtil.logAndThrowException(e, + "Unable to update the ReservationId %s into the FederationStateStore.", + reservationId); + } + } + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/clientrm/RouterClientRMService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/clientrm/RouterClientRMService.java index 955c48fd953a9..b60a267746e4f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/clientrm/RouterClientRMService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/clientrm/RouterClientRMService.java @@ -20,10 +20,7 @@ import java.io.IOException; import java.net.InetSocketAddress; -import java.util.ArrayList; -import java.util.Collection; import java.util.Collections; -import java.util.List; import java.util.Map; import org.apache.hadoop.classification.InterfaceAudience.Private; @@ -33,8 +30,6 @@ import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.security.authorize.PolicyProvider; import org.apache.hadoop.service.AbstractService; -import org.apache.hadoop.util.ReflectionUtils; -import org.apache.hadoop.util.StringUtils; import org.apache.hadoop.yarn.api.ApplicationClientProtocol; import org.apache.hadoop.yarn.api.protocolrecords.CancelDelegationTokenRequest; import org.apache.hadoop.yarn.api.protocolrecords.CancelDelegationTokenResponse; @@ -108,8 +103,8 @@ import org.apache.hadoop.yarn.api.protocolrecords.UpdateApplicationTimeoutsResponse; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.exceptions.YarnException; -import org.apache.hadoop.yarn.exceptions.YarnRuntimeException; import org.apache.hadoop.yarn.ipc.YarnRPC; +import org.apache.hadoop.yarn.server.router.RouterServerUtil; import org.apache.hadoop.yarn.server.router.security.authorize.RouterPolicyProvider; import org.apache.hadoop.yarn.util.LRUCacheHashMap; import org.slf4j.Logger; @@ -147,7 +142,7 @@ public RouterClientRMService() { @Override protected void serviceStart() throws Exception { - LOG.info("Starting Router ClientRMService"); + LOG.info("Starting Router ClientRMService."); Configuration conf = getConfig(); YarnRPC rpc = YarnRPC.create(conf); UserGroupInformation.setConfiguration(conf); @@ -161,9 +156,7 @@ protected void serviceStart() throws Exception { int maxCacheSize = conf.getInt(YarnConfiguration.ROUTER_PIPELINE_CACHE_MAX_SIZE, YarnConfiguration.DEFAULT_ROUTER_PIPELINE_CACHE_MAX_SIZE); - this.userPipelineMap = Collections.synchronizedMap( - new LRUCacheHashMap( - maxCacheSize, true)); + this.userPipelineMap = Collections.synchronizedMap(new LRUCacheHashMap<>(maxCacheSize, true)); Configuration serverConf = new Configuration(conf); @@ -181,14 +174,13 @@ protected void serviceStart() throws Exception { } this.server.start(); - LOG.info("Router ClientRMService listening on address: " - + this.server.getListenerAddress()); + LOG.info("Router ClientRMService listening on address: {}.", this.server.getListenerAddress()); super.serviceStart(); } @Override protected void serviceStop() throws Exception { - LOG.info("Stopping Router ClientRMService"); + LOG.info("Stopping Router ClientRMService."); if (this.server != null) { this.server.stop(); } @@ -201,27 +193,6 @@ public Server getServer() { return this.server; } - /** - * Returns the comma separated intercepter class names from the configuration. - * - * @param conf - * @return the intercepter class names as an instance of ArrayList - */ - private List getInterceptorClassNames(Configuration conf) { - String configuredInterceptorClassNames = - conf.get(YarnConfiguration.ROUTER_CLIENTRM_INTERCEPTOR_CLASS_PIPELINE, - YarnConfiguration.DEFAULT_ROUTER_CLIENTRM_INTERCEPTOR_CLASS); - - List interceptorClassNames = new ArrayList(); - Collection tempList = - StringUtils.getStringCollection(configuredInterceptorClassNames); - for (String item : tempList) { - interceptorClassNames.add(item.trim()); - } - - return interceptorClassNames; - } - @Override public GetNewApplicationResponse getNewApplication( GetNewApplicationRequest request) throws YarnException, IOException { @@ -473,7 +444,7 @@ public GetNodesToAttributesResponse getNodesToAttributes( } @VisibleForTesting - protected RequestInterceptorChainWrapper getInterceptorChain() + public RequestInterceptorChainWrapper getInterceptorChain() throws IOException { String user = UserGroupInformation.getCurrentUser().getUserName(); RequestInterceptorChainWrapper chain = userPipelineMap.get(user); @@ -489,9 +460,9 @@ void refreshServiceAcls(Configuration configuration, } /** - * Gets the Request intercepter chains for all the users. + * Gets the Request interceptor chains for all the users. * - * @return the request intercepter chains. + * @return the request interceptor chains. */ @VisibleForTesting protected Map getPipelines() { @@ -499,56 +470,22 @@ protected Map getPipelines() { } /** - * This method creates and returns reference of the first intercepter in the - * chain of request intercepter instances. + * This method creates and returns reference of the first interceptor in the + * chain of request interceptor instances. * - * @return the reference of the first intercepter in the chain + * @return the reference of the first interceptor in the chain */ @VisibleForTesting protected ClientRequestInterceptor createRequestInterceptorChain() { Configuration conf = getConfig(); - - List interceptorClassNames = getInterceptorClassNames(conf); - - ClientRequestInterceptor pipeline = null; - ClientRequestInterceptor current = null; - for (String interceptorClassName : interceptorClassNames) { - try { - Class interceptorClass = conf.getClassByName(interceptorClassName); - if (ClientRequestInterceptor.class.isAssignableFrom(interceptorClass)) { - ClientRequestInterceptor interceptorInstance = - (ClientRequestInterceptor) ReflectionUtils - .newInstance(interceptorClass, conf); - if (pipeline == null) { - pipeline = interceptorInstance; - current = interceptorInstance; - continue; - } else { - current.setNextInterceptor(interceptorInstance); - current = interceptorInstance; - } - } else { - throw new YarnRuntimeException( - "Class: " + interceptorClassName + " not instance of " - + ClientRequestInterceptor.class.getCanonicalName()); - } - } catch (ClassNotFoundException e) { - throw new YarnRuntimeException( - "Could not instantiate ApplicationClientRequestInterceptor: " - + interceptorClassName, - e); - } - } - - if (pipeline == null) { - throw new YarnRuntimeException( - "RequestInterceptor pipeline is not configured in the system"); - } - return pipeline; + return RouterServerUtil.createRequestInterceptorChain(conf, + YarnConfiguration.ROUTER_CLIENTRM_INTERCEPTOR_CLASS_PIPELINE, + YarnConfiguration.DEFAULT_ROUTER_CLIENTRM_INTERCEPTOR_CLASS, + ClientRequestInterceptor.class); } /** - * Initializes the request intercepter pipeline for the specified application. + * Initializes the request interceptor pipeline for the specified application. * * @param user */ @@ -565,15 +502,15 @@ private RequestInterceptorChainWrapper initializePipeline(String user) { try { // We should init the pipeline instance after it is created and then // add to the map, to ensure thread safe. - LOG.info("Initializing request processing pipeline for application " - + "for the user: {}", user); + LOG.info("Initializing request processing pipeline for application for the user: {}.", + user); ClientRequestInterceptor interceptorChain = this.createRequestInterceptorChain(); interceptorChain.init(user); chainWrapper.init(interceptorChain); } catch (Exception e) { - LOG.error("Init ClientRequestInterceptor error for user: " + user, e); + LOG.error("Init ClientRequestInterceptor error for user: {}.", user, e); throw e; } @@ -600,9 +537,9 @@ public synchronized void init(ClientRequestInterceptor interceptor) { } /** - * Gets the root request intercepter. + * Gets the root request interceptor. * - * @return the root request intercepter + * @return the root request interceptor */ public synchronized ClientRequestInterceptor getRootInterceptor() { return rootInterceptor; @@ -616,4 +553,9 @@ protected void finalize() { rootInterceptor.shutdown(); } } + + @VisibleForTesting + public Map getUserPipelineMap() { + return userPipelineMap; + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/clientrm/RouterYarnClientUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/clientrm/RouterYarnClientUtils.java index 46915738c965a..e70d5521ffcf7 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/clientrm/RouterYarnClientUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/clientrm/RouterYarnClientUtils.java @@ -34,6 +34,12 @@ import org.apache.hadoop.yarn.api.protocolrecords.GetQueueUserAclsInfoResponse; import org.apache.hadoop.yarn.api.protocolrecords.ReservationListResponse; import org.apache.hadoop.yarn.api.protocolrecords.GetAllResourceTypeInfoResponse; +import org.apache.hadoop.yarn.api.protocolrecords.GetQueueInfoResponse; +import org.apache.hadoop.yarn.api.protocolrecords.GetAllResourceProfilesResponse; +import org.apache.hadoop.yarn.api.protocolrecords.GetResourceProfileResponse; +import org.apache.hadoop.yarn.api.protocolrecords.GetAttributesToNodesResponse; +import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodeAttributesResponse; +import org.apache.hadoop.yarn.api.protocolrecords.GetNodesToAttributesResponse; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ApplicationReport; import org.apache.hadoop.yarn.api.records.ApplicationResourceUsageReport; @@ -44,6 +50,12 @@ import org.apache.hadoop.yarn.api.records.QueueUserACLInfo; import org.apache.hadoop.yarn.api.records.ReservationAllocationState; import org.apache.hadoop.yarn.api.records.ResourceTypeInfo; +import org.apache.hadoop.yarn.api.records.QueueInfo; +import org.apache.hadoop.yarn.api.records.Resource; +import org.apache.hadoop.yarn.api.records.NodeAttributeKey; +import org.apache.hadoop.yarn.api.records.NodeToAttributeValue; +import org.apache.hadoop.yarn.api.records.NodeAttribute; +import org.apache.hadoop.yarn.api.records.NodeAttributeInfo; import org.apache.hadoop.yarn.server.uam.UnmanagedApplicationManager; import org.apache.hadoop.yarn.util.Records; import org.apache.hadoop.yarn.util.resource.Resources; @@ -365,5 +377,153 @@ public static GetAllResourceTypeInfoResponse mergeResourceTypes( new ArrayList<>(resourceTypeInfoSet)); return resourceTypeInfoResponse; } + + /** + * Merges a list of GetQueueInfoResponse. + * + * @param responses a list of GetQueueInfoResponse to merge. + * @return the merged GetQueueInfoResponse. + */ + public static GetQueueInfoResponse mergeQueues( + Collection responses) { + GetQueueInfoResponse queueResponse = Records.newRecord( + GetQueueInfoResponse.class); + + QueueInfo queueInfo = null; + for (GetQueueInfoResponse response : responses) { + if (response != null && response.getQueueInfo() != null) { + if (queueInfo == null) { + queueInfo = response.getQueueInfo(); + } else { + // set Capacity\MaximumCapacity\CurrentCapacity + queueInfo.setCapacity(queueInfo.getCapacity() + response.getQueueInfo().getCapacity()); + queueInfo.setMaximumCapacity( + queueInfo.getMaximumCapacity() + response.getQueueInfo().getMaximumCapacity()); + queueInfo.setCurrentCapacity( + queueInfo.getCurrentCapacity() + response.getQueueInfo().getCurrentCapacity()); + + // set childQueues + List childQueues = new ArrayList<>(queueInfo.getChildQueues()); + childQueues.addAll(response.getQueueInfo().getChildQueues()); + queueInfo.setChildQueues(childQueues); + + // set applications + List applicationReports = new ArrayList<>(queueInfo.getApplications()); + applicationReports.addAll(response.getQueueInfo().getApplications()); + queueInfo.setApplications(applicationReports); + + // set accessibleNodeLabels + Set accessibleNodeLabels = new HashSet<>(); + if (queueInfo.getAccessibleNodeLabels() != null) { + accessibleNodeLabels.addAll(queueInfo.getAccessibleNodeLabels()); + } + if (response.getQueueInfo() != null) { + accessibleNodeLabels.addAll(response.getQueueInfo().getAccessibleNodeLabels()); + } + queueInfo.setAccessibleNodeLabels(accessibleNodeLabels); + } + } + } + queueResponse.setQueueInfo(queueInfo); + return queueResponse; + } + + /** + * Merges a list of GetAllResourceProfilesResponse. + * + * @param responses a list of GetAllResourceProfilesResponse to merge. + * @return the merged GetAllResourceProfilesResponse. + */ + public static GetAllResourceProfilesResponse mergeClusterResourceProfilesResponse( + Collection responses) { + GetAllResourceProfilesResponse profilesResponse = + Records.newRecord(GetAllResourceProfilesResponse.class); + Map profilesMap = new HashMap<>(); + for (GetAllResourceProfilesResponse response : responses) { + if (response != null && response.getResourceProfiles() != null) { + for (Map.Entry entry : response.getResourceProfiles().entrySet()) { + String key = entry.getKey(); + Resource r1 = profilesMap.getOrDefault(key, null); + Resource r2 = entry.getValue(); + Resource rAdd = r1 == null ? r2 : Resources.add(r1, r2); + profilesMap.put(key, rAdd); + } + } + } + profilesResponse.setResourceProfiles(profilesMap); + return profilesResponse; + } + + /** + * Merges a list of GetResourceProfileResponse. + * + * @param responses a list of GetResourceProfileResponse to merge. + * @return the merged GetResourceProfileResponse. + */ + public static GetResourceProfileResponse mergeClusterResourceProfileResponse( + Collection responses) { + GetResourceProfileResponse profileResponse = + Records.newRecord(GetResourceProfileResponse.class); + Resource resource = Resource.newInstance(0, 0); + for (GetResourceProfileResponse response : responses) { + if (response != null && response.getResource() != null) { + Resource responseResource = response.getResource(); + resource = Resources.add(resource, responseResource); + } + } + profileResponse.setResource(resource); + return profileResponse; + } + + /** + * Merges a list of GetAttributesToNodesResponse. + * + * @param responses a list of GetAttributesToNodesResponse to merge. + * @return the merged GetAttributesToNodesResponse. + */ + public static GetAttributesToNodesResponse mergeAttributesToNodesResponse( + Collection responses) { + Map> nodeAttributeMap = new HashMap<>(); + for (GetAttributesToNodesResponse response : responses) { + if (response != null && response.getAttributesToNodes() != null) { + nodeAttributeMap.putAll(response.getAttributesToNodes()); + } + } + return GetAttributesToNodesResponse.newInstance(nodeAttributeMap); + } + + /** + * Merges a list of GetClusterNodeAttributesResponse. + * + * @param responses a list of GetClusterNodeAttributesResponse to merge. + * @return the merged GetClusterNodeAttributesResponse. + */ + public static GetClusterNodeAttributesResponse mergeClusterNodeAttributesResponse( + Collection responses) { + Set nodeAttributeInfo = new HashSet<>(); + for (GetClusterNodeAttributesResponse response : responses) { + if (response != null && response.getNodeAttributes() != null) { + nodeAttributeInfo.addAll(response.getNodeAttributes()); + } + } + return GetClusterNodeAttributesResponse.newInstance(nodeAttributeInfo); + } + + /** + * Merges a list of GetNodesToAttributesResponse. + * + * @param responses a list of GetNodesToAttributesResponse to merge. + * @return the merged GetNodesToAttributesResponse. + */ + public static GetNodesToAttributesResponse mergeNodesToAttributesResponse( + Collection responses) { + Map> attributesMap = new HashMap<>(); + for (GetNodesToAttributesResponse response : responses) { + if (response != null && response.getNodeToAttributes() != null) { + attributesMap.putAll(response.getNodeToAttributes()); + } + } + return GetNodesToAttributesResponse.newInstance(attributesMap); + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/rmadmin/AbstractRMAdminRequestInterceptor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/rmadmin/AbstractRMAdminRequestInterceptor.java index a4972fcb9ad84..f789aa2b47e47 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/rmadmin/AbstractRMAdminRequestInterceptor.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/rmadmin/AbstractRMAdminRequestInterceptor.java @@ -23,7 +23,7 @@ /** * Implements the {@link RMAdminRequestInterceptor} interface and provides * common functionality which can can be used and/or extended by other concrete - * intercepter classes. + * interceptor classes. * */ public abstract class AbstractRMAdminRequestInterceptor diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/rmadmin/DefaultRMAdminRequestInterceptor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/rmadmin/DefaultRMAdminRequestInterceptor.java index 9625eb4d3662c..122782aef47ad 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/rmadmin/DefaultRMAdminRequestInterceptor.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/rmadmin/DefaultRMAdminRequestInterceptor.java @@ -80,8 +80,9 @@ public void init(String userName) { try { // Do not create a proxy user if user name matches the user name on // current UGI - if (userName.equalsIgnoreCase( - UserGroupInformation.getCurrentUser().getUserName())) { + if (UserGroupInformation.isSecurityEnabled()) { + user = UserGroupInformation.createProxyUser(userName, UserGroupInformation.getLoginUser()); + } else if (userName.equalsIgnoreCase(UserGroupInformation.getCurrentUser().getUserName())) { user = UserGroupInformation.getCurrentUser(); } else { user = UserGroupInformation.createProxyUser(userName, diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/rmadmin/RMAdminRequestInterceptor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/rmadmin/RMAdminRequestInterceptor.java index dc4bda01b909c..29ab9e6add2cd 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/rmadmin/RMAdminRequestInterceptor.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/rmadmin/RMAdminRequestInterceptor.java @@ -22,14 +22,14 @@ import org.apache.hadoop.yarn.server.api.ResourceManagerAdministrationProtocol; /** - * Defines the contract to be implemented by the request intercepter classes, + * Defines the contract to be implemented by the request interceptor classes, * that can be used to intercept and inspect messages sent from the client to * the resource manager. */ public interface RMAdminRequestInterceptor extends ResourceManagerAdministrationProtocol, Configurable { /** - * This method is called for initializing the intercepter. This is guaranteed + * This method is called for initializing the interceptor. This is guaranteed * to be called only once in the lifetime of this instance. * * @param user the name of the client @@ -37,28 +37,28 @@ public interface RMAdminRequestInterceptor void init(String user); /** - * This method is called to release the resources held by the intercepter. + * This method is called to release the resources held by the interceptor. * This will be called when the application pipeline is being destroyed. The * concrete implementations should dispose the resources and forward the - * request to the next intercepter, if any. + * request to the next interceptor, if any. */ void shutdown(); /** - * Sets the next intercepter in the pipeline. The concrete implementation of + * Sets the next interceptor in the pipeline. The concrete implementation of * this interface should always pass the request to the nextInterceptor after - * inspecting the message. The last intercepter in the chain is responsible to + * inspecting the message. The last interceptor in the chain is responsible to * send the messages to the resource manager service and so the last - * intercepter will not receive this method call. + * interceptor will not receive this method call. * * @param nextInterceptor the RMAdminRequestInterceptor to set in the pipeline */ void setNextInterceptor(RMAdminRequestInterceptor nextInterceptor); /** - * Returns the next intercepter in the chain. + * Returns the next interceptor in the chain. * - * @return the next intercepter in the chain + * @return the next interceptor in the chain */ RMAdminRequestInterceptor getNextInterceptor(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/rmadmin/RouterRMAdminService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/rmadmin/RouterRMAdminService.java index e05de7a899488..8024cdb82f126 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/rmadmin/RouterRMAdminService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/rmadmin/RouterRMAdminService.java @@ -20,10 +20,7 @@ import java.io.IOException; import java.net.InetSocketAddress; -import java.util.ArrayList; -import java.util.Collection; import java.util.Collections; -import java.util.List; import java.util.Map; import org.apache.hadoop.classification.InterfaceAudience.Private; @@ -34,11 +31,8 @@ import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.security.authorize.PolicyProvider; import org.apache.hadoop.service.AbstractService; -import org.apache.hadoop.util.ReflectionUtils; -import org.apache.hadoop.util.StringUtils; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.exceptions.YarnException; -import org.apache.hadoop.yarn.exceptions.YarnRuntimeException; import org.apache.hadoop.yarn.ipc.YarnRPC; import org.apache.hadoop.yarn.server.api.ResourceManagerAdministrationProtocol; import org.apache.hadoop.yarn.server.api.protocolrecords.AddToClusterNodeLabelsRequest; @@ -69,6 +63,7 @@ import org.apache.hadoop.yarn.server.api.protocolrecords.ReplaceLabelsOnNodeResponse; import org.apache.hadoop.yarn.server.api.protocolrecords.UpdateNodeResourceRequest; import org.apache.hadoop.yarn.server.api.protocolrecords.UpdateNodeResourceResponse; +import org.apache.hadoop.yarn.server.router.RouterServerUtil; import org.apache.hadoop.yarn.server.router.security.authorize.RouterPolicyProvider; import org.apache.hadoop.yarn.util.LRUCacheHashMap; import org.slf4j.Logger; @@ -82,7 +77,7 @@ * messages from client to the cluster resource manager. It listens * {@code ResourceManagerAdministrationProtocol} messages from the client and * creates a request intercepting pipeline instance for each client. The - * pipeline is a chain of intercepter instances that can inspect and modify the + * pipeline is a chain of interceptor instances that can inspect and modify the * request/response as needed. The main difference with AMRMProxyService is the * protocol they implement. */ @@ -106,7 +101,7 @@ public RouterRMAdminService() { @Override protected void serviceStart() throws Exception { - LOG.info("Starting Router RMAdmin Service"); + LOG.info("Starting Router RMAdmin Service."); Configuration conf = getConfig(); YarnRPC rpc = YarnRPC.create(conf); UserGroupInformation.setConfiguration(conf); @@ -120,9 +115,7 @@ protected void serviceStart() throws Exception { int maxCacheSize = conf.getInt(YarnConfiguration.ROUTER_PIPELINE_CACHE_MAX_SIZE, YarnConfiguration.DEFAULT_ROUTER_PIPELINE_CACHE_MAX_SIZE); - this.userPipelineMap = Collections.synchronizedMap( - new LRUCacheHashMap( - maxCacheSize, true)); + this.userPipelineMap = Collections.synchronizedMap(new LRUCacheHashMap<>(maxCacheSize, true)); Configuration serverConf = new Configuration(conf); @@ -139,14 +132,13 @@ protected void serviceStart() throws Exception { } this.server.start(); - LOG.info("Router RMAdminService listening on address: " - + this.server.getListenerAddress()); + LOG.info("Router RMAdminService listening on address: {}.", this.server.getListenerAddress()); super.serviceStart(); } @Override protected void serviceStop() throws Exception { - LOG.info("Stopping Router RMAdminService"); + LOG.info("Stopping Router RMAdminService."); if (this.server != null) { this.server.stop(); } @@ -164,29 +156,8 @@ public Server getServer() { return this.server; } - /** - * Returns the comma separated intercepter class names from the configuration. - * - * @param conf - * @return the intercepter class names as an instance of ArrayList - */ - private List getInterceptorClassNames(Configuration conf) { - String configuredInterceptorClassNames = - conf.get(YarnConfiguration.ROUTER_RMADMIN_INTERCEPTOR_CLASS_PIPELINE, - YarnConfiguration.DEFAULT_ROUTER_RMADMIN_INTERCEPTOR_CLASS); - - List interceptorClassNames = new ArrayList(); - Collection tempList = - StringUtils.getStringCollection(configuredInterceptorClassNames); - for (String item : tempList) { - interceptorClassNames.add(item.trim()); - } - - return interceptorClassNames; - } - @VisibleForTesting - protected RequestInterceptorChainWrapper getInterceptorChain() + public RequestInterceptorChainWrapper getInterceptorChain() throws IOException { String user = UserGroupInformation.getCurrentUser().getUserName(); RequestInterceptorChainWrapper chain = userPipelineMap.get(user); @@ -197,9 +168,9 @@ protected RequestInterceptorChainWrapper getInterceptorChain() } /** - * Gets the Request intercepter chains for all the users. + * Gets the Request interceptor chains for all the users. * - * @return the request intercepter chains. + * @return the request interceptor chains. */ @VisibleForTesting protected Map getPipelines() { @@ -207,57 +178,22 @@ protected Map getPipelines() { } /** - * This method creates and returns reference of the first intercepter in the - * chain of request intercepter instances. + * This method creates and returns reference of the first interceptor in the + * chain of request interceptor instances. * - * @return the reference of the first intercepter in the chain + * @return the reference of the first interceptor in the chain */ @VisibleForTesting protected RMAdminRequestInterceptor createRequestInterceptorChain() { Configuration conf = getConfig(); - - List interceptorClassNames = getInterceptorClassNames(conf); - - RMAdminRequestInterceptor pipeline = null; - RMAdminRequestInterceptor current = null; - for (String interceptorClassName : interceptorClassNames) { - try { - Class interceptorClass = conf.getClassByName(interceptorClassName); - if (RMAdminRequestInterceptor.class - .isAssignableFrom(interceptorClass)) { - RMAdminRequestInterceptor interceptorInstance = - (RMAdminRequestInterceptor) ReflectionUtils - .newInstance(interceptorClass, conf); - if (pipeline == null) { - pipeline = interceptorInstance; - current = interceptorInstance; - continue; - } else { - current.setNextInterceptor(interceptorInstance); - current = interceptorInstance; - } - } else { - throw new YarnRuntimeException( - "Class: " + interceptorClassName + " not instance of " - + RMAdminRequestInterceptor.class.getCanonicalName()); - } - } catch (ClassNotFoundException e) { - throw new YarnRuntimeException( - "Could not instantiate RMAdminRequestInterceptor: " - + interceptorClassName, - e); - } - } - - if (pipeline == null) { - throw new YarnRuntimeException( - "RequestInterceptor pipeline is not configured in the system"); - } - return pipeline; + return RouterServerUtil.createRequestInterceptorChain(conf, + YarnConfiguration.ROUTER_RMADMIN_INTERCEPTOR_CLASS_PIPELINE, + YarnConfiguration.DEFAULT_ROUTER_RMADMIN_INTERCEPTOR_CLASS, + RMAdminRequestInterceptor.class); } /** - * Initializes the request intercepter pipeline for the specified user. + * Initializes the request interceptor pipeline for the specified user. * * @param user */ @@ -274,14 +210,14 @@ private RequestInterceptorChainWrapper initializePipeline(String user) { try { // We should init the pipeline instance after it is created and then // add to the map, to ensure thread safe. - LOG.info("Initializing request processing pipeline for user: {}", user); + LOG.info("Initializing request processing pipeline for user: {}.", user); RMAdminRequestInterceptor interceptorChain = this.createRequestInterceptorChain(); interceptorChain.init(user); chainWrapper.init(interceptorChain); } catch (Exception e) { - LOG.error("Init RMAdminRequestInterceptor error for user: " + user, e); + LOG.error("Init RMAdminRequestInterceptor error for user: {}.", user, e); throw e; } @@ -308,9 +244,9 @@ public synchronized void init(RMAdminRequestInterceptor interceptor) { } /** - * Gets the root request intercepter. + * Gets the root request interceptor. * - * @return the root request intercepter + * @return the root request interceptor */ public synchronized RMAdminRequestInterceptor getRootInterceptor() { return rootInterceptor; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/AbstractRESTRequestInterceptor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/AbstractRESTRequestInterceptor.java index a2d78a479ab78..f1919c2e5a174 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/AbstractRESTRequestInterceptor.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/AbstractRESTRequestInterceptor.java @@ -22,7 +22,7 @@ /** * Extends the RequestInterceptor class and provides common functionality which - * can be used and/or extended by other concrete intercepter classes. + * can be used and/or extended by other concrete interceptor classes. */ public abstract class AbstractRESTRequestInterceptor implements RESTRequestInterceptor { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/DefaultRequestInterceptorREST.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/DefaultRequestInterceptorREST.java index 21fd2be854618..c07056ce8a100 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/DefaultRequestInterceptorREST.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/DefaultRequestInterceptorREST.java @@ -66,6 +66,7 @@ import org.apache.hadoop.yarn.server.webapp.dao.AppAttemptInfo; import org.apache.hadoop.yarn.server.webapp.dao.ContainerInfo; import org.apache.hadoop.yarn.server.webapp.dao.ContainersInfo; +import org.apache.hadoop.yarn.webapp.dao.SchedConfUpdateInfo; import org.apache.hadoop.yarn.webapp.util.WebAppUtils; /** @@ -565,6 +566,25 @@ public ContainerInfo getContainer(HttpServletRequest req, null, null, getConf(), client); } + @Override + public Response updateSchedulerConfiguration(SchedConfUpdateInfo mutationInfo, + HttpServletRequest req) + throws AuthorizationException, InterruptedException { + return RouterWebServiceUtil.genericForward(webAppAddress, req, + Response.class, HTTPMethods.PUT, + RMWSConsts.RM_WEB_SERVICE_PATH + RMWSConsts.SCHEDULER_CONF, + mutationInfo, null, getConf(), client); + } + + @Override + public Response getSchedulerConfiguration(HttpServletRequest req) + throws AuthorizationException { + return RouterWebServiceUtil.genericForward(webAppAddress, req, + Response.class, HTTPMethods.GET, + RMWSConsts.RM_WEB_SERVICE_PATH + RMWSConsts.SCHEDULER_CONF, + null, null, getConf(), client); + } + @Override public void setNextInterceptor(RESTRequestInterceptor next) { throw new YarnRuntimeException("setNextInterceptor is being called on " diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/FederationInterceptorREST.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/FederationInterceptorREST.java index db2a8edcb2cdb..4756da4fa8015 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/FederationInterceptorREST.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/FederationInterceptorREST.java @@ -19,6 +19,7 @@ package org.apache.hadoop.yarn.server.router.webapp; import java.io.IOException; +import java.lang.reflect.Method; import java.security.Principal; import java.util.ArrayList; import java.util.Collection; @@ -33,6 +34,7 @@ import java.util.concurrent.ExecutorCompletionService; import java.util.concurrent.ExecutorService; import java.util.concurrent.Future; +import java.util.stream.Collectors; import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletRequestWrapper; @@ -45,9 +47,13 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.security.authorize.AuthorizationException; import org.apache.hadoop.util.ReflectionUtils; +import org.apache.hadoop.util.Sets; import org.apache.hadoop.util.concurrent.HadoopExecutors; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext; +import org.apache.hadoop.yarn.api.records.ContainerId; +import org.apache.hadoop.yarn.api.records.NodeLabel; +import org.apache.hadoop.yarn.api.records.ReservationId; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.exceptions.YarnRuntimeException; @@ -59,6 +65,7 @@ import org.apache.hadoop.yarn.server.federation.store.records.SubClusterId; import org.apache.hadoop.yarn.server.federation.store.records.SubClusterInfo; import org.apache.hadoop.yarn.server.federation.utils.FederationStateStoreFacade; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.NodeIDsInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.RMWebAppUtil; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ActivitiesInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppActivitiesInfo; @@ -90,11 +97,16 @@ import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ResourceOptionInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.BulkActivitiesInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.SchedulerTypeInfo; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.NodeLabelInfo; import org.apache.hadoop.yarn.server.router.RouterMetrics; import org.apache.hadoop.yarn.server.router.RouterServerUtil; +import org.apache.hadoop.yarn.server.router.clientrm.ClientMethod; +import org.apache.hadoop.yarn.server.router.webapp.cache.RouterAppInfoCacheKey; import org.apache.hadoop.yarn.server.webapp.dao.AppAttemptInfo; import org.apache.hadoop.yarn.server.webapp.dao.ContainerInfo; import org.apache.hadoop.yarn.server.webapp.dao.ContainersInfo; +import org.apache.hadoop.yarn.util.LRUCacheHashMap; +import org.apache.hadoop.yarn.webapp.dao.SchedConfUpdateInfo; import org.apache.hadoop.yarn.util.Clock; import org.apache.hadoop.yarn.util.MonotonicClock; import org.apache.hadoop.yarn.webapp.NotFoundException; @@ -109,7 +121,7 @@ * Extends the {@code AbstractRESTRequestInterceptor} class and provides an * implementation for federation of YARN RM and scaling an application across * multiple YARN SubClusters. All the federation specific implementation is - * encapsulated in this class. This is always the last intercepter in the chain. + * encapsulated in this class. This is always the last interceptor in the chain. */ public class FederationInterceptorREST extends AbstractRESTRequestInterceptor { @@ -123,8 +135,11 @@ public class FederationInterceptorREST extends AbstractRESTRequestInterceptor { private RouterMetrics routerMetrics; private final Clock clock = new MonotonicClock(); private boolean returnPartialReport; + private boolean appInfosCacheEnabled; + private int appInfosCacheCount; private Map interceptors; + private LRUCacheHashMap appInfosCaches; /** * Thread pool used for asynchronous operations. @@ -161,6 +176,17 @@ public void init(String user) { returnPartialReport = conf.getBoolean( YarnConfiguration.ROUTER_WEBAPP_PARTIAL_RESULTS_ENABLED, YarnConfiguration.DEFAULT_ROUTER_WEBAPP_PARTIAL_RESULTS_ENABLED); + + appInfosCacheEnabled = conf.getBoolean( + YarnConfiguration.ROUTER_APPSINFO_ENABLED, + YarnConfiguration.DEFAULT_ROUTER_APPSINFO_ENABLED); + + if(appInfosCacheEnabled) { + appInfosCacheCount = conf.getInt( + YarnConfiguration.ROUTER_APPSINFO_CACHED_COUNT, + YarnConfiguration.DEFAULT_ROUTER_APPSINFO_CACHED_COUNT); + appInfosCaches = new LRUCacheHashMap<>(appInfosCacheCount, true); + } } private SubClusterId getRandomActiveSubCluster( @@ -171,19 +197,13 @@ private SubClusterId getRandomActiveSubCluster( RouterServerUtil.logAndThrowException( FederationPolicyUtils.NO_ACTIVE_SUBCLUSTER_AVAILABLE, null); } - List list = new ArrayList<>(activeSubclusters.keySet()); - - FederationPolicyUtils.validateSubClusterAvailability( - list, blackListSubClusters); - + Collection keySet = activeSubclusters.keySet(); + FederationPolicyUtils.validateSubClusterAvailability(keySet, blackListSubClusters); if (blackListSubClusters != null) { - - // Remove from the active SubClusters from StateStore the blacklisted ones - for (SubClusterId scId : blackListSubClusters) { - list.remove(scId); - } + keySet.removeAll(blackListSubClusters); } + List list = keySet.stream().collect(Collectors.toList()); return list.get(rand.nextInt(list.size())); } @@ -280,7 +300,7 @@ public Response createNewApplication(HttpServletRequest hsr) .entity(e.getLocalizedMessage()).build(); } - List blacklist = new ArrayList(); + List blacklist = new ArrayList<>(); for (int i = 0; i < numSubmitRetries; ++i) { @@ -293,7 +313,7 @@ public Response createNewApplication(HttpServletRequest hsr) .entity(e.getLocalizedMessage()).build(); } - LOG.debug("getNewApplication try #{} on SubCluster {}", i, subClusterId); + LOG.debug("getNewApplication try #{} on SubCluster {}.", i, subClusterId); DefaultRequestInterceptorREST interceptor = getOrCreateInterceptorForSubCluster(subClusterId, @@ -302,7 +322,7 @@ public Response createNewApplication(HttpServletRequest hsr) try { response = interceptor.createNewApplication(hsr); } catch (Exception e) { - LOG.warn("Unable to create a new ApplicationId in SubCluster {}", + LOG.warn("Unable to create a new ApplicationId in SubCluster {}.", subClusterId.getId(), e); } @@ -404,7 +424,7 @@ public Response submitApplication(ApplicationSubmissionContextInfo newApp, if (newApp == null || newApp.getApplicationId() == null) { routerMetrics.incrAppsFailedSubmitted(); String errMsg = "Missing ApplicationSubmissionContextInfo or " - + "applicationSubmissionContex information."; + + "applicationSubmissionContext information."; return Response .status(Status.BAD_REQUEST) .entity(errMsg) @@ -422,7 +442,7 @@ public Response submitApplication(ApplicationSubmissionContextInfo newApp, .build(); } - List blacklist = new ArrayList(); + List blacklist = new ArrayList<>(); for (int i = 0; i < numSubmitRetries; ++i) { @@ -439,7 +459,7 @@ public Response submitApplication(ApplicationSubmissionContextInfo newApp, .entity(e.getLocalizedMessage()) .build(); } - LOG.info("submitApplication appId {} try #{} on SubCluster {}", + LOG.info("submitApplication appId {} try #{} on SubCluster {}.", applicationId, i, subClusterId); ApplicationHomeSubCluster appHomeSubCluster = @@ -480,7 +500,7 @@ public Response submitApplication(ApplicationSubmissionContextInfo newApp, .build(); } if (subClusterId == subClusterIdInStateStore) { - LOG.info("Application {} already submitted on SubCluster {}", + LOG.info("Application {} already submitted on SubCluster {}.", applicationId, subClusterId); } else { routerMetrics.incrAppsFailedSubmitted(); @@ -665,7 +685,7 @@ public Response updateAppState(AppState targetState, HttpServletRequest hsr, *

    * ResourceManager: the Router calls each YARN RM in parallel by using one * thread for each YARN RM. In case a YARN RM fails, a single call will - * timeout. However the Router will merge the ApplicationReports it got, and + * timeout. However, the Router will merge the ApplicationReports it got, and * provides a partial list to the client. *

    * State Store: the Router will timeout and it will retry depending on the @@ -678,6 +698,18 @@ public AppsInfo getApps(HttpServletRequest hsr, String stateQuery, String queueQuery, String count, String startedBegin, String startedEnd, String finishBegin, String finishEnd, Set applicationTypes, Set applicationTags, String name, Set unselectedFields) { + + RouterAppInfoCacheKey routerAppInfoCacheKey = RouterAppInfoCacheKey.newInstance( + hsr, stateQuery, statesQuery, finalStatusQuery, userQuery, queueQuery, count, + startedBegin, startedEnd, finishBegin, finishEnd, applicationTypes, + applicationTags, name, unselectedFields); + + if (appInfosCacheEnabled && routerAppInfoCacheKey != null) { + if (appInfosCaches.containsKey(routerAppInfoCacheKey)) { + return appInfosCaches.get(routerAppInfoCacheKey); + } + } + AppsInfo apps = new AppsInfo(); long startTime = clock.getTime(); @@ -710,8 +742,7 @@ public AppsInfo call() { if (rmApps == null) { routerMetrics.incrMultipleAppsFailedRetrieved(); - LOG.error("Subcluster {} failed to return appReport.", - info.getSubClusterId()); + LOG.error("Subcluster {} failed to return appReport.", info.getSubClusterId()); return null; } return rmApps; @@ -742,8 +773,14 @@ public AppsInfo call() { } // Merge all the application reports got from all the available YARN RMs - return RouterWebServiceUtil.mergeAppsInfo( + AppsInfo resultAppsInfo = RouterWebServiceUtil.mergeAppsInfo( apps.getApps(), returnPartialReport); + + if (appInfosCacheEnabled && routerAppInfoCacheKey != null) { + appInfosCaches.put(routerAppInfoCacheKey, resultAppsInfo); + } + + return resultAppsInfo; } /** @@ -871,8 +908,7 @@ private Map getNode( subclusterId, subcluster.getRMWebServiceAddress()); return interceptor.getNode(nodeId); } catch (Exception e) { - LOG.error("Subcluster {} failed to return nodeInfo.", - subclusterId); + LOG.error("Subcluster {} failed to return nodeInfo.", subclusterId, e); return null; } }); @@ -940,7 +976,7 @@ private SubClusterInfo getNodeSubcluster(String nodeId) *

    * ResourceManager: the Router calls each YARN RM in parallel by using one * thread for each YARN RM. In case a YARN RM fails, a single call will - * timeout. However the Router will use the NodesInfo it got, and provides a + * timeout. However, the Router will use the NodesInfo it got, and provides a * partial list to the client. *

    * State Store: the Router will timeout and it will retry depending on the @@ -951,58 +987,28 @@ private SubClusterInfo getNodeSubcluster(String nodeId) public NodesInfo getNodes(String states) { NodesInfo nodes = new NodesInfo(); - - final Map subClustersActive; try { - subClustersActive = getActiveSubclusters(); - } catch (Exception e) { - LOG.error("Cannot get nodes: {}", e.getMessage()); - return new NodesInfo(); - } - - // Send the requests in parallel - CompletionService compSvc = - new ExecutorCompletionService(this.threadpool); - - for (final SubClusterInfo info : subClustersActive.values()) { - compSvc.submit(new Callable() { - @Override - public NodesInfo call() { - DefaultRequestInterceptorREST interceptor = - getOrCreateInterceptorForSubCluster( - info.getSubClusterId(), info.getRMWebServiceAddress()); - try { - NodesInfo nodesInfo = interceptor.getNodes(states); - return nodesInfo; - } catch (Exception e) { - LOG.error("Subcluster {} failed to return nodesInfo.", - info.getSubClusterId()); - return null; - } - } + Map subClustersActive = getActiveSubclusters(); + Class[] argsClasses = new Class[]{String.class}; + Object[] args = new Object[]{states}; + ClientMethod remoteMethod = new ClientMethod("getNodes", argsClasses, args); + Map nodesMap = + invokeConcurrent(subClustersActive.values(), remoteMethod, NodesInfo.class); + nodesMap.values().stream().forEach(nodesInfo -> { + nodes.addAll(nodesInfo.getNodes()); }); - } - - // Collect all the responses in parallel - - for (int i = 0; i < subClustersActive.size(); i++) { - try { - Future future = compSvc.take(); - NodesInfo nodesResponse = future.get(); - - if (nodesResponse != null) { - nodes.addAll(nodesResponse.getNodes()); - } - } catch (Throwable e) { - LOG.warn("Failed to get nodes report ", e); - } + } catch (NotFoundException e) { + LOG.error("Get all active sub cluster(s) error.", e); + } catch (YarnException e) { + LOG.error("getNodes error.", e); + } catch (IOException e) { + LOG.error("getNodes error with io error.", e); } // Delete duplicate from all the node reports got from all the available // YARN RMs. Nodes can be moved from one subclusters to another. In this // operation they result LOST/RUNNING in the previous SubCluster and // NEW/RUNNING in the new one. - return RouterWebServiceUtil.deleteDuplicateNodesInfo(nodes.getNodes()); } @@ -1053,7 +1059,6 @@ public ClusterMetricsInfo call() { } // Collect all the responses in parallel - for (int i = 0; i < subClustersActive.size(); i++) { try { Future future = compSvc.take(); @@ -1159,25 +1164,102 @@ public AppActivitiesInfo getAppActivities(HttpServletRequest hsr, String appId, String time, Set requestPriorities, Set allocationRequestIds, String groupBy, String limit, Set actions, boolean summarize) { - throw new NotImplementedException("Code is not implemented"); + + // Only verify the app_id, + // because the specific subCluster needs to be found according to the app_id, + // and other verifications are directly handed over to the corresponding subCluster RM + if (appId == null || appId.isEmpty()) { + throw new IllegalArgumentException("Parameter error, the appId is empty or null."); + } + + try { + SubClusterInfo subClusterInfo = getHomeSubClusterInfoByAppId(appId); + DefaultRequestInterceptorREST interceptor = getOrCreateInterceptorForSubCluster( + subClusterInfo.getSubClusterId(), subClusterInfo.getRMWebServiceAddress()); + + final HttpServletRequest hsrCopy = clone(hsr); + return interceptor.getAppActivities(hsrCopy, appId, time, requestPriorities, + allocationRequestIds, groupBy, limit, actions, summarize); + } catch (IllegalArgumentException e) { + RouterServerUtil.logAndThrowRunTimeException(e, "Unable to get subCluster by appId: %s.", + appId); + } catch (YarnException e) { + RouterServerUtil.logAndThrowRunTimeException("getAppActivities Failed.", e); + } + + return null; } @Override public ApplicationStatisticsInfo getAppStatistics(HttpServletRequest hsr, Set stateQueries, Set typeQueries) { - throw new NotImplementedException("Code is not implemented"); + try { + Map subClustersActive = getActiveSubclusters(); + final HttpServletRequest hsrCopy = clone(hsr); + Class[] argsClasses = new Class[]{HttpServletRequest.class, Set.class, Set.class}; + Object[] args = new Object[]{hsrCopy, stateQueries, typeQueries}; + ClientMethod remoteMethod = new ClientMethod("getAppStatistics", argsClasses, args); + Map appStatisticsMap = invokeConcurrent( + subClustersActive.values(), remoteMethod, ApplicationStatisticsInfo.class); + return RouterWebServiceUtil.mergeApplicationStatisticsInfo(appStatisticsMap.values()); + } catch (IOException e) { + RouterServerUtil.logAndThrowRunTimeException(e, "Get all active sub cluster(s) error."); + } catch (YarnException e) { + RouterServerUtil.logAndThrowRunTimeException(e, "getAppStatistics error."); + } + return null; } @Override public NodeToLabelsInfo getNodeToLabels(HttpServletRequest hsr) throws IOException { - throw new NotImplementedException("Code is not implemented"); + try { + Map subClustersActive = getActiveSubclusters(); + final HttpServletRequest hsrCopy = clone(hsr); + Class[] argsClasses = new Class[]{HttpServletRequest.class}; + Object[] args = new Object[]{hsrCopy}; + ClientMethod remoteMethod = new ClientMethod("getNodeToLabels", argsClasses, args); + Map nodeToLabelsInfoMap = + invokeConcurrent(subClustersActive.values(), remoteMethod, NodeToLabelsInfo.class); + return RouterWebServiceUtil.mergeNodeToLabels(nodeToLabelsInfoMap); + } catch (NotFoundException e) { + LOG.error("Get all active sub cluster(s) error.", e); + throw new IOException("Get all active sub cluster(s) error.", e); + } catch (YarnException e) { + LOG.error("getNodeToLabels error.", e); + throw new IOException("getNodeToLabels error.", e); + } } @Override public LabelsToNodesInfo getLabelsToNodes(Set labels) throws IOException { - throw new NotImplementedException("Code is not implemented"); + try { + Map subClustersActive = getActiveSubclusters(); + Class[] argsClasses = new Class[]{Set.class}; + Object[] args = new Object[]{labels}; + ClientMethod remoteMethod = new ClientMethod("getLabelsToNodes", argsClasses, args); + Map labelsToNodesInfoMap = + invokeConcurrent(subClustersActive.values(), remoteMethod, LabelsToNodesInfo.class); + + Map labelToNodesMap = new HashMap<>(); + labelsToNodesInfoMap.values().forEach(labelsToNode -> { + Map values = labelsToNode.getLabelsToNodes(); + for (Map.Entry item : values.entrySet()) { + NodeLabelInfo key = item.getKey(); + NodeIDsInfo leftValue = item.getValue(); + NodeIDsInfo rightValue = labelToNodesMap.getOrDefault(key, null); + NodeIDsInfo newValue = NodeIDsInfo.add(leftValue, rightValue); + labelToNodesMap.put(key, newValue); + } + }); + return new LabelsToNodesInfo(labelToNodesMap); + } catch (NotFoundException e) { + RouterServerUtil.logAndThrowIOException("Get all active sub cluster(s) error.", e); + } catch (YarnException e) { + RouterServerUtil.logAndThrowIOException("getLabelsToNodes error.", e); + } + return null; } @Override @@ -1195,7 +1277,23 @@ public Response replaceLabelsOnNode(Set newNodeLabelsName, @Override public NodeLabelsInfo getClusterNodeLabels(HttpServletRequest hsr) throws IOException { - throw new NotImplementedException("Code is not implemented"); + try { + Map subClustersActive = getActiveSubclusters(); + final HttpServletRequest hsrCopy = clone(hsr); + Class[] argsClasses = new Class[]{HttpServletRequest.class}; + Object[] args = new Object[]{hsrCopy}; + ClientMethod remoteMethod = new ClientMethod("getClusterNodeLabels", argsClasses, args); + Map nodeToLabelsInfoMap = + invokeConcurrent(subClustersActive.values(), remoteMethod, NodeLabelsInfo.class); + Set hashSets = Sets.newHashSet(); + nodeToLabelsInfoMap.values().forEach(item -> hashSets.addAll(item.getNodeLabels())); + return new NodeLabelsInfo(hashSets); + } catch (NotFoundException e) { + RouterServerUtil.logAndThrowIOException("Get all active sub cluster(s) error.", e); + } catch (YarnException e) { + RouterServerUtil.logAndThrowIOException("getClusterNodeLabels error.", e); + } + return null; } @Override @@ -1213,33 +1311,125 @@ public Response removeFromClusterNodeLabels(Set oldNodeLabels, @Override public NodeLabelsInfo getLabelsOnNode(HttpServletRequest hsr, String nodeId) throws IOException { - throw new NotImplementedException("Code is not implemented"); + try { + Map subClustersActive = getActiveSubclusters(); + final HttpServletRequest hsrCopy = clone(hsr); + Class[] argsClasses = new Class[]{HttpServletRequest.class, String.class}; + Object[] args = new Object[]{hsrCopy, nodeId}; + ClientMethod remoteMethod = new ClientMethod("getLabelsOnNode", argsClasses, args); + Map nodeToLabelsInfoMap = + invokeConcurrent(subClustersActive.values(), remoteMethod, NodeLabelsInfo.class); + Set hashSets = Sets.newHashSet(); + nodeToLabelsInfoMap.values().forEach(item -> hashSets.addAll(item.getNodeLabels())); + return new NodeLabelsInfo(hashSets); + } catch (NotFoundException e) { + RouterServerUtil.logAndThrowIOException("Get all active sub cluster(s) error.", e); + } catch (YarnException e) { + RouterServerUtil.logAndThrowIOException("getClusterNodeLabels error.", e); + } + return null; } @Override public AppPriority getAppPriority(HttpServletRequest hsr, String appId) throws AuthorizationException { - throw new NotImplementedException("Code is not implemented"); + + if (appId == null || appId.isEmpty()) { + throw new IllegalArgumentException("Parameter error, the appId is empty or null."); + } + + try { + SubClusterInfo subClusterInfo = getHomeSubClusterInfoByAppId(appId); + DefaultRequestInterceptorREST interceptor = getOrCreateInterceptorForSubCluster( + subClusterInfo.getSubClusterId(), subClusterInfo.getRMWebServiceAddress()); + return interceptor.getAppPriority(hsr, appId); + } catch (IllegalArgumentException e) { + RouterServerUtil.logAndThrowRunTimeException(e, + "Unable to get the getAppPriority appId: %s.", appId); + } catch (YarnException e) { + RouterServerUtil.logAndThrowRunTimeException("getAppPriority Failed.", e); + } + + return null; } @Override public Response updateApplicationPriority(AppPriority targetPriority, HttpServletRequest hsr, String appId) throws AuthorizationException, YarnException, InterruptedException, IOException { - throw new NotImplementedException("Code is not implemented"); + + if (appId == null || appId.isEmpty()) { + throw new IllegalArgumentException("Parameter error, the appId is empty or null."); + } + + if (targetPriority == null) { + throw new IllegalArgumentException("Parameter error, the targetPriority is empty or null."); + } + + try { + SubClusterInfo subClusterInfo = getHomeSubClusterInfoByAppId(appId); + DefaultRequestInterceptorREST interceptor = getOrCreateInterceptorForSubCluster( + subClusterInfo.getSubClusterId(), subClusterInfo.getRMWebServiceAddress()); + return interceptor.updateApplicationPriority(targetPriority, hsr, appId); + } catch (IllegalArgumentException e) { + RouterServerUtil.logAndThrowRunTimeException(e, + "Unable to get the updateApplicationPriority appId: %s.", appId); + } catch (YarnException e) { + RouterServerUtil.logAndThrowRunTimeException("updateApplicationPriority Failed.", e); + } + + return null; } @Override public AppQueue getAppQueue(HttpServletRequest hsr, String appId) throws AuthorizationException { - throw new NotImplementedException("Code is not implemented"); + + if (appId == null || appId.isEmpty()) { + throw new IllegalArgumentException("Parameter error, the appId is empty or null."); + } + + try { + SubClusterInfo subClusterInfo = getHomeSubClusterInfoByAppId(appId); + DefaultRequestInterceptorREST interceptor = getOrCreateInterceptorForSubCluster( + subClusterInfo.getSubClusterId(), subClusterInfo.getRMWebServiceAddress()); + return interceptor.getAppQueue(hsr, appId); + } catch (IllegalArgumentException e) { + RouterServerUtil.logAndThrowRunTimeException(e, + "Unable to get queue by appId: %s.", appId); + } catch (YarnException e) { + RouterServerUtil.logAndThrowRunTimeException("getAppQueue Failed.", e); + } + + return null; } @Override public Response updateAppQueue(AppQueue targetQueue, HttpServletRequest hsr, String appId) throws AuthorizationException, YarnException, InterruptedException, IOException { - throw new NotImplementedException("Code is not implemented"); + + if (appId == null || appId.isEmpty()) { + throw new IllegalArgumentException("Parameter error, the appId is empty or null."); + } + + if (targetQueue == null) { + throw new IllegalArgumentException("Parameter error, the targetQueue is null."); + } + + try { + SubClusterInfo subClusterInfo = getHomeSubClusterInfoByAppId(appId); + DefaultRequestInterceptorREST interceptor = getOrCreateInterceptorForSubCluster( + subClusterInfo.getSubClusterId(), subClusterInfo.getRMWebServiceAddress()); + return interceptor.updateAppQueue(targetQueue, hsr, appId); + } catch (IllegalArgumentException e) { + RouterServerUtil.logAndThrowRunTimeException(e, + "Unable to update app queue by appId: %s.", appId); + } catch (YarnException e) { + RouterServerUtil.logAndThrowRunTimeException("updateAppQueue Failed.", e); + } + + return null; } @Override @@ -1294,31 +1484,130 @@ public Response deleteReservation(ReservationDeleteRequestInfo resContext, public Response listReservation(String queue, String reservationId, long startTime, long endTime, boolean includeResourceAllocations, HttpServletRequest hsr) throws Exception { - throw new NotImplementedException("Code is not implemented"); + + if (queue == null || queue.isEmpty()) { + routerMetrics.incrListReservationFailedRetrieved(); + throw new IllegalArgumentException("Parameter error, the queue is empty or null."); + } + + if (reservationId == null || reservationId.isEmpty()) { + routerMetrics.incrListReservationFailedRetrieved(); + throw new IllegalArgumentException("Parameter error, the reservationId is empty or null."); + } + + try { + SubClusterInfo subClusterInfo = getHomeSubClusterInfoByReservationId(reservationId); + DefaultRequestInterceptorREST interceptor = getOrCreateInterceptorForSubCluster( + subClusterInfo.getSubClusterId(), subClusterInfo.getRMWebServiceAddress()); + HttpServletRequest hsrCopy = clone(hsr); + Response response = interceptor.listReservation(queue, reservationId, startTime, endTime, + includeResourceAllocations, hsrCopy); + if (response != null) { + return response; + } + } catch (YarnException e) { + routerMetrics.incrListReservationFailedRetrieved(); + RouterServerUtil.logAndThrowRunTimeException("listReservation Failed.", e); + } + + routerMetrics.incrListReservationFailedRetrieved(); + throw new YarnException("listReservation Failed."); } @Override public AppTimeoutInfo getAppTimeout(HttpServletRequest hsr, String appId, String type) throws AuthorizationException { - throw new NotImplementedException("Code is not implemented"); + + if (appId == null || appId.isEmpty()) { + throw new IllegalArgumentException("Parameter error, the appId is empty or null."); + } + + if (type == null || type.isEmpty()) { + throw new IllegalArgumentException("Parameter error, the type is empty or null."); + } + + try { + SubClusterInfo subClusterInfo = getHomeSubClusterInfoByAppId(appId); + DefaultRequestInterceptorREST interceptor = getOrCreateInterceptorForSubCluster( + subClusterInfo.getSubClusterId(), subClusterInfo.getRMWebServiceAddress()); + return interceptor.getAppTimeout(hsr, appId, type); + } catch (IllegalArgumentException e) { + RouterServerUtil.logAndThrowRunTimeException(e, + "Unable to get the getAppTimeout appId: %s.", appId); + } catch (YarnException e) { + RouterServerUtil.logAndThrowRunTimeException("getAppTimeout Failed.", e); + } + return null; } @Override public AppTimeoutsInfo getAppTimeouts(HttpServletRequest hsr, String appId) throws AuthorizationException { - throw new NotImplementedException("Code is not implemented"); + + if (appId == null || appId.isEmpty()) { + throw new IllegalArgumentException("Parameter error, the appId is empty or null."); + } + + try { + SubClusterInfo subClusterInfo = getHomeSubClusterInfoByAppId(appId); + DefaultRequestInterceptorREST interceptor = getOrCreateInterceptorForSubCluster( + subClusterInfo.getSubClusterId(), subClusterInfo.getRMWebServiceAddress()); + return interceptor.getAppTimeouts(hsr, appId); + } catch (IllegalArgumentException e) { + RouterServerUtil.logAndThrowRunTimeException(e, + "Unable to get the getAppTimeouts appId: %s.", appId); + } catch (YarnException e) { + RouterServerUtil.logAndThrowRunTimeException("getAppTimeouts Failed.", e); + } + return null; } @Override public Response updateApplicationTimeout(AppTimeoutInfo appTimeout, HttpServletRequest hsr, String appId) throws AuthorizationException, YarnException, InterruptedException, IOException { - throw new NotImplementedException("Code is not implemented"); + + if (appId == null || appId.isEmpty()) { + throw new IllegalArgumentException("Parameter error, the appId is empty or null."); + } + + if (appTimeout == null) { + throw new IllegalArgumentException("Parameter error, the appTimeout is null."); + } + + try { + SubClusterInfo subClusterInfo = getHomeSubClusterInfoByAppId(appId); + DefaultRequestInterceptorREST interceptor = getOrCreateInterceptorForSubCluster( + subClusterInfo.getSubClusterId(), subClusterInfo.getRMWebServiceAddress()); + return interceptor.updateApplicationTimeout(appTimeout, hsr, appId); + } catch (IllegalArgumentException e) { + RouterServerUtil.logAndThrowRunTimeException(e, + "Unable to get the updateApplicationTimeout appId: %s.", appId); + } catch (YarnException e) { + RouterServerUtil.logAndThrowRunTimeException("updateApplicationTimeout Failed.", e); + } + return null; } @Override public AppAttemptsInfo getAppAttempts(HttpServletRequest hsr, String appId) { - throw new NotImplementedException("Code is not implemented"); + + if (appId == null || appId.isEmpty()) { + throw new IllegalArgumentException("Parameter error, the appId is empty or null."); + } + + try { + SubClusterInfo subClusterInfo = getHomeSubClusterInfoByAppId(appId); + DefaultRequestInterceptorREST interceptor = getOrCreateInterceptorForSubCluster( + subClusterInfo.getSubClusterId(), subClusterInfo.getRMWebServiceAddress()); + return interceptor.getAppAttempts(hsr, appId); + } catch (IllegalArgumentException e) { + RouterServerUtil.logAndThrowRunTimeException(e, + "Unable to get the AppAttempt appId: %s.", appId); + } catch (YarnException e) { + RouterServerUtil.logAndThrowRunTimeException("getAppAttempts Failed.", e); + } + return null; } @Override @@ -1330,19 +1619,105 @@ public RMQueueAclInfo checkUserAccessToQueue(String queue, String username, @Override public AppAttemptInfo getAppAttempt(HttpServletRequest req, HttpServletResponse res, String appId, String appAttemptId) { - throw new NotImplementedException("Code is not implemented"); + + if (appId == null || appId.isEmpty()) { + throw new IllegalArgumentException("Parameter error, the appId is empty or null."); + } + if (appAttemptId == null || appAttemptId.isEmpty()) { + throw new IllegalArgumentException("Parameter error, the appAttemptId is empty or null."); + } + + try { + SubClusterInfo subClusterInfo = getHomeSubClusterInfoByAppId(appId); + + DefaultRequestInterceptorREST interceptor = getOrCreateInterceptorForSubCluster( + subClusterInfo.getSubClusterId(), subClusterInfo.getRMWebServiceAddress()); + return interceptor.getAppAttempt(req, res, appId, appAttemptId); + } catch (IllegalArgumentException e) { + RouterServerUtil.logAndThrowRunTimeException(e, + "Unable to get the AppAttempt appId: %s, appAttemptId: %s.", appId, appAttemptId); + } catch (YarnException e) { + RouterServerUtil.logAndThrowRunTimeException("getContainer Failed.", e); + } + + return null; } @Override public ContainersInfo getContainers(HttpServletRequest req, HttpServletResponse res, String appId, String appAttemptId) { - throw new NotImplementedException("Code is not implemented"); + + ContainersInfo containersInfo = new ContainersInfo(); + + Map subClustersActive; + try { + subClustersActive = getActiveSubclusters(); + } catch (NotFoundException e) { + LOG.error("Get all active sub cluster(s) error.", e); + return containersInfo; + } + + try { + Class[] argsClasses = new Class[]{ + HttpServletRequest.class, HttpServletResponse.class, String.class, String.class}; + Object[] args = new Object[]{req, res, appId, appAttemptId}; + ClientMethod remoteMethod = new ClientMethod("getContainers", argsClasses, args); + Map containersInfoMap = + invokeConcurrent(subClustersActive.values(), remoteMethod, ContainersInfo.class); + if (containersInfoMap != null) { + containersInfoMap.values().forEach(containers -> + containersInfo.addAll(containers.getContainers())); + } + } catch (Exception ex) { + LOG.error("Failed to return GetContainers.", ex); + } + + return containersInfo; } @Override public ContainerInfo getContainer(HttpServletRequest req, HttpServletResponse res, String appId, String appAttemptId, String containerId) { + + if (appId == null || appId.isEmpty()) { + throw new IllegalArgumentException("Parameter error, the appId is empty or null."); + } + if (appAttemptId == null || appAttemptId.isEmpty()) { + throw new IllegalArgumentException("Parameter error, the appAttemptId is empty or null."); + } + if (containerId == null || containerId.isEmpty()) { + throw new IllegalArgumentException("Parameter error, the containerId is empty or null."); + } + + try { + SubClusterInfo subClusterInfo = getHomeSubClusterInfoByAppId(appId); + + DefaultRequestInterceptorREST interceptor = getOrCreateInterceptorForSubCluster( + subClusterInfo.getSubClusterId(), subClusterInfo.getRMWebServiceAddress()); + return interceptor.getContainer(req, res, appId, appAttemptId, containerId); + } catch (IllegalArgumentException e) { + String msg = String.format( + "Unable to get the AppAttempt appId: %s, appAttemptId: %s, containerId: %s.", appId, + appAttemptId, containerId); + RouterServerUtil.logAndThrowRunTimeException(msg, e); + } catch (YarnException e) { + RouterServerUtil.logAndThrowRunTimeException("getContainer Failed.", e); + } + + return null; + } + + @Override + public Response updateSchedulerConfiguration(SchedConfUpdateInfo mutationInfo, + HttpServletRequest hsr) + throws AuthorizationException, InterruptedException { + throw new NotImplementedException("Code is not implemented"); + } + + @Override + public Response getSchedulerConfiguration(HttpServletRequest hsr) + throws AuthorizationException { throw new NotImplementedException("Code is not implemented"); } @@ -1357,7 +1732,32 @@ public void setNextInterceptor(RESTRequestInterceptor next) { @Override public Response signalToContainer(String containerId, String command, HttpServletRequest req) { - throw new NotImplementedException("Code is not implemented"); + + if (containerId == null || containerId.isEmpty()) { + throw new IllegalArgumentException("Parameter error, the containerId is empty or null."); + } + + if (command == null || command.isEmpty()) { + throw new IllegalArgumentException("Parameter error, the command is empty or null."); + } + + try { + ContainerId containerIdObj = ContainerId.fromString(containerId); + ApplicationId applicationId = containerIdObj.getApplicationAttemptId().getApplicationId(); + + SubClusterInfo subClusterInfo = getHomeSubClusterInfoByAppId(applicationId.toString()); + + DefaultRequestInterceptorREST interceptor = getOrCreateInterceptorForSubCluster( + subClusterInfo.getSubClusterId(), subClusterInfo.getRMWebServiceAddress()); + return interceptor.signalToContainer(containerId, command, req); + + } catch (YarnException e) { + RouterServerUtil.logAndThrowRunTimeException("signalToContainer Failed.", e); + } catch (AuthorizationException e) { + RouterServerUtil.logAndThrowRunTimeException("signalToContainer Author Failed.", e); + } + + return null; } @Override @@ -1366,4 +1766,103 @@ public void shutdown() { threadpool.shutdown(); } } + + private Map invokeConcurrent(Collection clusterIds, + ClientMethod request, Class clazz) throws YarnException { + + Map results = new HashMap<>(); + + // Send the requests in parallel + CompletionService compSvc = new ExecutorCompletionService<>(this.threadpool); + + for (final SubClusterInfo info : clusterIds) { + compSvc.submit(() -> { + DefaultRequestInterceptorREST interceptor = getOrCreateInterceptorForSubCluster( + info.getSubClusterId(), info.getRMWebServiceAddress()); + try { + Method method = DefaultRequestInterceptorREST.class. + getMethod(request.getMethodName(), request.getTypes()); + Object retObj = method.invoke(interceptor, request.getParams()); + R ret = clazz.cast(retObj); + return ret; + } catch (Exception e) { + LOG.error("SubCluster {} failed to call {} method.", + info.getSubClusterId(), request.getMethodName(), e); + return null; + } + }); + } + + clusterIds.stream().forEach(clusterId -> { + try { + Future future = compSvc.take(); + R response = future.get(); + if (response != null) { + results.put(clusterId, response); + } + } catch (Throwable e) { + String msg = String.format("SubCluster %s failed to %s report.", + clusterId, request.getMethodName()); + LOG.warn(msg, e); + throw new YarnRuntimeException(msg, e); + } + }); + return results; + } + + /** + * get the HomeSubCluster according to ApplicationId. + * + * @param appId applicationId + * @return HomeSubCluster + * @throws YarnException on failure + */ + private SubClusterInfo getHomeSubClusterInfoByAppId(String appId) + throws YarnException { + SubClusterInfo subClusterInfo = null; + try { + ApplicationId applicationId = ApplicationId.fromString(appId); + SubClusterId subClusterId = federationFacade.getApplicationHomeSubCluster(applicationId); + if (subClusterId == null) { + RouterServerUtil.logAndThrowException(null, + "Can't get HomeSubCluster by applicationId %s", applicationId); + } + subClusterInfo = federationFacade.getSubCluster(subClusterId); + return subClusterInfo; + } catch (YarnException e) { + RouterServerUtil.logAndThrowException(e, + "Get HomeSubClusterInfo by applicationId %s failed.", appId); + } + throw new YarnException("Unable to get subCluster by applicationId = " + appId); + } + + /** + * get the HomeSubCluster according to ReservationId. + * + * @param resId reservationId + * @return HomeSubCluster + * @throws YarnException on failure + */ + private SubClusterInfo getHomeSubClusterInfoByReservationId(String resId) + throws YarnException { + try { + ReservationId reservationId = ReservationId.parseReservationId(resId); + SubClusterId subClusterId = federationFacade.getReservationHomeSubCluster(reservationId); + if (subClusterId == null) { + RouterServerUtil.logAndThrowException(null, + "Can't get HomeSubCluster by reservationId %s", resId); + } + SubClusterInfo subClusterInfo = federationFacade.getSubCluster(subClusterId); + return subClusterInfo; + } catch (YarnException | IOException e) { + RouterServerUtil.logAndThrowException(e, + "Get HomeSubClusterInfo by reservationId %s failed.", resId); + } + throw new YarnException("Unable to get subCluster by reservationId = " + resId); + } + + @VisibleForTesting + public LRUCacheHashMap getAppInfosCaches() { + return appInfosCaches; + } } \ No newline at end of file diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/RESTRequestInterceptor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/RESTRequestInterceptor.java index 06f39b5e393aa..917809ad6cd60 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/RESTRequestInterceptor.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/RESTRequestInterceptor.java @@ -29,7 +29,7 @@ import org.apache.hadoop.yarn.server.webapp.dao.ContainersInfo; /** - * Defines the contract to be implemented by the request intercepter classes, + * Defines the contract to be implemented by the request interceptor classes, * that can be used to intercept and inspect messages sent from the client to * the resource manager server. * @@ -42,7 +42,7 @@ public interface RESTRequestInterceptor extends RMWebServiceProtocol, Configurable { /** - * This method is called for initializing the intercepter. This is guaranteed + * This method is called for initializing the interceptor. This is guaranteed * to be called only once in the lifetime of this instance. * * @param user the name of the client @@ -50,28 +50,28 @@ public interface RESTRequestInterceptor void init(String user); /** - * This method is called to release the resources held by the intercepter. + * This method is called to release the resources held by the interceptor. * This will be called when the application pipeline is being destroyed. The * concrete implementations should dispose the resources and forward the - * request to the next intercepter, if any. + * request to the next interceptor, if any. */ void shutdown(); /** - * Sets the next intercepter in the pipeline. The concrete implementation of + * Sets the next interceptor in the pipeline. The concrete implementation of * this interface should always pass the request to the nextInterceptor after - * inspecting the message. The last intercepter in the chain is responsible to + * inspecting the message. The last interceptor in the chain is responsible to * send the messages to the resource manager service and so the last - * intercepter will not receive this method call. + * interceptor will not receive this method call. * * @param nextInterceptor the RESTRequestInterceptor to set in the pipeline */ void setNextInterceptor(RESTRequestInterceptor nextInterceptor); /** - * Returns the next intercepter in the chain. + * Returns the next interceptor in the chain. * - * @return the next intercepter in the chain + * @return the next interceptor in the chain */ RESTRequestInterceptor getNextInterceptor(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/RouterWebServiceUtil.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/RouterWebServiceUtil.java index 6d7dc5f5c19b3..7423c8c907bb7 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/RouterWebServiceUtil.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/RouterWebServiceUtil.java @@ -30,6 +30,9 @@ import java.util.List; import java.util.Map; import java.util.Map.Entry; +import java.util.Collection; +import java.util.Set; +import java.util.HashSet; import java.util.concurrent.TimeUnit; import javax.servlet.http.HttpServletRequest; @@ -43,13 +46,19 @@ import org.apache.hadoop.net.NetUtils; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.yarn.api.records.YarnApplicationState; +import org.apache.hadoop.yarn.api.records.NodeLabel; import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.server.federation.store.records.SubClusterInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.RMWebAppUtil; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppsInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ClusterMetricsInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.NodeInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.NodesInfo; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.NodeLabelsInfo; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.NodeToLabelsInfo; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ApplicationStatisticsInfo; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.StatisticsItemInfo; import org.apache.hadoop.yarn.server.uam.UnmanagedApplicationManager; import org.apache.hadoop.yarn.webapp.BadRequestException; import org.apache.hadoop.yarn.webapp.ForbiddenException; @@ -83,7 +92,7 @@ private RouterWebServiceUtil() { /** * Creates and performs a REST call to a specific WebService. * - * @param webApp the address of the remote webap + * @param webApp the address of the remote webapp * @param hsr the servlet request * @param returnType the return type of the REST call * @param Type of return object. @@ -170,7 +179,7 @@ public T run() { /** * Performs an invocation of a REST call on a remote RMWebService. - * @param webApp the address of the remote webap + * @param webApp the address of the remote webapp * @param path to add to the webapp address * @param method the HTTP method of the REST call * @param additionalPath the servlet request path @@ -280,7 +289,7 @@ public static void retrieveException(ClientResponse response) { /** * Merges a list of AppInfo grouping by ApplicationId. Our current policy is - * to merge the application reports from the reacheable SubClusters. Via + * to merge the application reports from the reachable SubClusters. Via * configuration parameter, we decide whether to return applications for which * the primary AM is missing or to omit them. * @@ -293,8 +302,8 @@ public static AppsInfo mergeAppsInfo(ArrayList appsInfo, boolean returnPartialResult) { AppsInfo allApps = new AppsInfo(); - Map federationAM = new HashMap(); - Map federationUAMSum = new HashMap(); + Map federationAM = new HashMap<>(); + Map federationUAMSum = new HashMap<>(); for (AppInfo a : appsInfo) { // Check if this AppInfo is an AM if (a.getAMHostHttpAddress() != null) { @@ -332,7 +341,7 @@ public static AppsInfo mergeAppsInfo(ArrayList appsInfo, } } - allApps.addAll(new ArrayList(federationAM.values())); + allApps.addAll(new ArrayList<>(federationAM.values())); return allApps; } @@ -419,7 +428,7 @@ public static NodesInfo deleteDuplicateNodesInfo(ArrayList nodes) { nodesMap.put(node.getNodeId(), node); } } - nodesInfo.addAll(new ArrayList(nodesMap.values())); + nodesInfo.addAll(new ArrayList<>(nodesMap.values())); return nodesInfo; } @@ -494,7 +503,7 @@ public static void mergeMetrics(ClusterMetricsInfo metrics, protected static String getMediaTypeFromHttpServletRequest( HttpServletRequest request, final Class returnType) { if (request == null) { - // By default we return XML for REST call without HttpServletRequest + // By default, we return XML for REST call without HttpServletRequest return MediaType.APPLICATION_XML; } // TODO @@ -503,10 +512,68 @@ protected static String getMediaTypeFromHttpServletRequest( } String header = request.getHeader(HttpHeaders.ACCEPT); if (header == null || header.equals("*")) { - // By default we return JSON + // By default, we return JSON return MediaType.APPLICATION_JSON; } return header; } + public static NodeToLabelsInfo mergeNodeToLabels( + Map nodeToLabelsInfoMap) { + + HashMap nodeToLabels = new HashMap<>(); + Collection nodeToLabelsInfos = nodeToLabelsInfoMap.values(); + + nodeToLabelsInfos.stream().forEach(nodeToLabelsInfo -> { + for (Map.Entry item : nodeToLabelsInfo.getNodeToLabels().entrySet()) { + String key = item.getKey(); + NodeLabelsInfo itemValue = item.getValue(); + NodeLabelsInfo nodeToLabelsValue = nodeToLabels.getOrDefault(item.getKey(), null); + Set hashSet = new HashSet<>(); + if (itemValue != null) { + hashSet.addAll(itemValue.getNodeLabels()); + } + if (nodeToLabelsValue != null) { + hashSet.addAll(nodeToLabelsValue.getNodeLabels()); + } + nodeToLabels.put(key, new NodeLabelsInfo(hashSet)); + } + }); + + return new NodeToLabelsInfo(nodeToLabels); + } + + public static ApplicationStatisticsInfo mergeApplicationStatisticsInfo( + Collection appStatistics) { + ApplicationStatisticsInfo result = new ApplicationStatisticsInfo(); + Map statisticsItemMap = new HashMap<>(); + + appStatistics.stream().forEach(appStatistic -> { + List statisticsItemInfos = appStatistic.getStatItems(); + for (StatisticsItemInfo statisticsItemInfo : statisticsItemInfos) { + + String statisticsItemKey = + statisticsItemInfo.getType() + "_" + statisticsItemInfo.getState().toString(); + + StatisticsItemInfo statisticsItemValue; + if (statisticsItemMap.containsKey(statisticsItemKey)) { + statisticsItemValue = statisticsItemMap.get(statisticsItemKey); + long statisticsItemValueCount = statisticsItemValue.getCount(); + long statisticsItemInfoCount = statisticsItemInfo.getCount(); + long newCount = statisticsItemValueCount + statisticsItemInfoCount; + statisticsItemValue.setCount(newCount); + } else { + statisticsItemValue = new StatisticsItemInfo(statisticsItemInfo); + } + + statisticsItemMap.put(statisticsItemKey, statisticsItemValue); + } + }); + + if (!statisticsItemMap.isEmpty()) { + result.getStatItems().addAll(statisticsItemMap.values()); + } + + return result; + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/RouterWebServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/RouterWebServices.java index c221b86e98433..b1dc8635b3fd1 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/RouterWebServices.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/RouterWebServices.java @@ -19,10 +19,7 @@ package org.apache.hadoop.yarn.server.router.webapp; import java.io.IOException; -import java.util.ArrayList; -import java.util.Collection; import java.util.Collections; -import java.util.List; import java.util.Map; import java.util.Set; @@ -48,11 +45,8 @@ import org.apache.hadoop.http.JettyUtils; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.security.authorize.AuthorizationException; -import org.apache.hadoop.util.ReflectionUtils; -import org.apache.hadoop.util.StringUtils; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.exceptions.YarnException; -import org.apache.hadoop.yarn.exceptions.YarnRuntimeException; import org.apache.hadoop.yarn.server.resourcemanager.webapp.RMWSConsts; import org.apache.hadoop.yarn.server.resourcemanager.webapp.RMWebServiceProtocol; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ActivitiesInfo; @@ -86,9 +80,11 @@ import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.BulkActivitiesInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.SchedulerTypeInfo; import org.apache.hadoop.yarn.server.router.Router; +import org.apache.hadoop.yarn.server.router.RouterServerUtil; import org.apache.hadoop.yarn.server.webapp.dao.ContainerInfo; import org.apache.hadoop.yarn.server.webapp.dao.ContainersInfo; import org.apache.hadoop.yarn.util.LRUCacheHashMap; +import org.apache.hadoop.yarn.webapp.dao.SchedConfUpdateInfo; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -135,30 +131,7 @@ public RouterWebServices(final Router router, Configuration conf) { int maxCacheSize = conf.getInt(YarnConfiguration.ROUTER_PIPELINE_CACHE_MAX_SIZE, YarnConfiguration.DEFAULT_ROUTER_PIPELINE_CACHE_MAX_SIZE); - this.userPipelineMap = Collections.synchronizedMap( - new LRUCacheHashMap( - maxCacheSize, true)); - } - - /** - * Returns the comma separated intercepter class names from the configuration. - * - * @param conf - * @return the intercepter class names as an instance of ArrayList - */ - private List getInterceptorClassNames(Configuration config) { - String configuredInterceptorClassNames = - config.get(YarnConfiguration.ROUTER_WEBAPP_INTERCEPTOR_CLASS_PIPELINE, - YarnConfiguration.DEFAULT_ROUTER_WEBAPP_INTERCEPTOR_CLASS); - - List interceptorClassNames = new ArrayList(); - Collection tempList = - StringUtils.getStringCollection(configuredInterceptorClassNames); - for (String item : tempList) { - interceptorClassNames.add(item.trim()); - } - - return interceptorClassNames; + this.userPipelineMap = Collections.synchronizedMap(new LRUCacheHashMap<>(maxCacheSize, true)); } private void init() { @@ -189,9 +162,9 @@ protected RequestInterceptorChainWrapper getInterceptorChain( } /** - * Gets the Request intercepter chains for all the users. + * Gets the Request interceptor chains for all the users. * - * @return the request intercepter chains. + * @return the request interceptor chains. */ @VisibleForTesting protected Map getPipelines() { @@ -199,57 +172,23 @@ protected Map getPipelines() { } /** - * This method creates and returns reference of the first intercepter in the - * chain of request intercepter instances. + * This method creates and returns reference of the first interceptor in the + * chain of request interceptor instances. * - * @return the reference of the first intercepter in the chain + * @return the reference of the first interceptor in the chain */ @VisibleForTesting protected RESTRequestInterceptor createRequestInterceptorChain() { - - List interceptorClassNames = getInterceptorClassNames(conf); - - RESTRequestInterceptor pipeline = null; - RESTRequestInterceptor current = null; - for (String interceptorClassName : interceptorClassNames) { - try { - Class interceptorClass = conf.getClassByName(interceptorClassName); - if (RESTRequestInterceptor.class.isAssignableFrom(interceptorClass)) { - RESTRequestInterceptor interceptorInstance = - (RESTRequestInterceptor) ReflectionUtils - .newInstance(interceptorClass, conf); - if (pipeline == null) { - pipeline = interceptorInstance; - current = interceptorInstance; - continue; - } else { - current.setNextInterceptor(interceptorInstance); - current = interceptorInstance; - } - } else { - throw new YarnRuntimeException( - "Class: " + interceptorClassName + " not instance of " - + RESTRequestInterceptor.class.getCanonicalName()); - } - } catch (ClassNotFoundException e) { - throw new YarnRuntimeException( - "Could not instantiate RESTRequestInterceptor: " - + interceptorClassName, - e); - } - } - - if (pipeline == null) { - throw new YarnRuntimeException( - "RequestInterceptor pipeline is not configured in the system"); - } - return pipeline; + return RouterServerUtil.createRequestInterceptorChain(conf, + YarnConfiguration.ROUTER_WEBAPP_INTERCEPTOR_CLASS_PIPELINE, + YarnConfiguration.DEFAULT_ROUTER_WEBAPP_INTERCEPTOR_CLASS, + RESTRequestInterceptor.class); } /** - * Initializes the request intercepter pipeline for the specified user. + * Initializes the request interceptor pipeline for the specified user. * - * @param user + * @param user specified user. */ private RequestInterceptorChainWrapper initializePipeline(String user) { synchronized (this.userPipelineMap) { @@ -264,14 +203,14 @@ private RequestInterceptorChainWrapper initializePipeline(String user) { try { // We should init the pipeline instance after it is created and then // add to the map, to ensure thread safe. - LOG.info("Initializing request processing pipeline for user: {}", user); + LOG.info("Initializing request processing pipeline for user: {}.", user); RESTRequestInterceptor interceptorChain = this.createRequestInterceptorChain(); interceptorChain.init(user); chainWrapper.init(interceptorChain); } catch (Exception e) { - LOG.error("Init RESTRequestInterceptor error for user: " + user, e); + LOG.error("Init RESTRequestInterceptor error for user: {}", user, e); throw e; } @@ -298,9 +237,9 @@ public synchronized void init(RESTRequestInterceptor interceptor) { } /** - * Gets the root request intercepter. + * Gets the root request interceptor. * - * @return the root request intercepter + * @return the root request interceptor */ public synchronized RESTRequestInterceptor getRootInterceptor() { return rootInterceptor; @@ -337,7 +276,7 @@ public ClusterInfo getClusterInfo() { @GET @Path(RMWSConsts.CLUSTER_USER_INFO) @Produces({ MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8, - MediaType.APPLICATION_XML + "; " + JettyUtils.UTF_8 }) + MediaType.APPLICATION_XML + "; " + JettyUtils.UTF_8 }) @Override public ClusterUserInfo getClusterUserInfo(@Context HttpServletRequest hsr) { init(); @@ -833,10 +772,12 @@ public Response deleteReservation(ReservationDeleteRequestInfo resContext, @Override public Response listReservation( @QueryParam(RMWSConsts.QUEUE) @DefaultValue(DEFAULT_QUEUE) String queue, - @QueryParam(RMWSConsts.RESERVATION_ID) @DefaultValue(DEFAULT_RESERVATION_ID) String reservationId, + @QueryParam(RMWSConsts.RESERVATION_ID) + @DefaultValue(DEFAULT_RESERVATION_ID) String reservationId, @QueryParam(RMWSConsts.START_TIME) @DefaultValue(DEFAULT_START_TIME) long startTime, @QueryParam(RMWSConsts.END_TIME) @DefaultValue(DEFAULT_END_TIME) long endTime, - @QueryParam(RMWSConsts.INCLUDE_RESOURCE) @DefaultValue(DEFAULT_INCLUDE_RESOURCE) boolean includeResourceAllocations, + @QueryParam(RMWSConsts.INCLUDE_RESOURCE) + @DefaultValue(DEFAULT_INCLUDE_RESOURCE) boolean includeResourceAllocations, @Context HttpServletRequest hsr) throws Exception { init(); RequestInterceptorChainWrapper pipeline = getInterceptorChain(hsr); @@ -956,6 +897,33 @@ public ContainerInfo getContainer(@Context HttpServletRequest req, appAttemptId, containerId); } + @PUT + @Path(RMWSConsts.SCHEDULER_CONF) + @Produces({ MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8, + MediaType.APPLICATION_XML + "; " + JettyUtils.UTF_8 }) + @Consumes({ MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML }) + @Override + public Response updateSchedulerConfiguration(SchedConfUpdateInfo mutationInfo, + HttpServletRequest hsr) + throws AuthorizationException, InterruptedException { + init(); + RequestInterceptorChainWrapper pipeline = getInterceptorChain(hsr); + return pipeline.getRootInterceptor() + .updateSchedulerConfiguration(mutationInfo, hsr); + } + + @GET + @Path(RMWSConsts.SCHEDULER_CONF) + @Produces({ MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8, + MediaType.APPLICATION_XML + "; " + JettyUtils.UTF_8 }) + @Override + public Response getSchedulerConfiguration(HttpServletRequest hsr) + throws AuthorizationException { + init(); + RequestInterceptorChainWrapper pipeline = getInterceptorChain(hsr); + return pipeline.getRootInterceptor().getSchedulerConfiguration(hsr); + } + @VisibleForTesting protected void setResponse(HttpServletResponse response) { this.response = response; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/cache/RouterAppInfoCacheKey.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/cache/RouterAppInfoCacheKey.java new file mode 100644 index 0000000000000..27164f00415fb --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/cache/RouterAppInfoCacheKey.java @@ -0,0 +1,156 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.yarn.server.router.webapp.cache; + +import org.apache.commons.lang3.builder.EqualsBuilder; +import org.apache.commons.lang3.builder.HashCodeBuilder; +import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.RMWebAppUtil; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import javax.servlet.http.HttpServletRequest; +import java.util.Set; + +public class RouterAppInfoCacheKey { + + private static String user = "YarnRouter"; + + private static final Logger LOG = + LoggerFactory.getLogger(RouterAppInfoCacheKey.class.getName()); + + private UserGroupInformation ugi; + private String stateQuery; + private Set statesQuery; + private String finalStatusQuery; + private String userQuery; + private String queueQuery; + private String count; + private String startedBegin; + private String startedEnd; + private String finishBegin; + private String finishEnd; + private Set applicationTypes; + private Set applicationTags; + private String name; + private Set unselectedFields; + + public RouterAppInfoCacheKey() { + + } + + @SuppressWarnings("checkstyle:ParameterNumber") + public RouterAppInfoCacheKey(UserGroupInformation ugi, String stateQuery, + Set statesQuery, String finalStatusQuery, String userQuery, + String queueQuery, String count, String startedBegin, String startedEnd, + String finishBegin, String finishEnd, Set applicationTypes, + Set applicationTags, String name, Set unselectedFields) { + this.ugi = ugi; + this.stateQuery = stateQuery; + this.statesQuery = statesQuery; + this.finalStatusQuery = finalStatusQuery; + this.userQuery = userQuery; + this.queueQuery = queueQuery; + this.count = count; + this.startedBegin = startedBegin; + this.startedEnd = startedEnd; + this.finishBegin = finishBegin; + this.finishEnd = finishEnd; + this.applicationTypes = applicationTypes; + this.applicationTags = applicationTags; + this.name = name; + this.unselectedFields = unselectedFields; + } + + + @SuppressWarnings("checkstyle:ParameterNumber") + public static RouterAppInfoCacheKey newInstance(HttpServletRequest hsr, String stateQuery, + Set statesQuery, String finalStatusQuery, String userQuery, + String queueQuery, String count, String startedBegin, String startedEnd, + String finishBegin, String finishEnd, Set applicationTypes, + Set applicationTags, String name, Set unselectedFields) { + + UserGroupInformation callerUGI = null; + if (hsr != null) { + callerUGI = RMWebAppUtil.getCallerUserGroupInformation(hsr, true); + } else { + // user not required + callerUGI = UserGroupInformation.createRemoteUser("YarnRouter"); + } + + if (callerUGI == null) { + LOG.error("Unable to obtain user name, user not authenticated."); + return null; + } + + return new RouterAppInfoCacheKey( + callerUGI, stateQuery, statesQuery, finalStatusQuery, userQuery, + queueQuery, count, startedBegin, startedEnd, finishBegin, finishEnd, + applicationTypes, applicationTags, name, unselectedFields); + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + RouterAppInfoCacheKey that = (RouterAppInfoCacheKey) o; + + return new EqualsBuilder() + .append(this.ugi.getUserName(), that.ugi.getUserName()) + .append(this.stateQuery, that.stateQuery) + .append(this.statesQuery, that.statesQuery) + .append(this.finalStatusQuery, that.finalStatusQuery) + .append(this.userQuery, that.userQuery) + .append(this.queueQuery, that.queueQuery) + .append(this.count, that.count) + .append(this.startedBegin, that.startedBegin) + .append(this.startedEnd, that.startedEnd) + .append(this.finishBegin, that.finishBegin) + .append(this.finishEnd, that.finishEnd) + .append(this.applicationTypes, that.applicationTypes) + .append(this.applicationTags, that.applicationTags) + .append(this.name, that.name) + .append(this.unselectedFields, that.unselectedFields) + .isEquals(); + } + + @Override + public int hashCode() { + return new HashCodeBuilder() + .append(this.ugi.getUserName()) + .append(this.stateQuery) + .append(this.statesQuery) + .append(this.finalStatusQuery) + .append(this.userQuery) + .append(this.queueQuery) + .append(this.count) + .append(this.startedBegin) + .append(this.startedEnd) + .append(this.finishBegin) + .append(this.finishEnd) + .append(this.applicationTypes) + .append(this.applicationTags) + .append(this.name) + .append(this.unselectedFields) + .toHashCode(); + } +} diff --git a/hadoop-maven-plugins/src/main/java/org/apache/hadoop/maven/plugin/shade/resource/package-info.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/cache/package-info.java similarity index 79% rename from hadoop-maven-plugins/src/main/java/org/apache/hadoop/maven/plugin/shade/resource/package-info.java rename to hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/cache/package-info.java index 571491c03540f..187cd72fe257f 100644 --- a/hadoop-maven-plugins/src/main/java/org/apache/hadoop/maven/plugin/shade/resource/package-info.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/cache/package-info.java @@ -1,4 +1,4 @@ -/* +/** * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -15,9 +15,4 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - -/** - * Resource handling plugins used internal to the Hadoop build. - * IA.Private (build structure encourages not using the actual annotations) - */ -package org.apache.hadoop.maven.plugin.shade.resource; +package org.apache.hadoop.yarn.server.router.webapp.cache; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/TestRouterAuditLogger.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/TestRouterAuditLogger.java index 40e22964f8094..48d3ef6c0fea4 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/TestRouterAuditLogger.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/TestRouterAuditLogger.java @@ -22,11 +22,28 @@ import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.ipc.ClientId; +import org.apache.hadoop.ipc.ProtobufRpcEngine2; +import org.apache.hadoop.ipc.RPC; +import org.apache.hadoop.ipc.Server; +import org.apache.hadoop.ipc.TestRpcBase; +import org.apache.hadoop.ipc.protobuf.TestProtos; +import org.apache.hadoop.ipc.protobuf.TestRpcServiceProtos; +import org.apache.hadoop.ipc.TestRPC; +import org.apache.hadoop.net.NetUtils; +import org.apache.hadoop.thirdparty.protobuf.BlockingService; +import org.apache.hadoop.thirdparty.protobuf.RpcController; +import org.apache.hadoop.thirdparty.protobuf.ServiceException; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.server.federation.store.records.SubClusterId; +import org.junit.Assert; import org.junit.Before; import org.junit.Test; +import java.net.InetAddress; +import java.net.InetSocketAddress; + /** * Tests {@link RouterAuditLogger}. */ @@ -76,13 +93,17 @@ public void testKeyValLogFormat() throws Exception { /** * Test the AuditLog format for successful events. */ - private void testSuccessLogFormatHelper(ApplicationId appId, + private void testSuccessLogFormatHelper(boolean checkIP, ApplicationId appId, SubClusterId subClusterId) { // check without the IP String sLog = RouterAuditLogger .createSuccessLog(USER, OPERATION, TARGET, appId, subClusterId); StringBuilder expLog = new StringBuilder(); expLog.append("USER=test\t"); + if (checkIP) { + InetAddress ip = Server.getRemoteIp(); + expLog.append(RouterAuditLogger.Keys.IP.name() + "=" + ip.getHostAddress() + "\t"); + } expLog.append("OPERATION=oper\tTARGET=tgt\tRESULT=SUCCESS"); if (appId != null) { expLog.append("\tAPPID=app_1"); @@ -109,23 +130,27 @@ private void testSuccessLogNulls() { * Test the AuditLog format for successful events with the various * parameters. */ - private void testSuccessLogFormat() { - testSuccessLogFormatHelper(null, null); - testSuccessLogFormatHelper(APPID, null); - testSuccessLogFormatHelper(null, SUBCLUSTERID); - testSuccessLogFormatHelper(APPID, SUBCLUSTERID); + private void testSuccessLogFormat(boolean checkIP) { + testSuccessLogFormatHelper(checkIP, null, null); + testSuccessLogFormatHelper(checkIP, APPID, null); + testSuccessLogFormatHelper(checkIP, null, SUBCLUSTERID); + testSuccessLogFormatHelper(checkIP, APPID, SUBCLUSTERID); } /** * Test the AuditLog format for failure events. */ - private void testFailureLogFormatHelper(ApplicationId appId, + private void testFailureLogFormatHelper(boolean checkIP, ApplicationId appId, SubClusterId subClusterId) { String fLog = RouterAuditLogger .createFailureLog(USER, OPERATION, "UNKNOWN", TARGET, DESC, appId, subClusterId); StringBuilder expLog = new StringBuilder(); expLog.append("USER=test\t"); + if (checkIP) { + InetAddress ip = Server.getRemoteIp(); + expLog.append(RouterAuditLogger.Keys.IP.name() + "=" + ip.getHostAddress() + "\t"); + } expLog.append("OPERATION=oper\tTARGET=tgt\tRESULT=FAILURE\t"); expLog.append("DESCRIPTION=description of an audit log"); expLog.append("\tPERMISSIONS=UNKNOWN"); @@ -143,18 +168,79 @@ private void testFailureLogFormatHelper(ApplicationId appId, * Test the AuditLog format for failure events with the various * parameters. */ - private void testFailureLogFormat() { - testFailureLogFormatHelper(null, null); - testFailureLogFormatHelper(APPID, null); - testFailureLogFormatHelper(null, SUBCLUSTERID); - testFailureLogFormatHelper(APPID, SUBCLUSTERID); + private void testFailureLogFormat(boolean checkIP) { + testFailureLogFormatHelper(checkIP, null, null); + testFailureLogFormatHelper(checkIP, APPID, null); + testFailureLogFormatHelper(checkIP, null, SUBCLUSTERID); + testFailureLogFormatHelper(checkIP, APPID, SUBCLUSTERID); } /** * Test {@link RouterAuditLogger}. */ - @Test public void testRouterAuditLogger() throws Exception { - testSuccessLogFormat(); - testFailureLogFormat(); + @Test + public void testRouterAuditLoggerWithOutIP() throws Exception { + testSuccessLogFormat(false); + testFailureLogFormat(false); + } + + /** + * A special extension of {@link TestRPC.TestImpl} RPC server with + * {@link TestRPC.TestImpl#ping()} testing the audit logs. + */ + private class MyTestRouterRPCServer extends TestRpcBase.PBServerImpl { + @Override + public TestProtos.EmptyResponseProto ping( + RpcController unused, TestProtos.EmptyRequestProto request) + throws ServiceException { + // Ensure clientId is received + byte[] clientId = Server.getClientId(); + Assert.assertNotNull(clientId); + Assert.assertEquals(ClientId.BYTE_LENGTH, clientId.length); + // test with ip set + testSuccessLogFormat(true); + testFailureLogFormat(true); + return TestProtos.EmptyResponseProto.newBuilder().build(); + } + } + + /** + * Test {@link RouterAuditLogger} with IP set. + */ + @Test + public void testRouterAuditLoggerWithIP() throws Exception { + Configuration conf = new Configuration(); + RPC.setProtocolEngine(conf, TestRpcBase.TestRpcService.class, ProtobufRpcEngine2.class); + + // Create server side implementation + MyTestRouterRPCServer serverImpl = new MyTestRouterRPCServer(); + BlockingService service = TestRpcServiceProtos.TestProtobufRpcProto + .newReflectiveBlockingService(serverImpl); + + // start the IPC server + Server server = new RPC.Builder(conf) + .setProtocol(TestRpcBase.TestRpcService.class) + .setInstance(service).setBindAddress("0.0.0.0") + .setPort(0).setNumHandlers(5).setVerbose(true).build(); + + server.start(); + + InetSocketAddress address = NetUtils.getConnectAddress(server); + + // Make a client connection and test the audit log + TestRpcBase.TestRpcService proxy = null; + try { + proxy = RPC.getProxy(TestRpcBase.TestRpcService.class, + TestRPC.TestProtocol.versionID, address, conf); + // Start the testcase + TestProtos.EmptyRequestProto pingRequest = + TestProtos.EmptyRequestProto.newBuilder().build(); + proxy.ping(null, pingRequest); + } finally { + server.stop(); + if (proxy != null) { + RPC.stopProxy(proxy); + } + } } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/TestRouterMetrics.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/TestRouterMetrics.java index eddd2a0ab4816..cc36ca2a49093 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/TestRouterMetrics.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/TestRouterMetrics.java @@ -418,6 +418,66 @@ public void getSignalContainer() { LOG.info("Mocked: failed signalContainer call"); metrics.incrSignalToContainerFailedRetrieved(); } + + public void getQueueInfo() { + LOG.info("Mocked: failed getQueueInfo call"); + metrics.incrGetQueueInfoFailedRetrieved(); + } + + public void moveApplicationAcrossQueuesFailed() { + LOG.info("Mocked: failed moveApplicationAcrossQueuesFailed call"); + metrics.incrMoveApplicationAcrossQueuesFailedRetrieved(); + } + + public void getResourceProfilesFailed() { + LOG.info("Mocked: failed getResourceProfilesFailed call"); + metrics.incrGetResourceProfilesFailedRetrieved(); + } + + public void getResourceProfileFailed() { + LOG.info("Mocked: failed getResourceProfileFailed call"); + metrics.incrGetResourceProfileFailedRetrieved(); + } + + public void getAttributesToNodesFailed() { + LOG.info("Mocked: failed getAttributesToNodesFailed call"); + metrics.incrGetAttributesToNodesFailedRetrieved(); + } + + public void getClusterNodeAttributesFailed() { + LOG.info("Mocked: failed getClusterNodeAttributesFailed call"); + metrics.incrGetClusterNodeAttributesFailedRetrieved(); + } + + public void getNodesToAttributesFailed() { + LOG.info("Mocked: failed getNodesToAttributesFailed call"); + metrics.incrGetNodesToAttributesFailedRetrieved(); + } + + public void getNewReservationFailed() { + LOG.info("Mocked: failed getNewReservationFailed call"); + metrics.incrGetNewReservationFailedRetrieved(); + } + + public void getSubmitReservationFailed() { + LOG.info("Mocked: failed getSubmitReservationFailed call"); + metrics.incrSubmitReservationFailedRetrieved(); + } + + public void getUpdateReservationFailed() { + LOG.info("Mocked: failed getUpdateReservationFailed call"); + metrics.incrUpdateReservationFailedRetrieved(); + } + + public void getDeleteReservationFailed() { + LOG.info("Mocked: failed getDeleteReservationFailed call"); + metrics.incrDeleteReservationFailedRetrieved(); + } + + public void getListReservationFailed() { + LOG.info("Mocked: failed getListReservationFailed call"); + metrics.incrListReservationFailedRetrieved(); + } } // Records successes for all calls @@ -533,6 +593,66 @@ public void getSignalToContainerTimeouts(long duration) { LOG.info("Mocked: successful signalToContainer call with duration {}", duration); metrics.succeededSignalToContainerRetrieved(duration); } + + public void getQueueInfoRetrieved(long duration) { + LOG.info("Mocked: successful getQueueInfo call with duration {}", duration); + metrics.succeededGetQueueInfoRetrieved(duration); + } + + public void moveApplicationAcrossQueuesRetrieved(long duration) { + LOG.info("Mocked: successful moveApplicationAcrossQueues call with duration {}", duration); + metrics.succeededMoveApplicationAcrossQueuesRetrieved(duration); + } + + public void getResourceProfilesRetrieved(long duration) { + LOG.info("Mocked: successful getResourceProfiles call with duration {}", duration); + metrics.succeededGetResourceProfilesRetrieved(duration); + } + + public void getResourceProfileRetrieved(long duration) { + LOG.info("Mocked: successful getResourceProfile call with duration {}", duration); + metrics.succeededGetResourceProfileRetrieved(duration); + } + + public void getAttributesToNodesRetrieved(long duration) { + LOG.info("Mocked: successful getAttributesToNodes call with duration {}", duration); + metrics.succeededGetAttributesToNodesRetrieved(duration); + } + + public void getClusterNodeAttributesRetrieved(long duration) { + LOG.info("Mocked: successful getClusterNodeAttributes call with duration {}", duration); + metrics.succeededGetClusterNodeAttributesRetrieved(duration); + } + + public void getNodesToAttributesRetrieved(long duration) { + LOG.info("Mocked: successful getNodesToAttributes call with duration {}", duration); + metrics.succeededGetNodesToAttributesRetrieved(duration); + } + + public void getNewReservationRetrieved(long duration) { + LOG.info("Mocked: successful getNewReservation call with duration {}", duration); + metrics.succeededGetNewReservationRetrieved(duration); + } + + public void getSubmitReservationRetrieved(long duration) { + LOG.info("Mocked: successful getSubmitReservation call with duration {}", duration); + metrics.succeededSubmitReservationRetrieved(duration); + } + + public void getUpdateReservationRetrieved(long duration) { + LOG.info("Mocked: successful getUpdateReservation call with duration {}", duration); + metrics.succeededUpdateReservationRetrieved(duration); + } + + public void getDeleteReservationRetrieved(long duration) { + LOG.info("Mocked: successful getDeleteReservation call with duration {}", duration); + metrics.succeededDeleteReservationRetrieved(duration); + } + + public void getListReservationRetrieved(long duration) { + LOG.info("Mocked: successful getListReservation call with duration {}", duration); + metrics.succeededListReservationRetrieved(duration); + } } @Test @@ -839,4 +959,279 @@ public void testSignalToContainerFailed() { metrics.getSignalToContainerFailedRetrieved()); } + @Test + public void testSucceededGetQueueInfoRetrieved() { + long totalGoodBefore = metrics.getNumSucceededGetQueueInfoRetrieved(); + goodSubCluster.getQueueInfoRetrieved(150); + Assert.assertEquals(totalGoodBefore + 1, + metrics.getNumSucceededGetQueueInfoRetrieved()); + Assert.assertEquals(150, + metrics.getLatencySucceededGetQueueInfoRetrieved(), ASSERT_DOUBLE_DELTA); + goodSubCluster.getQueueInfoRetrieved(300); + Assert.assertEquals(totalGoodBefore + 2, + metrics.getNumSucceededGetQueueInfoRetrieved()); + Assert.assertEquals(225, + metrics.getLatencySucceededGetQueueInfoRetrieved(), ASSERT_DOUBLE_DELTA); + } + + @Test + public void testGetQueueInfoFailed() { + long totalBadBefore = metrics.getQueueInfoFailedRetrieved(); + badSubCluster.getQueueInfo(); + Assert.assertEquals(totalBadBefore + 1, + metrics.getQueueInfoFailedRetrieved()); + } + + @Test + public void testSucceededMoveApplicationAcrossQueuesRetrieved() { + long totalGoodBefore = metrics.getNumSucceededMoveApplicationAcrossQueuesRetrieved(); + goodSubCluster.moveApplicationAcrossQueuesRetrieved(150); + Assert.assertEquals(totalGoodBefore + 1, + metrics.getNumSucceededMoveApplicationAcrossQueuesRetrieved()); + Assert.assertEquals(150, + metrics.getLatencySucceededMoveApplicationAcrossQueuesRetrieved(), ASSERT_DOUBLE_DELTA); + goodSubCluster.moveApplicationAcrossQueuesRetrieved(300); + Assert.assertEquals(totalGoodBefore + 2, + metrics.getNumSucceededMoveApplicationAcrossQueuesRetrieved()); + Assert.assertEquals(225, + metrics.getLatencySucceededMoveApplicationAcrossQueuesRetrieved(), ASSERT_DOUBLE_DELTA); + } + + @Test + public void testMoveApplicationAcrossQueuesRetrievedFailed() { + long totalBadBefore = metrics.getMoveApplicationAcrossQueuesFailedRetrieved(); + badSubCluster.moveApplicationAcrossQueuesFailed(); + Assert.assertEquals(totalBadBefore + 1, + metrics.getMoveApplicationAcrossQueuesFailedRetrieved()); + } + + @Test + public void testSucceededGetResourceProfilesRetrieved() { + long totalGoodBefore = metrics.getNumSucceededGetResourceProfilesRetrieved(); + goodSubCluster.getResourceProfilesRetrieved(150); + Assert.assertEquals(totalGoodBefore + 1, + metrics.getNumSucceededGetResourceProfilesRetrieved()); + Assert.assertEquals(150, + metrics.getLatencySucceededGetResourceProfilesRetrieved(), ASSERT_DOUBLE_DELTA); + goodSubCluster.getResourceProfilesRetrieved(300); + Assert.assertEquals(totalGoodBefore + 2, + metrics.getNumSucceededGetResourceProfilesRetrieved()); + Assert.assertEquals(225, + metrics.getLatencySucceededGetResourceProfilesRetrieved(), ASSERT_DOUBLE_DELTA); + } + + @Test + public void testGetResourceProfilesRetrievedFailed() { + long totalBadBefore = metrics.getResourceProfilesFailedRetrieved(); + badSubCluster.getResourceProfilesFailed(); + Assert.assertEquals(totalBadBefore + 1, + metrics.getResourceProfilesFailedRetrieved()); + } + + @Test + public void testSucceededGetResourceProfileRetrieved() { + long totalGoodBefore = metrics.getNumSucceededGetResourceProfileRetrieved(); + goodSubCluster.getResourceProfileRetrieved(150); + Assert.assertEquals(totalGoodBefore + 1, + metrics.getNumSucceededGetResourceProfileRetrieved()); + Assert.assertEquals(150, + metrics.getLatencySucceededGetResourceProfileRetrieved(), ASSERT_DOUBLE_DELTA); + goodSubCluster.getResourceProfileRetrieved(300); + Assert.assertEquals(totalGoodBefore + 2, + metrics.getNumSucceededGetResourceProfileRetrieved()); + Assert.assertEquals(225, + metrics.getLatencySucceededGetResourceProfileRetrieved(), ASSERT_DOUBLE_DELTA); + } + + @Test + public void testGetResourceProfileRetrievedFailed() { + long totalBadBefore = metrics.getResourceProfileFailedRetrieved(); + badSubCluster.getResourceProfileFailed(); + Assert.assertEquals(totalBadBefore + 1, + metrics.getResourceProfileFailedRetrieved()); + } + + @Test + public void testSucceededGetAttributesToNodesRetrieved() { + long totalGoodBefore = metrics.getNumSucceededGetAttributesToNodesRetrieved(); + goodSubCluster.getAttributesToNodesRetrieved(150); + Assert.assertEquals(totalGoodBefore + 1, + metrics.getNumSucceededGetAttributesToNodesRetrieved()); + Assert.assertEquals(150, + metrics.getLatencySucceededGetAttributesToNodesRetrieved(), ASSERT_DOUBLE_DELTA); + goodSubCluster.getAttributesToNodesRetrieved(300); + Assert.assertEquals(totalGoodBefore + 2, + metrics.getNumSucceededGetAttributesToNodesRetrieved()); + Assert.assertEquals(225, + metrics.getLatencySucceededGetAttributesToNodesRetrieved(), ASSERT_DOUBLE_DELTA); + } + + @Test + public void testGetAttributesToNodesRetrievedFailed() { + long totalBadBefore = metrics.getAttributesToNodesFailedRetrieved(); + badSubCluster.getAttributesToNodesFailed(); + Assert.assertEquals(totalBadBefore + 1, + metrics.getAttributesToNodesFailedRetrieved()); + } + + @Test + public void testGetClusterNodeAttributesRetrieved() { + long totalGoodBefore = metrics.getNumSucceededGetClusterNodeAttributesRetrieved(); + goodSubCluster.getClusterNodeAttributesRetrieved(150); + Assert.assertEquals(totalGoodBefore + 1, + metrics.getNumSucceededGetClusterNodeAttributesRetrieved()); + Assert.assertEquals(150, + metrics.getLatencySucceededGetClusterNodeAttributesRetrieved(), ASSERT_DOUBLE_DELTA); + goodSubCluster.getClusterNodeAttributesRetrieved(300); + Assert.assertEquals(totalGoodBefore + 2, + metrics.getNumSucceededGetClusterNodeAttributesRetrieved()); + Assert.assertEquals(225, + metrics.getLatencySucceededGetClusterNodeAttributesRetrieved(), ASSERT_DOUBLE_DELTA); + } + + @Test + public void testGetClusterNodeAttributesRetrievedFailed() { + long totalBadBefore = metrics.getClusterNodeAttributesFailedRetrieved(); + badSubCluster.getClusterNodeAttributesFailed(); + Assert.assertEquals(totalBadBefore + 1, + metrics.getClusterNodeAttributesFailedRetrieved()); + } + + @Test + public void testGetNodesToAttributesRetrieved() { + long totalGoodBefore = metrics.getNumSucceededGetNodesToAttributesRetrieved(); + goodSubCluster.getNodesToAttributesRetrieved(150); + Assert.assertEquals(totalGoodBefore + 1, + metrics.getNumSucceededGetNodesToAttributesRetrieved()); + Assert.assertEquals(150, + metrics.getLatencySucceededGetNodesToAttributesRetrieved(), ASSERT_DOUBLE_DELTA); + goodSubCluster.getNodesToAttributesRetrieved(300); + Assert.assertEquals(totalGoodBefore + 2, + metrics.getNumSucceededGetNodesToAttributesRetrieved()); + Assert.assertEquals(225, + metrics.getLatencySucceededGetNodesToAttributesRetrieved(), ASSERT_DOUBLE_DELTA); + } + + @Test + public void testGetNodesToAttributesRetrievedFailed() { + long totalBadBefore = metrics.getNodesToAttributesFailedRetrieved(); + badSubCluster.getNodesToAttributesFailed(); + Assert.assertEquals(totalBadBefore + 1, + metrics.getNodesToAttributesFailedRetrieved()); + } + + @Test + public void testGetNewReservationRetrieved() { + long totalGoodBefore = metrics.getNumSucceededGetNewReservationRetrieved(); + goodSubCluster.getNewReservationRetrieved(150); + Assert.assertEquals(totalGoodBefore + 1, + metrics.getNumSucceededGetNewReservationRetrieved()); + Assert.assertEquals(150, + metrics.getLatencySucceededGetNewReservationRetrieved(), ASSERT_DOUBLE_DELTA); + goodSubCluster.getNewReservationRetrieved(300); + Assert.assertEquals(totalGoodBefore + 2, + metrics.getNumSucceededGetNewReservationRetrieved()); + Assert.assertEquals(225, + metrics.getLatencySucceededGetNewReservationRetrieved(), ASSERT_DOUBLE_DELTA); + } + + @Test + public void testGetNewReservationRetrievedFailed() { + long totalBadBefore = metrics.getNewReservationFailedRetrieved(); + badSubCluster.getNewReservationFailed(); + Assert.assertEquals(totalBadBefore + 1, + metrics.getNewReservationFailedRetrieved()); + } + + @Test + public void testGetSubmitReservationRetrieved() { + long totalGoodBefore = metrics.getNumSucceededSubmitReservationRetrieved(); + goodSubCluster.getSubmitReservationRetrieved(150); + Assert.assertEquals(totalGoodBefore + 1, + metrics.getNumSucceededSubmitReservationRetrieved()); + Assert.assertEquals(150, + metrics.getLatencySucceededSubmitReservationRetrieved(), ASSERT_DOUBLE_DELTA); + goodSubCluster.getSubmitReservationRetrieved(300); + Assert.assertEquals(totalGoodBefore + 2, + metrics.getNumSucceededSubmitReservationRetrieved()); + Assert.assertEquals(225, + metrics.getLatencySucceededSubmitReservationRetrieved(), ASSERT_DOUBLE_DELTA); + } + + @Test + public void testGetSubmitReservationRetrievedFailed() { + long totalBadBefore = metrics.getSubmitReservationFailedRetrieved(); + badSubCluster.getSubmitReservationFailed(); + Assert.assertEquals(totalBadBefore + 1, + metrics.getSubmitReservationFailedRetrieved()); + } + + @Test + public void testGetUpdateReservationRetrieved() { + long totalGoodBefore = metrics.getNumSucceededUpdateReservationRetrieved(); + goodSubCluster.getUpdateReservationRetrieved(150); + Assert.assertEquals(totalGoodBefore + 1, + metrics.getNumSucceededUpdateReservationRetrieved()); + Assert.assertEquals(150, + metrics.getLatencySucceededUpdateReservationRetrieved(), ASSERT_DOUBLE_DELTA); + goodSubCluster.getUpdateReservationRetrieved(300); + Assert.assertEquals(totalGoodBefore + 2, + metrics.getNumSucceededUpdateReservationRetrieved()); + Assert.assertEquals(225, + metrics.getLatencySucceededUpdateReservationRetrieved(), ASSERT_DOUBLE_DELTA); + } + + @Test + public void testGetUpdateReservationRetrievedFailed() { + long totalBadBefore = metrics.getUpdateReservationFailedRetrieved(); + badSubCluster.getUpdateReservationFailed(); + Assert.assertEquals(totalBadBefore + 1, + metrics.getUpdateReservationFailedRetrieved()); + } + + @Test + public void testGetDeleteReservationRetrieved() { + long totalGoodBefore = metrics.getNumSucceededDeleteReservationRetrieved(); + goodSubCluster.getDeleteReservationRetrieved(150); + Assert.assertEquals(totalGoodBefore + 1, + metrics.getNumSucceededDeleteReservationRetrieved()); + Assert.assertEquals(150, + metrics.getLatencySucceededDeleteReservationRetrieved(), ASSERT_DOUBLE_DELTA); + goodSubCluster.getDeleteReservationRetrieved(300); + Assert.assertEquals(totalGoodBefore + 2, + metrics.getNumSucceededDeleteReservationRetrieved()); + Assert.assertEquals(225, + metrics.getLatencySucceededDeleteReservationRetrieved(), ASSERT_DOUBLE_DELTA); + } + + @Test + public void testGetDeleteReservationRetrievedFailed() { + long totalBadBefore = metrics.getDeleteReservationFailedRetrieved(); + badSubCluster.getDeleteReservationFailed(); + Assert.assertEquals(totalBadBefore + 1, + metrics.getDeleteReservationFailedRetrieved()); + } + + @Test + public void testGetListReservationRetrieved() { + long totalGoodBefore = metrics.getNumSucceededListReservationRetrieved(); + goodSubCluster.getListReservationRetrieved(150); + Assert.assertEquals(totalGoodBefore + 1, + metrics.getNumSucceededListReservationRetrieved()); + Assert.assertEquals(150, + metrics.getLatencySucceededListReservationRetrieved(), ASSERT_DOUBLE_DELTA); + goodSubCluster.getListReservationRetrieved(300); + Assert.assertEquals(totalGoodBefore + 2, + metrics.getNumSucceededListReservationRetrieved()); + Assert.assertEquals(225, + metrics.getLatencySucceededListReservationRetrieved(), ASSERT_DOUBLE_DELTA); + } + + @Test + public void testGetListReservationRetrievedFailed() { + long totalBadBefore = metrics.getListReservationFailedRetrieved(); + badSubCluster.getListReservationFailed(); + Assert.assertEquals(totalBadBefore + 1, + metrics.getListReservationFailedRetrieved()); + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/clientrm/BaseRouterClientRMTest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/clientrm/BaseRouterClientRMTest.java index 1d726d3ff1697..905b60f31ee82 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/clientrm/BaseRouterClientRMTest.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/clientrm/BaseRouterClientRMTest.java @@ -134,7 +134,7 @@ protected Configuration createConfiguration() { String mockPassThroughInterceptorClass = PassThroughClientRequestInterceptor.class.getName(); - // Create a request intercepter pipeline for testing. The last one in the + // Create a request interceptor pipeline for testing. The last one in the // chain will call the mock resource manager. The others in the chain will // simply forward it to the next one in the chain config.set(YarnConfiguration.ROUTER_CLIENTRM_INTERCEPTOR_CLASS_PIPELINE, diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/clientrm/MockClientRequestInterceptor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/clientrm/MockClientRequestInterceptor.java index e0538b8a422db..c85bbd6f2eaf1 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/clientrm/MockClientRequestInterceptor.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/clientrm/MockClientRequestInterceptor.java @@ -31,8 +31,10 @@ public class MockClientRequestInterceptor extends DefaultClientRequestInterceptor { + MockRM mockRM = null; + public void init(String user) { - MockRM mockRM = new MockRM(super.getConf()) { + mockRM = new MockRM(super.getConf()) { @Override protected ClientRMService createClientRMService() { return new ClientRMService(getRMContext(), getResourceScheduler(), @@ -68,4 +70,11 @@ public MoveApplicationAcrossQueuesResponse moveApplicationAcrossQueues( super.setRMClient(mockRM.getClientRMService()); } + @Override + public void shutdown() { + if (mockRM != null) { + mockRM.stop(); + } + super.shutdown(); + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/clientrm/PassThroughClientRequestInterceptor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/clientrm/PassThroughClientRequestInterceptor.java index a35feaeaa319a..d4820fc12b560 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/clientrm/PassThroughClientRequestInterceptor.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/clientrm/PassThroughClientRequestInterceptor.java @@ -93,8 +93,8 @@ import org.apache.hadoop.yarn.exceptions.YarnException; /** - * Mock intercepter that does not do anything other than forwarding it to the - * next intercepter in the chain. + * Mock interceptor that does not do anything other than forwarding it to the + * next interceptor in the chain. */ public class PassThroughClientRequestInterceptor extends AbstractClientRequestInterceptor { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/clientrm/TestFederationClientInterceptor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/clientrm/TestFederationClientInterceptor.java index 3037738240266..ac980b48581c9 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/clientrm/TestFederationClientInterceptor.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/clientrm/TestFederationClientInterceptor.java @@ -29,8 +29,11 @@ import java.util.HashMap; import java.util.Set; import java.util.stream.Collectors; +import java.util.Arrays; +import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem; import org.apache.hadoop.test.LambdaTestUtils; +import org.apache.hadoop.util.Time; import org.apache.hadoop.yarn.MockApps; import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationAttemptReportRequest; import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationAttemptReportResponse; @@ -74,6 +77,28 @@ import org.apache.hadoop.yarn.api.protocolrecords.UpdateApplicationTimeoutsResponse; import org.apache.hadoop.yarn.api.protocolrecords.SignalContainerRequest; import org.apache.hadoop.yarn.api.protocolrecords.SignalContainerResponse; +import org.apache.hadoop.yarn.api.protocolrecords.MoveApplicationAcrossQueuesRequest; +import org.apache.hadoop.yarn.api.protocolrecords.MoveApplicationAcrossQueuesResponse; +import org.apache.hadoop.yarn.api.protocolrecords.GetQueueInfoRequest; +import org.apache.hadoop.yarn.api.protocolrecords.GetQueueInfoResponse; +import org.apache.hadoop.yarn.api.protocolrecords.GetAllResourceProfilesRequest; +import org.apache.hadoop.yarn.api.protocolrecords.GetAllResourceProfilesResponse; +import org.apache.hadoop.yarn.api.protocolrecords.GetResourceProfileRequest; +import org.apache.hadoop.yarn.api.protocolrecords.GetResourceProfileResponse; +import org.apache.hadoop.yarn.api.protocolrecords.GetAttributesToNodesRequest; +import org.apache.hadoop.yarn.api.protocolrecords.GetAttributesToNodesResponse; +import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodeAttributesRequest; +import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodeAttributesResponse; +import org.apache.hadoop.yarn.api.protocolrecords.GetNodesToAttributesRequest; +import org.apache.hadoop.yarn.api.protocolrecords.GetNodesToAttributesResponse; +import org.apache.hadoop.yarn.api.protocolrecords.GetNewReservationRequest; +import org.apache.hadoop.yarn.api.protocolrecords.GetNewReservationResponse; +import org.apache.hadoop.yarn.api.protocolrecords.ReservationSubmissionRequest; +import org.apache.hadoop.yarn.api.protocolrecords.ReservationSubmissionResponse; +import org.apache.hadoop.yarn.api.protocolrecords.ReservationUpdateRequest; +import org.apache.hadoop.yarn.api.protocolrecords.ReservationUpdateResponse; +import org.apache.hadoop.yarn.api.protocolrecords.ReservationDeleteRequest; +import org.apache.hadoop.yarn.api.protocolrecords.ReservationDeleteResponse; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext; @@ -86,6 +111,17 @@ import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.ApplicationTimeoutType; import org.apache.hadoop.yarn.api.records.SignalContainerCommand; +import org.apache.hadoop.yarn.api.records.QueueInfo; +import org.apache.hadoop.yarn.api.records.Resource; +import org.apache.hadoop.yarn.api.records.NodeAttributeKey; +import org.apache.hadoop.yarn.api.records.NodeToAttributeValue; +import org.apache.hadoop.yarn.api.records.NodeAttribute; +import org.apache.hadoop.yarn.api.records.NodeAttributeInfo; +import org.apache.hadoop.yarn.api.records.NodeAttributeType; +import org.apache.hadoop.yarn.api.records.ReservationRequest; +import org.apache.hadoop.yarn.api.records.ReservationDefinition; +import org.apache.hadoop.yarn.api.records.ReservationRequestInterpreter; +import org.apache.hadoop.yarn.api.records.ReservationRequests; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.server.federation.policies.manager.UniformBroadcastPolicyManager; @@ -96,6 +132,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.MockRM; import org.apache.hadoop.yarn.server.resourcemanager.MockNM; import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager; +import org.apache.hadoop.yarn.server.resourcemanager.reservation.ReservationSystem; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppState; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptState; @@ -111,7 +148,7 @@ * use the {@code RouterClientRMService} pipeline test cases for testing the * {@code FederationInterceptor} class. The tests for * {@code RouterClientRMService} has been written cleverly so that it can be - * reused to validate different request intercepter chains. + * reused to validate different request interceptor chains. */ public class TestFederationClientInterceptor extends BaseRouterClientRMTest { private static final Logger LOG = @@ -128,6 +165,8 @@ public class TestFederationClientInterceptor extends BaseRouterClientRMTest { private final static int APP_PRIORITY_ZERO = 0; + private final static long DEFAULT_DURATION = 10 * 60 * 1000; + @Override public void setUp() { super.setUpConfig(); @@ -135,14 +174,13 @@ public void setUp() { stateStore = new MemoryFederationStateStore(); stateStore.init(this.getConf()); - FederationStateStoreFacade.getInstance().reinitialize(stateStore, - getConf()); + FederationStateStoreFacade.getInstance().reinitialize(stateStore, getConf()); stateStoreUtil = new FederationStateStoreTestUtil(stateStore); interceptor.setConf(this.getConf()); interceptor.init(user); - subClusters = new ArrayList(); + subClusters = new ArrayList<>(); try { for (int i = 0; i < NUM_SUBCLUSTER; i++) { @@ -155,6 +193,7 @@ public void setUp() { Assert.fail(); } + DefaultMetricsSystem.setMiniClusterMode(true); } @Override @@ -170,13 +209,13 @@ protected YarnConfiguration createConfiguration() { String mockPassThroughInterceptorClass = PassThroughClientRequestInterceptor.class.getName(); - // Create a request intercepter pipeline for testing. The last one in the - // chain is the federation intercepter that calls the mock resource manager. + // Create a request interceptor pipeline for testing. The last one in the + // chain is the federation interceptor that calls the mock resource manager. // The others in the chain will simply forward it to the next one in the // chain conf.set(YarnConfiguration.ROUTER_CLIENTRM_INTERCEPTOR_CLASS_PIPELINE, mockPassThroughInterceptorClass + "," + mockPassThroughInterceptorClass - + "," + TestableFederationClientInterceptor.class.getName()); + + "," + TestableFederationClientInterceptor.class.getName()); conf.set(YarnConfiguration.FEDERATION_POLICY_MANAGER, UniformBroadcastPolicyManager.class.getName()); @@ -184,6 +223,11 @@ protected YarnConfiguration createConfiguration() { // Disable StateStoreFacade cache conf.setInt(YarnConfiguration.FEDERATION_CACHE_TIME_TO_LIVE_SECS, 0); + conf.setInt("yarn.scheduler.minimum-allocation-mb", 512); + conf.setInt("yarn.scheduler.minimum-allocation-vcores", 1); + conf.setInt("yarn.scheduler.maximum-allocation-mb", 100 * 1024); + conf.setInt("yarn.scheduler.maximum-allocation-vcores", 100); + return conf; } @@ -192,17 +236,16 @@ protected YarnConfiguration createConfiguration() { * ApplicationId has to belong to one of the SubCluster in the cluster. */ @Test - public void testGetNewApplication() - throws YarnException, IOException, InterruptedException { - LOG.info("Test FederationClientInterceptor: Get New Application"); + public void testGetNewApplication() throws YarnException, IOException { + LOG.info("Test FederationClientInterceptor: Get New Application."); GetNewApplicationRequest request = GetNewApplicationRequest.newInstance(); GetNewApplicationResponse response = interceptor.getNewApplication(request); Assert.assertNotNull(response); Assert.assertNotNull(response.getApplicationId()); - Assert.assertTrue(response.getApplicationId() - .getClusterTimestamp() == ResourceManager.getClusterTimeStamp()); + Assert.assertEquals(response.getApplicationId().getClusterTimestamp(), + ResourceManager.getClusterTimeStamp()); } /** @@ -212,10 +255,9 @@ public void testGetNewApplication() @Test public void testSubmitApplication() throws YarnException, IOException { - LOG.info("Test FederationClientInterceptor: Submit Application"); + LOG.info("Test FederationClientInterceptor: Submit Application."); - ApplicationId appId = ApplicationId.newInstance(System.currentTimeMillis(), - 1); + ApplicationId appId = ApplicationId.newInstance(System.currentTimeMillis(), 1); SubmitApplicationRequest request = mockSubmitApplicationRequest(appId); SubmitApplicationResponse response = interceptor.submitApplication(request); @@ -229,14 +271,12 @@ public void testSubmitApplication() private SubmitApplicationRequest mockSubmitApplicationRequest( ApplicationId appId) { ContainerLaunchContext amContainerSpec = mock(ContainerLaunchContext.class); - ApplicationSubmissionContext context = ApplicationSubmissionContext - .newInstance(appId, MockApps.newAppName(), "default", - Priority.newInstance(APP_PRIORITY_ZERO), amContainerSpec, false, false, -1, - Resources.createResource( - YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_MB), - "MockApp"); - SubmitApplicationRequest request = SubmitApplicationRequest - .newInstance(context); + ApplicationSubmissionContext context = ApplicationSubmissionContext.newInstance( + appId, MockApps.newAppName(), "default", + Priority.newInstance(APP_PRIORITY_ZERO), amContainerSpec, false, false, -1, + Resources.createResource(YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_MB), + "MockApp"); + SubmitApplicationRequest request = SubmitApplicationRequest.newInstance(context); return request; } @@ -277,37 +317,27 @@ public void testSubmitApplicationMultipleSubmission() */ @Test public void testSubmitApplicationEmptyRequest() - throws YarnException, IOException, InterruptedException { - LOG.info( - "Test FederationClientInterceptor: Submit Application - Empty"); - try { - interceptor.submitApplication(null); - Assert.fail(); - } catch (YarnException e) { - Assert.assertTrue( - e.getMessage().startsWith("Missing submitApplication request or " - + "applicationSubmissionContext information.")); - } - try { - interceptor.submitApplication(SubmitApplicationRequest.newInstance(null)); - Assert.fail(); - } catch (YarnException e) { - Assert.assertTrue( - e.getMessage().startsWith("Missing submitApplication request or " - + "applicationSubmissionContext information.")); - } - try { - ApplicationSubmissionContext context = ApplicationSubmissionContext - .newInstance(null, "", "", null, null, false, false, -1, null, null); - SubmitApplicationRequest request = - SubmitApplicationRequest.newInstance(context); - interceptor.submitApplication(request); - Assert.fail(); - } catch (YarnException e) { - Assert.assertTrue( - e.getMessage().startsWith("Missing submitApplication request or " - + "applicationSubmissionContext information.")); - } + throws Exception { + LOG.info("Test FederationClientInterceptor: Submit Application - Empty."); + + // null request1 + LambdaTestUtils.intercept(YarnException.class, + "Missing submitApplication request or applicationSubmissionContext information.", + () -> interceptor.submitApplication(null)); + + // null request2 + LambdaTestUtils.intercept(YarnException.class, + "Missing submitApplication request or applicationSubmissionContext information.", + () -> interceptor.submitApplication(SubmitApplicationRequest.newInstance(null))); + + // null request3 + ApplicationSubmissionContext context = ApplicationSubmissionContext + .newInstance(null, "", "", null, null, false, false, -1, null, null); + SubmitApplicationRequest request = + SubmitApplicationRequest.newInstance(context); + LambdaTestUtils.intercept(YarnException.class, + "Missing submitApplication request or applicationSubmissionContext information.", + () -> interceptor.submitApplication(request)); } /** @@ -317,7 +347,7 @@ public void testSubmitApplicationEmptyRequest() @Test public void testForceKillApplication() throws YarnException, IOException, InterruptedException { - LOG.info("Test FederationClientInterceptor: Force Kill Application"); + LOG.info("Test FederationClientInterceptor: Force Kill Application."); ApplicationId appId = ApplicationId.newInstance(System.currentTimeMillis(), 1); @@ -329,10 +359,8 @@ public void testForceKillApplication() Assert.assertNotNull(response); Assert.assertNotNull(stateStoreUtil.queryApplicationHomeSC(appId)); - KillApplicationRequest requestKill = - KillApplicationRequest.newInstance(appId); - KillApplicationResponse responseKill = - interceptor.forceKillApplication(requestKill); + KillApplicationRequest requestKill = KillApplicationRequest.newInstance(appId); + KillApplicationResponse responseKill = interceptor.forceKillApplication(requestKill); Assert.assertNotNull(responseKill); } @@ -341,22 +369,17 @@ public void testForceKillApplication() * application does not exist in StateStore. */ @Test - public void testForceKillApplicationNotExists() - throws YarnException, IOException, InterruptedException { - LOG.info("Test FederationClientInterceptor: " - + "Force Kill Application - Not Exists"); + public void testForceKillApplicationNotExists() throws Exception { + LOG.info("Test FederationClientInterceptor: Force Kill Application - Not Exists"); ApplicationId appId = ApplicationId.newInstance(System.currentTimeMillis(), 1); KillApplicationRequest requestKill = KillApplicationRequest.newInstance(appId); - try { - interceptor.forceKillApplication(requestKill); - Assert.fail(); - } catch (YarnException e) { - Assert.assertTrue(e.getMessage().equals( - "Application " + appId + " does not exist in FederationStateStore")); - } + + LambdaTestUtils.intercept(YarnException.class, + "Application " + appId + " does not exist in FederationStateStore.", + () -> interceptor.forceKillApplication(requestKill)); } /** @@ -365,24 +388,19 @@ public void testForceKillApplicationNotExists() */ @Test public void testForceKillApplicationEmptyRequest() - throws YarnException, IOException, InterruptedException { - LOG.info( - "Test FederationClientInterceptor: Force Kill Application - Empty"); - try { - interceptor.forceKillApplication(null); - Assert.fail(); - } catch (YarnException e) { - Assert.assertTrue(e.getMessage().startsWith( - "Missing forceKillApplication request or ApplicationId.")); - } - try { - interceptor - .forceKillApplication(KillApplicationRequest.newInstance(null)); - Assert.fail(); - } catch (YarnException e) { - Assert.assertTrue(e.getMessage().startsWith( - "Missing forceKillApplication request or ApplicationId.")); - } + throws Exception { + LOG.info("Test FederationClientInterceptor: Force Kill Application - Empty."); + + // null request1 + LambdaTestUtils.intercept(YarnException.class, + "Missing forceKillApplication request or ApplicationId.", + () -> interceptor.forceKillApplication(null)); + + // null request2 + KillApplicationRequest killRequest = KillApplicationRequest.newInstance(null); + LambdaTestUtils.intercept(YarnException.class, + "Missing forceKillApplication request or ApplicationId.", + () -> interceptor.forceKillApplication(killRequest)); } /** @@ -419,20 +437,14 @@ public void testGetApplicationReport() */ @Test public void testGetApplicationNotExists() - throws YarnException, IOException, InterruptedException { - LOG.info( - "Test ApplicationClientProtocol: Get Application Report - Not Exists"); - ApplicationId appId = - ApplicationId.newInstance(System.currentTimeMillis(), 1); - GetApplicationReportRequest requestGet = - GetApplicationReportRequest.newInstance(appId); - try { - interceptor.getApplicationReport(requestGet); - Assert.fail(); - } catch (YarnException e) { - Assert.assertTrue(e.getMessage().equals( - "Application " + appId + " does not exist in FederationStateStore")); - } + throws Exception { + LOG.info("Test ApplicationClientProtocol: Get Application Report - Not Exists."); + + ApplicationId appId = ApplicationId.newInstance(System.currentTimeMillis(), 1); + GetApplicationReportRequest requestGet = GetApplicationReportRequest.newInstance(appId); + LambdaTestUtils.intercept(YarnException.class, + "Application " + appId + " does not exist in FederationStateStore.", + () -> interceptor.getApplicationReport(requestGet)); } /** @@ -441,31 +453,23 @@ public void testGetApplicationNotExists() */ @Test public void testGetApplicationEmptyRequest() - throws YarnException, IOException, InterruptedException { - LOG.info( - "Test FederationClientInterceptor: Get Application Report - Empty"); - try { - interceptor.getApplicationReport(null); - Assert.fail(); - } catch (YarnException e) { - Assert.assertTrue( - e.getMessage().startsWith("Missing getApplicationReport request or " - + "applicationId information.")); - } - try { - interceptor - .getApplicationReport(GetApplicationReportRequest.newInstance(null)); - Assert.fail(); - } catch (YarnException e) { - Assert.assertTrue( - e.getMessage().startsWith("Missing getApplicationReport request or " - + "applicationId information.")); - } + throws Exception { + LOG.info("Test FederationClientInterceptor: Get Application Report - Empty."); + + // null request1 + LambdaTestUtils.intercept(YarnException.class, + "Missing getApplicationReport request or applicationId information.", + () -> interceptor.getApplicationReport(null)); + + // null request2 + GetApplicationReportRequest reportRequest = GetApplicationReportRequest.newInstance(null); + LambdaTestUtils.intercept(YarnException.class, + "Missing getApplicationReport request or applicationId information.", + () -> interceptor.getApplicationReport(reportRequest)); } /** - * This test validates the correctness of - * GetApplicationAttemptReport in case the + * This test validates the correctness of GetApplicationAttemptReport in case the * application exists in the cluster. */ @Test @@ -509,77 +513,68 @@ public void testGetApplicationAttemptReport() } /** - * This test validates the correctness of - * GetApplicationAttemptReport in case the + * This test validates the correctness of GetApplicationAttemptReport in case the * application does not exist in StateStore. */ @Test - public void testGetApplicationAttemptNotExists() - throws Exception { - LOG.info( - "Test ApplicationClientProtocol: " + - "Get ApplicationAttempt Report - Not Exists"); + public void testGetApplicationAttemptNotExists() throws Exception { + LOG.info("Test FederationClientInterceptor: Get ApplicationAttempt Report - Not Exists."); + ApplicationId appId = - ApplicationId.newInstance(System.currentTimeMillis(), 1); + ApplicationId.newInstance(System.currentTimeMillis(), 1); ApplicationAttemptId appAttemptID = - ApplicationAttemptId.newInstance(appId, 1); + ApplicationAttemptId.newInstance(appId, 1); GetApplicationAttemptReportRequest requestGet = - GetApplicationAttemptReportRequest.newInstance(appAttemptID); + GetApplicationAttemptReportRequest.newInstance(appAttemptID); LambdaTestUtils.intercept(YarnException.class, "ApplicationAttempt " + - appAttemptID + " belongs to Application " + - appId + " does not exist in FederationStateStore.", + appAttemptID + " belongs to Application " + + appId + " does not exist in FederationStateStore.", () -> interceptor.getApplicationAttemptReport(requestGet)); } /** - * This test validates - * the correctness of GetApplicationAttemptReport in case of + * This test validates the correctness of GetApplicationAttemptReport in case of * empty request. */ @Test public void testGetApplicationAttemptEmptyRequest() - throws Exception { - LOG.info("Test FederationClientInterceptor: " + - "Get ApplicationAttempt Report - Empty"); + throws Exception { + LOG.info("Test FederationClientInterceptor: Get ApplicationAttempt Report - Empty."); + // null request1 LambdaTestUtils.intercept(YarnException.class, - "Missing getApplicationAttemptReport " + - "request or applicationId " + - "or applicationAttemptId information.", + "Missing getApplicationAttemptReport request or applicationId " + + "or applicationAttemptId information.", () -> interceptor.getApplicationAttemptReport(null)); + // null request2 LambdaTestUtils.intercept(YarnException.class, - "Missing getApplicationAttemptReport " + - "request or applicationId " + - "or applicationAttemptId information.", - () -> interceptor - .getApplicationAttemptReport( - GetApplicationAttemptReportRequest - .newInstance(null))); + "Missing getApplicationAttemptReport request or applicationId " + + "or applicationAttemptId information.", + () -> interceptor.getApplicationAttemptReport( + GetApplicationAttemptReportRequest.newInstance(null))); + // null request3 LambdaTestUtils.intercept(YarnException.class, - "Missing getApplicationAttemptReport " + - "request or applicationId " + - "or applicationAttemptId information.", - () -> interceptor - .getApplicationAttemptReport( - GetApplicationAttemptReportRequest.newInstance( - ApplicationAttemptId - .newInstance(null, 1) - ))); + "Missing getApplicationAttemptReport request or applicationId " + + "or applicationAttemptId information.", + () -> interceptor.getApplicationAttemptReport( + GetApplicationAttemptReportRequest.newInstance( + ApplicationAttemptId.newInstance(null, 1)))); } @Test - public void testGetClusterMetricsRequest() throws YarnException, IOException { - LOG.info("Test FederationClientInterceptor : Get Cluster Metrics request"); + public void testGetClusterMetricsRequest() throws Exception { + LOG.info("Test FederationClientInterceptor : Get Cluster Metrics request."); + // null request - GetClusterMetricsResponse response = interceptor.getClusterMetrics(null); - Assert.assertEquals(subClusters.size(), - response.getClusterMetrics().getNumNodeManagers()); + LambdaTestUtils.intercept(YarnException.class, "Missing getClusterMetrics request.", + () -> interceptor.getClusterMetrics(null)); + // normal request. - response = + GetClusterMetricsResponse response = interceptor.getClusterMetrics(GetClusterMetricsRequest.newInstance()); Assert.assertEquals(subClusters.size(), response.getClusterMetrics().getNumNodeManagers()); @@ -587,23 +582,20 @@ public void testGetClusterMetricsRequest() throws YarnException, IOException { ClientMethod remoteMethod = new ClientMethod("getClusterMetrics", new Class[] {GetClusterMetricsRequest.class}, new Object[] {GetClusterMetricsRequest.newInstance()}); - Map clusterMetrics =interceptor. - invokeConcurrent(new ArrayList<>(), remoteMethod, - GetClusterMetricsResponse.class); - Assert.assertEquals(true, clusterMetrics.isEmpty()); + Map clusterMetrics = interceptor. + invokeConcurrent(new ArrayList<>(), remoteMethod, GetClusterMetricsResponse.class); + Assert.assertTrue(clusterMetrics.isEmpty()); } /** - * This test validates the correctness of - * GetApplicationsResponse in case the + * This test validates the correctness of GetApplicationsResponse in case the * application exists in the cluster. */ @Test public void testGetApplicationsResponse() throws YarnException, IOException, InterruptedException { - LOG.info("Test FederationClientInterceptor: Get Applications Response"); - ApplicationId appId = - ApplicationId.newInstance(System.currentTimeMillis(), 1); + LOG.info("Test FederationClientInterceptor: Get Applications Response."); + ApplicationId appId = ApplicationId.newInstance(System.currentTimeMillis(), 1); SubmitApplicationRequest request = mockSubmitApplicationRequest(appId); SubmitApplicationResponse response = interceptor.submitApplication(request); @@ -612,40 +604,32 @@ public void testGetApplicationsResponse() Assert.assertNotNull(stateStoreUtil.queryApplicationHomeSC(appId)); Set appTypes = Collections.singleton("MockApp"); - GetApplicationsRequest requestGet = - GetApplicationsRequest.newInstance(appTypes); - - GetApplicationsResponse responseGet = - interceptor.getApplications(requestGet); + GetApplicationsRequest requestGet = GetApplicationsRequest.newInstance(appTypes); + GetApplicationsResponse responseGet = interceptor.getApplications(requestGet); Assert.assertNotNull(responseGet); } /** - * This test validates - * the correctness of GetApplicationsResponse in case of + * This test validates the correctness of GetApplicationsResponse in case of * empty request. */ @Test public void testGetApplicationsNullRequest() throws Exception { - LOG.info("Test FederationClientInterceptor: Get Applications request"); - LambdaTestUtils.intercept(YarnException.class, - "Missing getApplications request.", + LOG.info("Test FederationClientInterceptor: Get Applications request."); + LambdaTestUtils.intercept(YarnException.class, "Missing getApplications request.", () -> interceptor.getApplications(null)); } /** - * This test validates - * the correctness of GetApplicationsResponse in case applications + * This test validates the correctness of GetApplicationsResponse in case applications * with given type does not exist. */ @Test public void testGetApplicationsApplicationTypeNotExists() throws Exception{ - LOG.info("Test FederationClientInterceptor: Application with type does " - + "not exist"); + LOG.info("Test FederationClientInterceptor: Application with type does not exist."); - ApplicationId appId = - ApplicationId.newInstance(System.currentTimeMillis(), 1); + ApplicationId appId = ApplicationId.newInstance(System.currentTimeMillis(), 1); SubmitApplicationRequest request = mockSubmitApplicationRequest(appId); SubmitApplicationResponse response = interceptor.submitApplication(request); @@ -655,25 +639,20 @@ public void testGetApplicationsApplicationTypeNotExists() throws Exception{ Set appTypes = Collections.singleton("SPARK"); - GetApplicationsRequest requestGet = - GetApplicationsRequest.newInstance(appTypes); - - GetApplicationsResponse responseGet = - interceptor.getApplications(requestGet); + GetApplicationsRequest requestGet = GetApplicationsRequest.newInstance(appTypes); + GetApplicationsResponse responseGet = interceptor.getApplications(requestGet); Assert.assertNotNull(responseGet); Assert.assertTrue(responseGet.getApplicationList().isEmpty()); } /** - * This test validates - * the correctness of GetApplicationsResponse in case applications + * This test validates the correctness of GetApplicationsResponse in case applications * with given YarnApplicationState does not exist. */ @Test public void testGetApplicationsApplicationStateNotExists() throws Exception { - LOG.info("Test FederationClientInterceptor:" + - " Application with state does not exist"); + LOG.info("Test FederationClientInterceptor: Application with state does not exist."); ApplicationId appId = ApplicationId.newInstance(System.currentTimeMillis(), 1); @@ -691,8 +670,7 @@ public void testGetApplicationsApplicationStateNotExists() throws Exception { GetApplicationsRequest requestGet = GetApplicationsRequest.newInstance(applicationStates); - GetApplicationsResponse responseGet = - interceptor.getApplications(requestGet); + GetApplicationsResponse responseGet = interceptor.getApplications(requestGet); Assert.assertNotNull(responseGet); Assert.assertTrue(responseGet.getApplicationList().isEmpty()); @@ -700,7 +678,7 @@ public void testGetApplicationsApplicationStateNotExists() throws Exception { @Test public void testGetClusterNodesRequest() throws Exception { - LOG.info("Test FederationClientInterceptor : Get Cluster Nodeds request"); + LOG.info("Test FederationClientInterceptor : Get Cluster Nodes request."); // null request LambdaTestUtils.intercept(YarnException.class, "Missing getClusterNodes request.", () -> interceptor.getClusterNodes(null)); @@ -712,7 +690,7 @@ public void testGetClusterNodesRequest() throws Exception { @Test public void testGetNodeToLabelsRequest() throws Exception { - LOG.info("Test FederationClientInterceptor : Get Node To Labels request"); + LOG.info("Test FederationClientInterceptor : Get Node To Labels request."); // null request LambdaTestUtils.intercept(YarnException.class, "Missing getNodesToLabels request.", () -> interceptor.getNodeToLabels(null)); @@ -724,7 +702,7 @@ public void testGetNodeToLabelsRequest() throws Exception { @Test public void testGetLabelsToNodesRequest() throws Exception { - LOG.info("Test FederationClientInterceptor : Get Labels To Node request"); + LOG.info("Test FederationClientInterceptor : Get Labels To Node request."); // null request LambdaTestUtils.intercept(YarnException.class, "Missing getLabelsToNodes request.", () -> interceptor.getLabelsToNodes(null)); @@ -736,7 +714,7 @@ public void testGetLabelsToNodesRequest() throws Exception { @Test public void testClusterNodeLabelsRequest() throws Exception { - LOG.info("Test FederationClientInterceptor : Get Cluster NodeLabels request"); + LOG.info("Test FederationClientInterceptor : Get Cluster NodeLabels request."); // null request LambdaTestUtils.intercept(YarnException.class, "Missing getClusterNodeLabels request.", () -> interceptor.getClusterNodeLabels(null)); @@ -748,13 +726,13 @@ public void testClusterNodeLabelsRequest() throws Exception { @Test public void testGetQueueUserAcls() throws Exception { - LOG.info("Test FederationClientInterceptor : Get QueueUserAcls request"); + LOG.info("Test FederationClientInterceptor : Get QueueUserAcls request."); // null request LambdaTestUtils.intercept(YarnException.class, "Missing getQueueUserAcls request.", () -> interceptor.getQueueUserAcls(null)); - // noraml request + // normal request GetQueueUserAclsInfoResponse response = interceptor.getQueueUserAcls( GetQueueUserAclsInfoRequest.newInstance()); @@ -776,7 +754,7 @@ public void testGetQueueUserAcls() throws Exception { @Test public void testListReservations() throws Exception { - LOG.info("Test FederationClientInterceptor : Get ListReservations request"); + LOG.info("Test FederationClientInterceptor : Get ListReservations request."); // null request LambdaTestUtils.intercept(YarnException.class, "Missing listReservations request.", @@ -792,7 +770,7 @@ public void testListReservations() throws Exception { @Test public void testGetContainersRequest() throws Exception { - LOG.info("Test FederationClientInterceptor : Get Containers request"); + LOG.info("Test FederationClientInterceptor : Get Containers request."); // null request LambdaTestUtils.intercept(YarnException.class, "Missing getContainers request " + @@ -908,7 +886,7 @@ public void getApplicationAttempts() throws Exception { @Test public void testGetResourceTypeInfoRequest() throws Exception { - LOG.info("Test FederationClientInterceptor : Get Resource TypeInfo request"); + LOG.info("Test FederationClientInterceptor : Get Resource TypeInfo request."); // null request LambdaTestUtils.intercept(YarnException.class, "Missing getResourceTypeInfo request.", () -> interceptor.getResourceTypeInfo(null)); @@ -1089,7 +1067,7 @@ public void testSignalContainer() throws Exception { RMAppAttemptState.SCHEDULED); MockNM nm = interceptor.getMockNMs().get(subClusterId); nm.nodeHeartbeat(true); - mockRM.waitForState(rmApp.getCurrentAppAttempt(), RMAppAttemptState.ALLOCATED); + MockRM.waitForState(rmApp.getCurrentAppAttempt(), RMAppAttemptState.ALLOCATED); mockRM.sendAMLaunched(rmApp.getCurrentAppAttempt().getAppAttemptId()); ContainerId containerId = rmApp.getCurrentAppAttempt().getMasterContainer().getId(); @@ -1101,4 +1079,435 @@ public void testSignalContainer() throws Exception { Assert.assertNotNull(signalContainerResponse); } + + @Test + public void testMoveApplicationAcrossQueues() throws Exception { + LOG.info("Test FederationClientInterceptor : MoveApplication AcrossQueues request."); + + // null request + LambdaTestUtils.intercept(YarnException.class, "Missing moveApplicationAcrossQueues request " + + "or applicationId or target queue.", () -> interceptor.moveApplicationAcrossQueues(null)); + + // normal request + ApplicationId appId = ApplicationId.newInstance(System.currentTimeMillis(), 1); + SubmitApplicationRequest request = mockSubmitApplicationRequest(appId); + + // Submit the application + SubmitApplicationResponse response = interceptor.submitApplication(request); + + Assert.assertNotNull(response); + Assert.assertNotNull(stateStoreUtil.queryApplicationHomeSC(appId)); + + SubClusterId subClusterId = interceptor.getApplicationHomeSubCluster(appId); + Assert.assertNotNull(subClusterId); + + MockRM mockRM = interceptor.getMockRMs().get(subClusterId); + mockRM.waitForState(appId, RMAppState.ACCEPTED); + RMApp rmApp = mockRM.getRMContext().getRMApps().get(appId); + mockRM.waitForState(rmApp.getCurrentAppAttempt().getAppAttemptId(), + RMAppAttemptState.SCHEDULED); + MockNM nm = interceptor.getMockNMs().get(subClusterId); + nm.nodeHeartbeat(true); + MockRM.waitForState(rmApp.getCurrentAppAttempt(), RMAppAttemptState.ALLOCATED); + mockRM.sendAMLaunched(rmApp.getCurrentAppAttempt().getAppAttemptId()); + + MoveApplicationAcrossQueuesRequest acrossQueuesRequest = + MoveApplicationAcrossQueuesRequest.newInstance(appId, "root.target"); + MoveApplicationAcrossQueuesResponse acrossQueuesResponse = + interceptor.moveApplicationAcrossQueues(acrossQueuesRequest); + + Assert.assertNotNull(acrossQueuesResponse); + } + + + @Test + public void testGetQueueInfo() throws Exception { + LOG.info("Test FederationClientInterceptor : Get Queue Info request."); + + // null request + LambdaTestUtils.intercept(YarnException.class, "Missing getQueueInfo request or queueName.", + () -> interceptor.getQueueInfo(null)); + + // normal request + GetQueueInfoResponse response = interceptor.getQueueInfo( + GetQueueInfoRequest.newInstance("root", true, true, true)); + + Assert.assertNotNull(response); + + QueueInfo queueInfo = response.getQueueInfo(); + Assert.assertNotNull(queueInfo); + Assert.assertEquals(queueInfo.getQueueName(), "root"); + Assert.assertEquals(queueInfo.getCapacity(), 4.0, 0); + Assert.assertEquals(queueInfo.getCurrentCapacity(), 0.0, 0); + Assert.assertEquals(queueInfo.getChildQueues().size(), 12, 0); + Assert.assertEquals(queueInfo.getAccessibleNodeLabels().size(), 1); + } + + @Test + public void testGetResourceProfiles() throws Exception { + LOG.info("Test FederationClientInterceptor : Get Resource Profiles request."); + + // null request + LambdaTestUtils.intercept(YarnException.class, "Missing getResourceProfiles request.", + () -> interceptor.getResourceProfiles(null)); + + // normal request + GetAllResourceProfilesRequest request = GetAllResourceProfilesRequest.newInstance(); + GetAllResourceProfilesResponse response = interceptor.getResourceProfiles(request); + + Assert.assertNotNull(response); + Map resProfiles = response.getResourceProfiles(); + + Resource maxResProfiles = resProfiles.get("maximum"); + Assert.assertEquals(32768, maxResProfiles.getMemorySize()); + Assert.assertEquals(16, maxResProfiles.getVirtualCores()); + + Resource defaultResProfiles = resProfiles.get("default"); + Assert.assertEquals(8192, defaultResProfiles.getMemorySize()); + Assert.assertEquals(8, defaultResProfiles.getVirtualCores()); + + Resource minimumResProfiles = resProfiles.get("minimum"); + Assert.assertEquals(4096, minimumResProfiles.getMemorySize()); + Assert.assertEquals(4, minimumResProfiles.getVirtualCores()); + } + + @Test + public void testGetResourceProfile() throws Exception { + LOG.info("Test FederationClientInterceptor : Get Resource Profile request."); + + // null request + LambdaTestUtils.intercept(YarnException.class, + "Missing getResourceProfile request or profileName.", + () -> interceptor.getResourceProfile(null)); + + // normal request + GetResourceProfileRequest request = GetResourceProfileRequest.newInstance("maximum"); + GetResourceProfileResponse response = interceptor.getResourceProfile(request); + + Assert.assertNotNull(response); + Assert.assertEquals(32768, response.getResource().getMemorySize()); + Assert.assertEquals(16, response.getResource().getVirtualCores()); + + GetResourceProfileRequest request2 = GetResourceProfileRequest.newInstance("default"); + GetResourceProfileResponse response2 = interceptor.getResourceProfile(request2); + + Assert.assertNotNull(response2); + Assert.assertEquals(8192, response2.getResource().getMemorySize()); + Assert.assertEquals(8, response2.getResource().getVirtualCores()); + + GetResourceProfileRequest request3 = GetResourceProfileRequest.newInstance("minimum"); + GetResourceProfileResponse response3 = interceptor.getResourceProfile(request3); + + Assert.assertNotNull(response3); + Assert.assertEquals(4096, response3.getResource().getMemorySize()); + Assert.assertEquals(4, response3.getResource().getVirtualCores()); + } + + @Test + public void testGetAttributesToNodes() throws Exception { + LOG.info("Test FederationClientInterceptor : Get AttributesToNodes request."); + + // null request + LambdaTestUtils.intercept(YarnException.class, "Missing getAttributesToNodes request " + + "or nodeAttributes.", () -> interceptor.getAttributesToNodes(null)); + + // normal request + GetAttributesToNodesResponse response = + interceptor.getAttributesToNodes(GetAttributesToNodesRequest.newInstance()); + + Assert.assertNotNull(response); + Map> attrs = response.getAttributesToNodes(); + Assert.assertNotNull(attrs); + Assert.assertEquals(4, attrs.size()); + + NodeAttribute gpu = NodeAttribute.newInstance(NodeAttribute.PREFIX_CENTRALIZED, "GPU", + NodeAttributeType.STRING, "nvidia"); + NodeToAttributeValue attributeValue1 = + NodeToAttributeValue.newInstance("0-host1", gpu.getAttributeValue()); + NodeAttributeKey gpuKey = gpu.getAttributeKey(); + Assert.assertTrue(attrs.get(gpuKey).contains(attributeValue1)); + } + + @Test + public void testClusterNodeAttributes() throws Exception { + LOG.info("Test FederationClientInterceptor : Get ClusterNodeAttributes request."); + + // null request + LambdaTestUtils.intercept(YarnException.class, "Missing getClusterNodeAttributes request.", + () -> interceptor.getClusterNodeAttributes(null)); + + // normal request + GetClusterNodeAttributesResponse response = + interceptor.getClusterNodeAttributes(GetClusterNodeAttributesRequest.newInstance()); + + Assert.assertNotNull(response); + Set nodeAttributeInfos = response.getNodeAttributes(); + Assert.assertNotNull(nodeAttributeInfos); + Assert.assertEquals(4, nodeAttributeInfos.size()); + + NodeAttributeInfo nodeAttributeInfo1 = + NodeAttributeInfo.newInstance(NodeAttributeKey.newInstance("GPU"), + NodeAttributeType.STRING); + Assert.assertTrue(nodeAttributeInfos.contains(nodeAttributeInfo1)); + + NodeAttributeInfo nodeAttributeInfo2 = + NodeAttributeInfo.newInstance(NodeAttributeKey.newInstance("OS"), + NodeAttributeType.STRING); + Assert.assertTrue(nodeAttributeInfos.contains(nodeAttributeInfo2)); + } + + @Test + public void testNodesToAttributes() throws Exception { + LOG.info("Test FederationClientInterceptor : Get NodesToAttributes request."); + + // null request + LambdaTestUtils.intercept(YarnException.class, + "Missing getNodesToAttributes request or hostNames.", + () -> interceptor.getNodesToAttributes(null)); + + // normal request + Set hostNames = Collections.singleton("0-host1"); + GetNodesToAttributesResponse response = + interceptor.getNodesToAttributes(GetNodesToAttributesRequest.newInstance(hostNames)); + Assert.assertNotNull(response); + + Map> nodeAttributeMap = response.getNodeToAttributes(); + Assert.assertNotNull(nodeAttributeMap); + Assert.assertEquals(1, nodeAttributeMap.size()); + + NodeAttribute gpu = NodeAttribute.newInstance(NodeAttribute.PREFIX_CENTRALIZED, "GPU", + NodeAttributeType.STRING, "nvida"); + Assert.assertTrue(nodeAttributeMap.get("0-host1").contains(gpu)); + } + + @Test + public void testGetNewReservation() throws Exception { + LOG.info("Test FederationClientInterceptor : Get NewReservation request."); + + // null request + LambdaTestUtils.intercept(YarnException.class, + "Missing getNewReservation request.", () -> interceptor.getNewReservation(null)); + + // normal request + GetNewReservationRequest request = GetNewReservationRequest.newInstance(); + GetNewReservationResponse response = interceptor.getNewReservation(request); + Assert.assertNotNull(response); + + ReservationId reservationId = response.getReservationId(); + Assert.assertNotNull(reservationId); + Assert.assertTrue(reservationId.toString().contains("reservation")); + Assert.assertEquals(reservationId.getClusterTimestamp(), ResourceManager.getClusterTimeStamp()); + } + + @Test + public void testSubmitReservation() throws Exception { + LOG.info("Test FederationClientInterceptor : SubmitReservation request."); + + // get new reservationId + GetNewReservationRequest request = GetNewReservationRequest.newInstance(); + GetNewReservationResponse response = interceptor.getNewReservation(request); + Assert.assertNotNull(response); + + // Submit Reservation + ReservationId reservationId = response.getReservationId(); + ReservationDefinition rDefinition = createReservationDefinition(1024, 1); + ReservationSubmissionRequest rSubmissionRequest = ReservationSubmissionRequest.newInstance( + rDefinition, "decided", reservationId); + + ReservationSubmissionResponse submissionResponse = + interceptor.submitReservation(rSubmissionRequest); + Assert.assertNotNull(submissionResponse); + + SubClusterId subClusterId = stateStoreUtil.queryReservationHomeSC(reservationId); + Assert.assertNotNull(subClusterId); + Assert.assertTrue(subClusters.contains(subClusterId)); + } + + @Test + public void testSubmitReservationEmptyRequest() throws Exception { + LOG.info("Test FederationClientInterceptor : SubmitReservation request empty."); + + String errorMsg = + "Missing submitReservation request or reservationId or reservation definition or queue."; + + // null request1 + LambdaTestUtils.intercept(YarnException.class, errorMsg, + () -> interceptor.submitReservation(null)); + + // null request2 + ReservationSubmissionRequest request2 = + ReservationSubmissionRequest.newInstance(null, null, null); + LambdaTestUtils.intercept(YarnException.class, errorMsg, + () -> interceptor.submitReservation(request2)); + + // null request3 + ReservationSubmissionRequest request3 = + ReservationSubmissionRequest.newInstance(null, "q1", null); + LambdaTestUtils.intercept(YarnException.class, errorMsg, + () -> interceptor.submitReservation(request3)); + + // null request4 + ReservationId reservationId = ReservationId.newInstance(Time.now(), 1); + ReservationSubmissionRequest request4 = + ReservationSubmissionRequest.newInstance(null, null, reservationId); + LambdaTestUtils.intercept(YarnException.class, errorMsg, + () -> interceptor.submitReservation(request4)); + + // null request5 + long arrival = Time.now(); + long deadline = arrival + (int)(DEFAULT_DURATION * 1.1); + + ReservationRequest rRequest = ReservationRequest.newInstance( + Resource.newInstance(1024, 1), 1, 1, DEFAULT_DURATION); + ReservationRequest[] rRequests = new ReservationRequest[] {rRequest}; + ReservationDefinition rDefinition = createReservationDefinition(arrival, deadline, rRequests, + ReservationRequestInterpreter.R_ALL, "u1"); + ReservationSubmissionRequest request5 = + ReservationSubmissionRequest.newInstance(rDefinition, null, reservationId); + LambdaTestUtils.intercept(YarnException.class, errorMsg, + () -> interceptor.submitReservation(request5)); + } + + @Test + public void testSubmitReservationMultipleSubmission() throws Exception { + LOG.info("Test FederationClientInterceptor: Submit Reservation - Multiple"); + + // get new reservationId + GetNewReservationRequest request = GetNewReservationRequest.newInstance(); + GetNewReservationResponse response = interceptor.getNewReservation(request); + Assert.assertNotNull(response); + + // First Submit Reservation + ReservationId reservationId = response.getReservationId(); + ReservationDefinition rDefinition = createReservationDefinition(1024, 1); + ReservationSubmissionRequest rSubmissionRequest = ReservationSubmissionRequest.newInstance( + rDefinition, "decided", reservationId); + ReservationSubmissionResponse submissionResponse = + interceptor.submitReservation(rSubmissionRequest); + Assert.assertNotNull(submissionResponse); + + SubClusterId subClusterId1 = stateStoreUtil.queryReservationHomeSC(reservationId); + Assert.assertNotNull(subClusterId1); + Assert.assertTrue(subClusters.contains(subClusterId1)); + + // First Retry, repeat the submission + ReservationSubmissionResponse submissionResponse1 = + interceptor.submitReservation(rSubmissionRequest); + Assert.assertNotNull(submissionResponse1); + + // Expect reserved clusters to be consistent + SubClusterId subClusterId2 = stateStoreUtil.queryReservationHomeSC(reservationId); + Assert.assertNotNull(subClusterId2); + Assert.assertEquals(subClusterId1, subClusterId2); + } + + @Test + public void testUpdateReservation() throws Exception { + LOG.info("Test FederationClientInterceptor : UpdateReservation request."); + + // get new reservationId + GetNewReservationRequest request = GetNewReservationRequest.newInstance(); + GetNewReservationResponse response = interceptor.getNewReservation(request); + Assert.assertNotNull(response); + + // allow plan follower to synchronize, manually trigger an assignment + Map mockRMs = interceptor.getMockRMs(); + for (MockRM mockRM : mockRMs.values()) { + ReservationSystem reservationSystem = mockRM.getReservationSystem(); + reservationSystem.synchronizePlan("root.decided", true); + } + + // Submit Reservation + ReservationId reservationId = response.getReservationId(); + ReservationDefinition rDefinition = createReservationDefinition(1024, 1); + ReservationSubmissionRequest rSubmissionRequest = ReservationSubmissionRequest.newInstance( + rDefinition, "decided", reservationId); + + ReservationSubmissionResponse submissionResponse = + interceptor.submitReservation(rSubmissionRequest); + Assert.assertNotNull(submissionResponse); + + // Update Reservation + ReservationDefinition rDefinition2 = createReservationDefinition(2048, 1); + ReservationUpdateRequest updateRequest = + ReservationUpdateRequest.newInstance(rDefinition2, reservationId); + ReservationUpdateResponse updateResponse = + interceptor.updateReservation(updateRequest); + Assert.assertNotNull(updateResponse); + + SubClusterId subClusterId = stateStoreUtil.queryReservationHomeSC(reservationId); + Assert.assertNotNull(subClusterId); + } + + @Test + public void testDeleteReservation() throws Exception { + LOG.info("Test FederationClientInterceptor : DeleteReservation request."); + + // get new reservationId + GetNewReservationRequest request = GetNewReservationRequest.newInstance(); + GetNewReservationResponse response = interceptor.getNewReservation(request); + Assert.assertNotNull(response); + + // allow plan follower to synchronize, manually trigger an assignment + Map mockRMs = interceptor.getMockRMs(); + for (MockRM mockRM : mockRMs.values()) { + ReservationSystem reservationSystem = mockRM.getReservationSystem(); + reservationSystem.synchronizePlan("root.decided", true); + } + + // Submit Reservation + ReservationId reservationId = response.getReservationId(); + ReservationDefinition rDefinition = createReservationDefinition(1024, 1); + ReservationSubmissionRequest rSubmissionRequest = ReservationSubmissionRequest.newInstance( + rDefinition, "decided", reservationId); + + ReservationSubmissionResponse submissionResponse = + interceptor.submitReservation(rSubmissionRequest); + Assert.assertNotNull(submissionResponse); + + // Delete Reservation + ReservationDeleteRequest deleteRequest = ReservationDeleteRequest.newInstance(reservationId); + ReservationDeleteResponse deleteResponse = interceptor.deleteReservation(deleteRequest); + Assert.assertNotNull(deleteResponse); + + LambdaTestUtils.intercept(YarnException.class, + "Reservation " + reservationId + " does not exist", + () -> stateStoreUtil.queryReservationHomeSC(reservationId)); + } + + + private ReservationDefinition createReservationDefinition(int memory, int core) { + // get reservationId + long arrival = Time.now(); + long deadline = arrival + (int)(DEFAULT_DURATION * 1.1); + + ReservationRequest rRequest = ReservationRequest.newInstance( + Resource.newInstance(memory, core), 1, 1, DEFAULT_DURATION); + ReservationRequest[] rRequests = new ReservationRequest[] {rRequest}; + + ReservationDefinition rDefinition = createReservationDefinition(arrival, deadline, rRequests, + ReservationRequestInterpreter.R_ALL, "u1"); + + return rDefinition; + } + + /** + * This method is used to create a ReservationDefinition. + * + * @param arrival Job arrival time + * @param deadline Job deadline + * @param reservationRequests reservationRequest Array + * @param rType Enumeration of various types of + * dependencies among multiple ReservationRequest + * @param username username + * @return ReservationDefinition + */ + private ReservationDefinition createReservationDefinition(long arrival, + long deadline, ReservationRequest[] reservationRequests, + ReservationRequestInterpreter rType, String username) { + ReservationRequests requests = ReservationRequests + .newInstance(Arrays.asList(reservationRequests), rType); + return ReservationDefinition.newInstance(arrival, deadline, + requests, username, "0", Priority.UNDEFINED); + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/clientrm/TestFederationClientInterceptorRetry.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/clientrm/TestFederationClientInterceptorRetry.java index 37278861afaa6..096fa0639079a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/clientrm/TestFederationClientInterceptorRetry.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/clientrm/TestFederationClientInterceptorRetry.java @@ -54,7 +54,7 @@ * use the {@code RouterClientRMService} pipeline test cases for testing the * {@code FederationInterceptor} class. The tests for * {@code RouterClientRMService} has been written cleverly so that it can be - * reused to validate different request intercepter chains. + * reused to validate different request interceptor chains. * * It tests the case with SubClusters down and the Router logic of retries. We * have 1 good SubCluster and 2 bad ones for all the tests. @@ -137,8 +137,8 @@ protected YarnConfiguration createConfiguration() { String mockPassThroughInterceptorClass = PassThroughClientRequestInterceptor.class.getName(); - // Create a request intercepter pipeline for testing. The last one in the - // chain is the federation intercepter that calls the mock resource manager. + // Create a request interceptor pipeline for testing. The last one in the + // chain is the federation interceptor that calls the mock resource manager. // The others in the chain will simply forward it to the next one in the // chain conf.set(YarnConfiguration.ROUTER_CLIENTRM_INTERCEPTOR_CLASS_PIPELINE, diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/clientrm/TestRouterYarnClientUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/clientrm/TestRouterYarnClientUtils.java index 435a8bd85176b..33cae61230057 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/clientrm/TestRouterYarnClientUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/clientrm/TestRouterYarnClientUtils.java @@ -35,6 +35,11 @@ import org.apache.hadoop.yarn.api.protocolrecords.GetQueueUserAclsInfoResponse; import org.apache.hadoop.yarn.api.protocolrecords.ReservationListResponse; import org.apache.hadoop.yarn.api.protocolrecords.GetAllResourceTypeInfoResponse; +import org.apache.hadoop.yarn.api.protocolrecords.GetResourceProfileResponse; +import org.apache.hadoop.yarn.api.protocolrecords.GetAllResourceProfilesResponse; +import org.apache.hadoop.yarn.api.protocolrecords.GetAttributesToNodesResponse; +import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodeAttributesResponse; +import org.apache.hadoop.yarn.api.protocolrecords.GetNodesToAttributesResponse; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ApplicationReport; @@ -51,6 +56,11 @@ import org.apache.hadoop.yarn.api.records.ReservationDefinition; import org.apache.hadoop.yarn.api.records.ReservationId; import org.apache.hadoop.yarn.api.records.ResourceTypeInfo; +import org.apache.hadoop.yarn.api.records.NodeAttribute; +import org.apache.hadoop.yarn.api.records.NodeAttributeType; +import org.apache.hadoop.yarn.api.records.NodeAttributeKey; +import org.apache.hadoop.yarn.api.records.NodeToAttributeValue; +import org.apache.hadoop.yarn.api.records.NodeAttributeInfo; import org.apache.hadoop.yarn.util.Records; import org.apache.hadoop.yarn.server.uam.UnmanagedApplicationManager; import org.junit.Assert; @@ -422,7 +432,7 @@ public void testMergeQueueUserAclsResponse() { GetQueueUserAclsInfoResponse response3 = Records.newRecord( GetQueueUserAclsInfoResponse.class); - // null responce + // null response GetQueueUserAclsInfoResponse response4 = null; List responses = new ArrayList<>(); @@ -539,4 +549,216 @@ public void testMergeResourceTypes() { Assert.assertTrue(CollectionUtils.isEqualCollection(expectedResponse, response.getResourceTypeInfo())); } + + @Test + public void testMergeResourceProfiles() { + // normal response1 + Map profiles = new HashMap<>(); + Resource resource1 = Resource.newInstance(1024, 1); + GetAllResourceProfilesResponse response1 = GetAllResourceProfilesResponse.newInstance(); + profiles.put("maximum", resource1); + response1.setResourceProfiles(profiles); + + // normal response2 + profiles = new HashMap<>(); + Resource resource2 = Resource.newInstance(2048, 2); + GetAllResourceProfilesResponse response2 = GetAllResourceProfilesResponse.newInstance(); + profiles.put("maximum", resource2); + response2.setResourceProfiles(profiles); + + // empty response + GetAllResourceProfilesResponse response3 = GetAllResourceProfilesResponse.newInstance(); + + // null response + GetAllResourceProfilesResponse response4 = null; + + List responses = new ArrayList<>(); + responses.add(response1); + responses.add(response2); + responses.add(response3); + responses.add(response4); + + GetAllResourceProfilesResponse response = + RouterYarnClientUtils.mergeClusterResourceProfilesResponse(responses); + Resource resource = response.getResourceProfiles().get("maximum"); + Assert.assertEquals(3, resource.getVirtualCores()); + Assert.assertEquals(3072, resource.getMemorySize()); + } + + @Test + public void testMergeResourceProfile() { + // normal response1 + Resource resource1 = Resource.newInstance(1024, 1); + GetResourceProfileResponse response1 = + Records.newRecord(GetResourceProfileResponse.class); + response1.setResource(resource1); + + // normal response2 + Resource resource2 = Resource.newInstance(2048, 2); + GetResourceProfileResponse response2 = + Records.newRecord(GetResourceProfileResponse.class); + response2.setResource(resource2); + + // empty response + GetResourceProfileResponse response3 = + Records.newRecord(GetResourceProfileResponse.class); + + // null response + GetResourceProfileResponse response4 = null; + + List responses = new ArrayList<>(); + responses.add(response1); + responses.add(response2); + responses.add(response3); + responses.add(response4); + + GetResourceProfileResponse response = + RouterYarnClientUtils.mergeClusterResourceProfileResponse(responses); + Resource resource = response.getResource(); + Assert.assertEquals(3, resource.getVirtualCores()); + Assert.assertEquals(3072, resource.getMemorySize()); + } + + @Test + public void testMergeAttributesToNodesResponse() { + // normal response1 + NodeAttribute gpu = NodeAttribute.newInstance(NodeAttribute.PREFIX_CENTRALIZED, "GPU", + NodeAttributeType.STRING, "nvidia"); + Map> map1 = new HashMap<>(); + List lists1 = new ArrayList<>(); + NodeToAttributeValue attributeValue1 = + NodeToAttributeValue.newInstance("node1", gpu.getAttributeValue()); + lists1.add(attributeValue1); + map1.put(gpu.getAttributeKey(), lists1); + GetAttributesToNodesResponse response1 = GetAttributesToNodesResponse.newInstance(map1); + + // normal response2 + NodeAttribute docker = NodeAttribute.newInstance(NodeAttribute.PREFIX_DISTRIBUTED, "DOCKER", + NodeAttributeType.STRING, "docker0"); + Map> map2 = new HashMap<>(); + List lists2 = new ArrayList<>(); + NodeToAttributeValue attributeValue2 = + NodeToAttributeValue.newInstance("node2", docker.getAttributeValue()); + lists2.add(attributeValue2); + map2.put(docker.getAttributeKey(), lists2); + GetAttributesToNodesResponse response2 = GetAttributesToNodesResponse.newInstance(map2); + + // empty response3 + GetAttributesToNodesResponse response3 = + GetAttributesToNodesResponse.newInstance(new HashMap<>()); + + // null response4 + GetAttributesToNodesResponse response4 = null; + + List responses = new ArrayList<>(); + responses.add(response1); + responses.add(response2); + responses.add(response3); + responses.add(response4); + + GetAttributesToNodesResponse response = + RouterYarnClientUtils.mergeAttributesToNodesResponse(responses); + + Assert.assertNotNull(response); + Assert.assertEquals(2, response.getAttributesToNodes().size()); + + Map> attrs = response.getAttributesToNodes(); + + NodeAttributeKey gpuKey = gpu.getAttributeKey(); + Assert.assertEquals(attributeValue1.toString(), attrs.get(gpuKey).get(0).toString()); + + NodeAttributeKey dockerKey = docker.getAttributeKey(); + Assert.assertEquals(attributeValue2.toString(), attrs.get(dockerKey).get(0).toString()); + } + + @Test + public void testMergeClusterNodeAttributesResponse() { + // normal response1 + NodeAttributeInfo nodeAttributeInfo1 = + NodeAttributeInfo.newInstance(NodeAttributeKey.newInstance("GPU"), + NodeAttributeType.STRING); + Set attributes1 = new HashSet<>(); + attributes1.add(nodeAttributeInfo1); + GetClusterNodeAttributesResponse response1 = + GetClusterNodeAttributesResponse.newInstance(attributes1); + + // normal response2 + NodeAttributeInfo nodeAttributeInfo2 = + NodeAttributeInfo.newInstance(NodeAttributeKey.newInstance("CPU"), + NodeAttributeType.STRING); + Set attributes2 = new HashSet<>(); + attributes2.add(nodeAttributeInfo2); + GetClusterNodeAttributesResponse response2 = + GetClusterNodeAttributesResponse.newInstance(attributes2); + + // empty response3 + GetClusterNodeAttributesResponse response3 = + GetClusterNodeAttributesResponse.newInstance(new HashSet<>()); + + // null response4 + GetClusterNodeAttributesResponse response4 = null; + + List responses = new ArrayList<>(); + responses.add(response1); + responses.add(response2); + responses.add(response3); + responses.add(response4); + + GetClusterNodeAttributesResponse response = + RouterYarnClientUtils.mergeClusterNodeAttributesResponse(responses); + + Assert.assertNotNull(response); + + Set nodeAttributeInfos = response.getNodeAttributes(); + Assert.assertEquals(2, nodeAttributeInfos.size()); + Assert.assertTrue(nodeAttributeInfos.contains(nodeAttributeInfo1)); + Assert.assertTrue(nodeAttributeInfos.contains(nodeAttributeInfo2)); + } + + @Test + public void testMergeNodesToAttributesResponse() { + // normal response1 + NodeAttribute gpu = NodeAttribute.newInstance(NodeAttribute.PREFIX_CENTRALIZED, "GPU", + NodeAttributeType.STRING, "nvida"); + NodeAttribute os = NodeAttribute.newInstance(NodeAttribute.PREFIX_CENTRALIZED, "OS", + NodeAttributeType.STRING, "windows64"); + NodeAttribute dist = NodeAttribute.newInstance(NodeAttribute.PREFIX_DISTRIBUTED, "VERSION", + NodeAttributeType.STRING, "3_0_2"); + Map> node1Map = new HashMap<>(); + node1Map.put("node1", ImmutableSet.of(gpu, os, dist)); + GetNodesToAttributesResponse response1 = GetNodesToAttributesResponse.newInstance(node1Map); + + // normal response2 + NodeAttribute docker = NodeAttribute.newInstance(NodeAttribute.PREFIX_DISTRIBUTED, "DOCKER", + NodeAttributeType.STRING, "docker0"); + Map> node2Map = new HashMap<>(); + node2Map.put("node2", ImmutableSet.of(docker)); + GetNodesToAttributesResponse response2 = GetNodesToAttributesResponse.newInstance(node2Map); + + // empty response3 + GetNodesToAttributesResponse response3 = + GetNodesToAttributesResponse.newInstance(new HashMap<>()); + + // null response4 + GetNodesToAttributesResponse response4 = null; + + List responses = new ArrayList<>(); + responses.add(response1); + responses.add(response2); + responses.add(response3); + responses.add(response4); + + GetNodesToAttributesResponse response = + RouterYarnClientUtils.mergeNodesToAttributesResponse(responses); + + Assert.assertNotNull(response); + + Map> hostToAttrs = response.getNodeToAttributes(); + Assert.assertNotNull(hostToAttrs); + Assert.assertEquals(2, hostToAttrs.size()); + Assert.assertTrue(hostToAttrs.get("node1").contains(dist)); + Assert.assertTrue(hostToAttrs.get("node1").contains(gpu)); + Assert.assertTrue(hostToAttrs.get("node1").contains(os)); + Assert.assertTrue(hostToAttrs.get("node2").contains(docker)); + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/clientrm/TestableFederationClientInterceptor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/clientrm/TestableFederationClientInterceptor.java index af1f45924c19c..8279899e387ad 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/clientrm/TestableFederationClientInterceptor.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/clientrm/TestableFederationClientInterceptor.java @@ -24,23 +24,37 @@ import java.net.ConnectException; import java.util.ArrayList; import java.util.List; +import java.util.Set; +import java.util.Map; +import java.util.HashMap; import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.TimeoutException; +import org.apache.hadoop.test.GenericTestUtils; +import org.apache.hadoop.thirdparty.com.google.common.collect.ImmutableSet; import org.apache.hadoop.yarn.api.ApplicationClientProtocol; import org.apache.hadoop.yarn.api.protocolrecords.SubmitApplicationRequest; import org.apache.hadoop.yarn.api.protocolrecords.SubmitApplicationResponse; +import org.apache.hadoop.yarn.api.records.NodeAttribute; +import org.apache.hadoop.yarn.api.records.NodeAttributeType; +import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.exceptions.YarnException; +import org.apache.hadoop.yarn.nodelabels.NodeAttributesManager; import org.apache.hadoop.yarn.server.federation.store.records.SubClusterId; import org.apache.hadoop.yarn.server.resourcemanager.ClientRMService; import org.apache.hadoop.yarn.server.resourcemanager.MockRM; import org.apache.hadoop.yarn.server.resourcemanager.MockNM; import org.apache.hadoop.yarn.server.resourcemanager.RMAppManager; import org.apache.hadoop.yarn.server.resourcemanager.RMContext; +import org.apache.hadoop.yarn.server.resourcemanager.reservation.Plan; +import org.apache.hadoop.yarn.server.resourcemanager.reservation.ReservationSystem; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.YarnScheduler; import org.apache.hadoop.yarn.server.resourcemanager.security.QueueACLsManager; import org.apache.hadoop.yarn.server.resourcemanager.security.RMDelegationTokenSecretManager; import org.apache.hadoop.yarn.server.security.ApplicationACLsManager; import org.junit.Assert; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** * Extends the FederationClientInterceptor and overrides methods to provide a @@ -49,6 +63,9 @@ public class TestableFederationClientInterceptor extends FederationClientInterceptor { + private static final Logger LOG = + LoggerFactory.getLogger(TestableFederationClientInterceptor.class); + private ConcurrentHashMap mockRMs = new ConcurrentHashMap<>(); @@ -82,6 +99,8 @@ protected ApplicationClientProtocol getClientRMProxyForSubCluster( } mockRMs.put(subClusterId, mockRM); } + initNodeAttributes(subClusterId, mockRM); + initReservationSystem(mockRM); return mockRM.getClientRMService(); } } @@ -127,4 +146,74 @@ public ConcurrentHashMap getMockRMs() { public ConcurrentHashMap getMockNMs() { return mockNMs; } -} + + private void initNodeAttributes(SubClusterId subClusterId, MockRM mockRM) { + String node1 = subClusterId.getId() +"-host1"; + String node2 = subClusterId.getId() +"-host2"; + NodeAttributesManager mgr = mockRM.getRMContext().getNodeAttributesManager(); + NodeAttribute gpu = + NodeAttribute.newInstance(NodeAttribute.PREFIX_CENTRALIZED, "GPU", + NodeAttributeType.STRING, "nvidia"); + NodeAttribute os = + NodeAttribute.newInstance(NodeAttribute.PREFIX_CENTRALIZED, "OS", + NodeAttributeType.STRING, "windows64"); + NodeAttribute docker = + NodeAttribute.newInstance(NodeAttribute.PREFIX_DISTRIBUTED, "DOCKER", + NodeAttributeType.STRING, "docker0"); + NodeAttribute dist = + NodeAttribute.newInstance(NodeAttribute.PREFIX_DISTRIBUTED, "VERSION", + NodeAttributeType.STRING, "3_0_2"); + Map> nodes = new HashMap<>(); + nodes.put(node1, ImmutableSet.of(gpu, os, dist)); + nodes.put(node2, ImmutableSet.of(docker, dist)); + try { + mgr.addNodeAttributes(nodes); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + private void initReservationSystem(MockRM mockRM) throws YarnException { + try { + // Ensure that the reserved resources of the RM#Reservation System are allocated + String planName = "root.decided"; + ReservationSystem reservationSystem = mockRM.getReservationSystem(); + reservationSystem.synchronizePlan(planName, true); + + GenericTestUtils.waitFor(() -> { + Plan plan = reservationSystem.getPlan(planName); + Resource resource = plan.getTotalCapacity(); + return (resource.getMemorySize() > 0 && resource.getVirtualCores() > 0); + }, 100, 2000); + } catch (TimeoutException | InterruptedException e) { + throw new YarnException(e); + } + } + + @Override + public void shutdown() { + if (mockRMs != null && !mockRMs.isEmpty()) { + for (Map.Entry item : mockRMs.entrySet()) { + SubClusterId subClusterId = item.getKey(); + + // close mockNM + MockNM mockNM = mockNMs.getOrDefault(subClusterId, null); + try { + mockNM.unRegisterNode(); + mockNM = null; + } catch (Exception e) { + LOG.error("mockNM unRegisterNode error.", e); + } + + // close mockRM + MockRM mockRM = item.getValue(); + if (mockRM != null) { + mockRM.stop(); + } + } + } + mockNMs.clear(); + mockRMs.clear(); + super.shutdown(); + } +} \ No newline at end of file diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/rmadmin/BaseRouterRMAdminTest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/rmadmin/BaseRouterRMAdminTest.java index d3eba6180282f..a8d730fbe87ff 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/rmadmin/BaseRouterRMAdminTest.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/rmadmin/BaseRouterRMAdminTest.java @@ -26,6 +26,7 @@ import java.util.concurrent.Executors; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.yarn.api.records.NodeId; import org.apache.hadoop.yarn.conf.YarnConfiguration; @@ -91,7 +92,7 @@ public void setUp() { String mockPassThroughInterceptorClass = PassThroughRMAdminRequestInterceptor.class.getName(); - // Create a request intercepter pipeline for testing. The last one in the + // Create a request interceptor pipeline for testing. The last one in the // chain will call the mock resource manager. The others in the chain will // simply forward it to the next one in the chain this.conf.set(YarnConfiguration.ROUTER_RMADMIN_INTERCEPTOR_CLASS_PIPELINE, @@ -106,6 +107,8 @@ public void setUp() { this.dispatcher.init(conf); this.dispatcher.start(); this.rmAdminService = createAndStartRouterRMAdminService(); + + DefaultMetricsSystem.setMiniClusterMode(true); } @After diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/rmadmin/MockRMAdminRequestInterceptor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/rmadmin/MockRMAdminRequestInterceptor.java index ee55b63ac40f9..a1a366f7664f7 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/rmadmin/MockRMAdminRequestInterceptor.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/rmadmin/MockRMAdminRequestInterceptor.java @@ -27,13 +27,14 @@ import org.apache.hadoop.yarn.server.resourcemanager.MockRM; /** - * This class mocks the RMAmdinRequestInterceptor. + * This class mocks the RMAdminRequestInterceptor. */ public class MockRMAdminRequestInterceptor extends DefaultRMAdminRequestInterceptor { + MockRM mockRM = null; public void init(String user) { - MockRM mockRM = new MockRM(super.getConf()) { + mockRM = new MockRM(super.getConf()) { @Override protected AdminService createAdminService() { return new AdminService(this) { @@ -60,4 +61,12 @@ protected void stopServer() { mockRM.start(); super.setRMAdmin(mockRM.getAdminService()); } + + @Override + public void shutdown() { + if (mockRM != null) { + mockRM.stop(); + } + super.shutdown(); + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/rmadmin/PassThroughRMAdminRequestInterceptor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/rmadmin/PassThroughRMAdminRequestInterceptor.java index 4d6a3eaf52b4a..e3be25009fc18 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/rmadmin/PassThroughRMAdminRequestInterceptor.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/rmadmin/PassThroughRMAdminRequestInterceptor.java @@ -52,8 +52,8 @@ import org.apache.hadoop.yarn.server.api.protocolrecords.UpdateNodeResourceResponse; /** - * Mock intercepter that does not do anything other than forwarding it to the - * next intercepter in the chain. + * Mock interceptor that does not do anything other than forwarding it to the + * next interceptor in the chain. */ public class PassThroughRMAdminRequestInterceptor extends AbstractRMAdminRequestInterceptor { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/rmadmin/TestRouterRMAdminService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/rmadmin/TestRouterRMAdminService.java index 07ef73c3cdb85..867c71fa82e54 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/rmadmin/TestRouterRMAdminService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/rmadmin/TestRouterRMAdminService.java @@ -23,7 +23,6 @@ import java.util.Map; import org.apache.hadoop.security.UserGroupInformation; -import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.server.api.protocolrecords.AddToClusterNodeLabelsResponse; import org.apache.hadoop.yarn.server.api.protocolrecords.CheckForDecommissioningNodesResponse; import org.apache.hadoop.yarn.server.api.protocolrecords.RefreshAdminAclsResponse; @@ -185,7 +184,7 @@ public void testRouterRMAdminServiceE2E() throws Exception { */ @Test public void testUsersChainMapWithLRUCache() - throws YarnException, IOException, InterruptedException { + throws IOException, InterruptedException { Map pipelines; RequestInterceptorChainWrapper chain; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/secure/AbstractSecureRouterTest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/secure/AbstractSecureRouterTest.java new file mode 100644 index 0000000000000..f9d1d047642fc --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/secure/AbstractSecureRouterTest.java @@ -0,0 +1,241 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.yarn.server.router.secure; + +import org.apache.commons.lang3.StringUtils; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.CommonConfigurationKeysPublic; +import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem; +import org.apache.hadoop.minikdc.MiniKdc; +import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.server.federation.store.records.SubClusterId; +import org.apache.hadoop.yarn.server.resourcemanager.MockRM; +import org.apache.hadoop.yarn.server.resourcemanager.TestRMRestart; +import org.apache.hadoop.yarn.server.router.Router; +import org.apache.hadoop.yarn.server.router.clientrm.FederationClientInterceptor; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.File; +import java.util.Map; +import java.util.Properties; +import java.util.concurrent.ConcurrentHashMap; + +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.assertNotNull; + +public abstract class AbstractSecureRouterTest { + + private static final Logger LOG = LoggerFactory.getLogger(AbstractSecureRouterTest.class); + + //////////////////////////////// + // Kerberos Constants + //////////////////////////////// + + public static final String REALM = "EXAMPLE.COM"; + public static final String ROUTER = "router"; + public static final String LOCALHOST = "localhost"; + public static final String IP127001 = "127.0.0.1"; + public static final String ROUTER_LOCALHOST = "router/" + LOCALHOST; + public static final String ROUTER_127001 = "router/" + IP127001; + public static final String ROUTER_REALM = "router@" + REALM; + public static final String ROUTER_LOCALHOST_REALM = ROUTER_LOCALHOST + "@" + REALM; + public static final String SUN_SECURITY_KRB5_DEBUG = "sun.security.krb5.debug"; + public static final String KERBEROS = "kerberos"; + + //////////////////////////////// + // BeforeSecureRouterTestClass Init + //////////////////////////////// + + private static MiniKdc kdc; + private static File routerKeytab; + private static File kdcWorkDir; + private static Configuration conf; + + //////////////////////////////// + // Specific Constant + // Like Mem, VCore, ClusterNum + //////////////////////////////// + private static final int NUM_SUBCLUSTER = 4; + private static final int GB = 1024; + private static final int NM_MEMORY = 8 * GB; + private static final int NM_VCORE = 4; + + //////////////////////////////// + // Test use in subclasses + //////////////////////////////// + + private Router router = null; + + private static ConcurrentHashMap mockRMs = + new ConcurrentHashMap<>(); + + @BeforeClass + public static void beforeSecureRouterTestClass() throws Exception { + // Sets up the KDC and Principals. + setupKDCAndPrincipals(); + + // Init YarnConfiguration + conf = new YarnConfiguration(); + + // Enable Kerberos authentication configuration + conf.setBoolean(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHORIZATION, true); + conf.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION, KERBEROS); + + // Router configuration + conf.set(YarnConfiguration.ROUTER_BIND_HOST, "0.0.0.0"); + conf.set(YarnConfiguration.ROUTER_CLIENTRM_INTERCEPTOR_CLASS_PIPELINE, + FederationClientInterceptor.class.getName()); + + // Router Kerberos KeyTab configuration + conf.set(YarnConfiguration.ROUTER_PRINCIPAL, ROUTER_LOCALHOST_REALM); + conf.set(YarnConfiguration.ROUTER_KEYTAB, routerKeytab.getAbsolutePath()); + + DefaultMetricsSystem.setMiniClusterMode(true); + } + + /** + * Sets up the KDC and Principals. + * + * @throws Exception an error occurred. + */ + public static void setupKDCAndPrincipals() throws Exception { + // set up the KDC + File target = new File(System.getProperty("test.dir", "target")); + kdcWorkDir = new File(target, "kdc"); + kdcWorkDir.mkdirs(); + if (!kdcWorkDir.mkdirs()) { + assertTrue(kdcWorkDir.isDirectory()); + } + Properties kdcConf = MiniKdc.createConf(); + kdcConf.setProperty(MiniKdc.DEBUG, "true"); + kdc = new MiniKdc(kdcConf, kdcWorkDir); + kdc.start(); + routerKeytab = createKeytab(ROUTER, "router.keytab"); + } + + /** + * Initialize RM in safe mode. + * + * @throws Exception an error occurred. + */ + public static void setupSecureMockRM() throws Exception { + for (int i = 0; i < NUM_SUBCLUSTER; i++) { + SubClusterId sc = SubClusterId.newInstance(i); + if (mockRMs.containsKey(sc)) { + continue; + } + MockRM mockRM = new TestRMRestart.TestSecurityMockRM(conf); + mockRM.start(); + mockRM.registerNode("127.0.0.1:1234", NM_MEMORY, NM_VCORE); + mockRMs.put(sc, mockRM); + } + } + + /** + * Create the keytab for the given principal, includes + * raw principal and $principal/localhost. + * + * @param principal principal short name. + * @param filename filename of keytab. + * @return file of keytab. + * @throws Exception an error occurred. + */ + public static File createKeytab(String principal, String filename) throws Exception { + assertTrue("empty principal", StringUtils.isNotBlank(principal)); + assertTrue("empty host", StringUtils.isNotBlank(filename)); + assertNotNull("null KDC", kdc); + File keytab = new File(kdcWorkDir, filename); + kdc.createPrincipal(keytab, + principal, + principal + "/localhost", + principal + "/127.0.0.1"); + return keytab; + } + + /** + * Start the router in safe mode. + * + * @throws Exception an error occurred. + */ + public synchronized void startSecureRouter() { + assertNull("Router is already running", router); + UserGroupInformation.setConfiguration(conf); + router = new Router(); + router.init(conf); + router.start(); + } + + /** + * Shut down the KDC service. + * + * @throws Exception an error occurred. + */ + public static void teardownKDC() throws Exception { + if (kdc != null) { + kdc.stop(); + kdc = null; + } + } + + /** + * Stop the router in safe mode. + * + * @throws Exception an error occurred. + */ + protected synchronized void stopSecureRouter() throws Exception { + if (router != null) { + router.stop(); + router = null; + } + } + + /** + * Stop the entire test service. + * + * @throws Exception an error occurred. + */ + @AfterClass + public static void afterSecureRouterTest() throws Exception { + LOG.info("teardown of kdc instance."); + teardownKDC(); + if (mockRMs != null && mockRMs.isEmpty()) { + for (MockRM mockRM : mockRMs.values()) { + mockRM.close(); + } + } + mockRMs.clear(); + } + + public static MiniKdc getKdc() { + return kdc; + } + + public Router getRouter() { + return router; + } + + public static ConcurrentHashMap getMockRMs() { + return mockRMs; + } + +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/secure/TestSecureLogins.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/secure/TestSecureLogins.java new file mode 100644 index 0000000000000..40911814c044c --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/secure/TestSecureLogins.java @@ -0,0 +1,156 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.yarn.server.router.secure; + +import org.apache.commons.collections.MapUtils; +import org.apache.hadoop.service.Service; +import org.apache.hadoop.yarn.api.ApplicationClientProtocol; +import org.apache.hadoop.yarn.api.protocolrecords.GetClusterMetricsRequest; +import org.apache.hadoop.yarn.api.protocolrecords.GetClusterMetricsResponse; +import org.apache.hadoop.yarn.api.records.YarnClusterMetrics; +import org.apache.hadoop.yarn.server.api.protocolrecords.RefreshNodesRequest; +import org.apache.hadoop.yarn.server.api.protocolrecords.RefreshNodesResponse; +import org.apache.hadoop.yarn.server.federation.store.FederationStateStore; +import org.apache.hadoop.yarn.server.federation.store.records.SubClusterId; +import org.apache.hadoop.yarn.server.federation.utils.FederationStateStoreFacade; +import org.apache.hadoop.yarn.server.federation.utils.FederationStateStoreTestUtil; +import org.apache.hadoop.yarn.server.resourcemanager.MockRM; +import org.apache.hadoop.yarn.server.router.Router; +import org.apache.hadoop.yarn.server.router.clientrm.FederationClientInterceptor; +import org.apache.hadoop.yarn.server.router.clientrm.RouterClientRMService; +import org.apache.hadoop.yarn.server.router.rmadmin.DefaultRMAdminRequestInterceptor; +import org.apache.hadoop.yarn.server.router.rmadmin.RouterRMAdminService; +import org.junit.Assert; +import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.List; +import java.util.Map; + +public class TestSecureLogins extends AbstractSecureRouterTest { + + private static final Logger LOG = LoggerFactory.getLogger(TestSecureLogins.class); + + @Test + public void testHasRealm() throws Throwable { + Assert.assertNotNull(getRealm()); + LOG.info("Router principal = {}", getPrincipalAndRealm(ROUTER_LOCALHOST)); + } + + @Test + public void testRouterSecureLogin() throws Exception { + startSecureRouter(); + + List services = this.getRouter().getServices(); + Assert.assertNotNull(services); + Assert.assertEquals(3, services.size()); + + stopSecureRouter(); + } + + @Test + public void testRouterClientRMService() throws Exception { + // Start the Router in Secure Mode + startSecureRouter(); + + // Start RM and RouterClientRMService in Secure mode + setupSecureMockRM(); + initRouterClientRMService(); + + // Test the simple rpc call of the Router in the Secure environment + RouterClientRMService routerClientRMService = this.getRouter().getClientRMProxyService(); + GetClusterMetricsRequest metricsRequest = GetClusterMetricsRequest.newInstance(); + GetClusterMetricsResponse metricsResponse = + routerClientRMService.getClusterMetrics(metricsRequest); + Assert.assertNotNull(metricsResponse); + YarnClusterMetrics clusterMetrics = metricsResponse.getClusterMetrics(); + Assert.assertEquals(4, clusterMetrics.getNumNodeManagers()); + Assert.assertEquals(0, clusterMetrics.getNumLostNodeManagers()); + + // Stop the Router in Secure Mode + stopSecureRouter(); + } + + @Test + public void testRouterRMAdminService() throws Exception { + // Start the Router in Secure Mode + startSecureRouter(); + + // Start RM and RouterClientRMService in Secure mode + setupSecureMockRM(); + initRouterRMAdminService(); + + // Test the simple rpc call of the Router in the Secure environment + RouterRMAdminService routerRMAdminService = this.getRouter().getRmAdminProxyService(); + RefreshNodesRequest refreshNodesRequest = RefreshNodesRequest.newInstance(); + RefreshNodesResponse refreshNodesResponse = + routerRMAdminService.refreshNodes(refreshNodesRequest); + Assert.assertNotNull(refreshNodesResponse); + + // Stop the Router in Secure Mode + stopSecureRouter(); + } + + public static String getPrincipalAndRealm(String principal) { + return principal + "@" + getRealm(); + } + + protected static String getRealm() { + return getKdc().getRealm(); + } + + private void initRouterClientRMService() throws Exception { + Router router = this.getRouter(); + Map mockRMs = getMockRMs(); + + RouterClientRMService rmService = router.getClientRMProxyService(); + RouterClientRMService.RequestInterceptorChainWrapper wrapper = rmService.getInterceptorChain(); + FederationClientInterceptor interceptor = + (FederationClientInterceptor) wrapper.getRootInterceptor(); + FederationStateStoreFacade stateStoreFacade = interceptor.getFederationFacade(); + FederationStateStore stateStore = stateStoreFacade.getStateStore(); + FederationStateStoreTestUtil stateStoreUtil = new FederationStateStoreTestUtil(stateStore); + Map clientRMProxies = interceptor.getClientRMProxies(); + + if (MapUtils.isNotEmpty(mockRMs)) { + for (Map.Entry entry : mockRMs.entrySet()) { + SubClusterId sc = entry.getKey(); + MockRM mockRM = entry.getValue(); + stateStoreUtil.registerSubCluster(sc); + if (clientRMProxies.containsKey(sc)) { + continue; + } + clientRMProxies.put(sc, mockRM.getClientRMService()); + } + } + } + + private void initRouterRMAdminService() throws Exception { + Router router = this.getRouter(); + Map mockRMs = getMockRMs(); + SubClusterId sc = SubClusterId.newInstance(0); + MockRM mockRM = mockRMs.get(sc); + RouterRMAdminService routerRMAdminService = router.getRmAdminProxyService(); + RouterRMAdminService.RequestInterceptorChainWrapper rmAdminChainWrapper = + routerRMAdminService.getInterceptorChain(); + DefaultRMAdminRequestInterceptor rmInterceptor = + (DefaultRMAdminRequestInterceptor) rmAdminChainWrapper.getRootInterceptor(); + rmInterceptor.setRMAdmin(mockRM.getAdminService()); + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/webapp/BaseRouterWebServicesTest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/webapp/BaseRouterWebServicesTest.java index e24ad47bf7c73..a4294bc36109a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/webapp/BaseRouterWebServicesTest.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/webapp/BaseRouterWebServicesTest.java @@ -94,7 +94,7 @@ protected YarnConfiguration createConfiguration() { String mockPassThroughInterceptorClass = PassThroughRESTRequestInterceptor.class.getName(); - // Create a request intercepter pipeline for testing. The last one in the + // Create a request interceptor pipeline for testing. The last one in the // chain will call the mock resource manager. The others in the chain will // simply forward it to the next one in the chain config.set(YarnConfiguration.ROUTER_WEBAPP_INTERCEPTOR_CLASS_PIPELINE, @@ -395,4 +395,16 @@ private HttpServletRequest createHttpServletRequest(String user) { when(request.getRemoteUser()).thenReturn(user); return request; } + + protected Response updateSchedulerConfiguration(String user) + throws IOException, InterruptedException { + return routerWebService.updateSchedulerConfiguration(null, + createHttpServletRequest(user)); + } + + protected Response getSchedulerConfiguration(String user) + throws IOException, InterruptedException { + return routerWebService. + getSchedulerConfiguration(createHttpServletRequest(user)); + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/webapp/JavaProcess.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/webapp/JavaProcess.java index e2611d91ba7a7..180cb5536b6f9 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/webapp/JavaProcess.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/webapp/JavaProcess.java @@ -34,15 +34,15 @@ public JavaProcess(Class clazz, File output) this(clazz, null, output); } - public JavaProcess(Class clazz, List addClasspaths, File output) + public JavaProcess(Class clazz, List addClassPaths, File output) throws IOException, InterruptedException { String javaHome = System.getProperty("java.home"); String javaBin = javaHome + File.separator + "bin" + File.separator + "java"; String classpath = System.getProperty("java.class.path"); classpath = classpath.concat("./src/test/resources"); - if (addClasspaths != null) { - for (String addClasspath : addClasspaths) { + if (addClassPaths != null) { + for (String addClasspath : addClassPaths) { classpath = classpath.concat(File.pathSeparatorChar + addClasspath); } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/webapp/MockDefaultRequestInterceptorREST.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/webapp/MockDefaultRequestInterceptorREST.java index b96aa5a581c64..e0a3148659acd 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/webapp/MockDefaultRequestInterceptorREST.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/webapp/MockDefaultRequestInterceptorREST.java @@ -20,21 +20,76 @@ import java.io.IOException; import java.net.ConnectException; -import java.util.HashSet; +import java.util.ArrayList; import java.util.Set; +import java.util.Map; +import java.util.HashMap; +import java.util.Collections; +import java.util.Arrays; import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.ConcurrentHashMap; +import java.util.stream.Collectors; import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; import javax.ws.rs.core.HttpHeaders; import javax.ws.rs.core.Response; import javax.ws.rs.core.Response.Status; +import org.apache.commons.lang3.EnumUtils; +import org.apache.commons.lang3.StringUtils; +import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem; import org.apache.hadoop.security.authorize.AuthorizationException; +import org.apache.hadoop.util.Sets; +import org.apache.hadoop.util.Time; import org.apache.hadoop.yarn.api.records.ApplicationId; +import org.apache.hadoop.yarn.api.records.ReservationId; import org.apache.hadoop.yarn.api.records.Resource; +import org.apache.hadoop.yarn.api.records.ContainerId; +import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; +import org.apache.hadoop.yarn.api.records.NodeId; +import org.apache.hadoop.yarn.api.records.Priority; +import org.apache.hadoop.yarn.api.records.ContainerState; +import org.apache.hadoop.yarn.api.records.ContainerReport; +import org.apache.hadoop.yarn.api.records.NodeLabel; +import org.apache.hadoop.yarn.api.records.SignalContainerCommand; +import org.apache.hadoop.yarn.api.records.ApplicationReport; +import org.apache.hadoop.yarn.api.records.YarnApplicationState; +import org.apache.hadoop.yarn.api.records.FinalApplicationStatus; +import org.apache.hadoop.yarn.api.records.ApplicationAttemptReport; +import org.apache.hadoop.yarn.api.records.YarnApplicationAttemptState; +import org.apache.hadoop.yarn.api.records.ApplicationTimeoutType; +import org.apache.hadoop.yarn.api.records.ApplicationTimeout; +import org.apache.hadoop.yarn.api.protocolrecords.ReservationSubmissionRequest; +import org.apache.hadoop.yarn.api.protocolrecords.ReservationListRequest; +import org.apache.hadoop.yarn.api.protocolrecords.ReservationListResponse; +import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.exceptions.ApplicationNotFoundException; import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.server.federation.store.records.SubClusterId; +import org.apache.hadoop.yarn.server.resourcemanager.ClientRMService; +import org.apache.hadoop.yarn.server.resourcemanager.MockRM; +import org.apache.hadoop.yarn.server.resourcemanager.RMContext; +import org.apache.hadoop.yarn.server.resourcemanager.reservation.ReservationSystem; +import org.apache.hadoop.yarn.server.resourcemanager.reservation.ReservationSystemTestUtil; +import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ActiveUsersManager; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerNode; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.activities.ActivitiesManager; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.activities.ActivitiesLogger; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.activities.ActivityDiagnosticConstant; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.activities.ActivityState; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.activities.ActivityLevel; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.LeafQueue; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.TestUtils; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerApp; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerNode; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.NodeIDsInfo; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.NodeLabelsInfo; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.LabelsToNodesInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppState; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ApplicationSubmissionContextInfo; @@ -45,10 +100,30 @@ import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.NodesInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ResourceInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ResourceOptionInfo; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.NodeToLabelsInfo; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.NodeLabelInfo; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppAttemptsInfo; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppTimeoutInfo; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppTimeoutsInfo; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppPriority; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppQueue; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.StatisticsItemInfo; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ApplicationStatisticsInfo; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppActivitiesInfo; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ReservationListInfo; +import org.apache.hadoop.yarn.server.scheduler.SchedulerRequestKey; +import org.apache.hadoop.yarn.server.webapp.dao.AppAttemptInfo; +import org.apache.hadoop.yarn.server.webapp.dao.ContainerInfo; +import org.apache.hadoop.yarn.server.webapp.dao.ContainersInfo; +import org.apache.hadoop.yarn.util.SystemClock; +import org.apache.hadoop.yarn.util.resource.Resources; import org.apache.hadoop.yarn.webapp.NotFoundException; +import org.mockito.Mockito; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import static org.mockito.Mockito.mock; + /** * This class mocks the RESTRequestInterceptor. */ @@ -62,9 +137,22 @@ public class MockDefaultRequestInterceptorREST // This property allows us to write tests for specific scenario as YARN RM // down e.g. network issue, failover. private boolean isRunning = true; - private HashSet applicationMap = new HashSet<>(); + private Map applicationMap = new HashMap<>(); public static final String APP_STATE_RUNNING = "RUNNING"; + private static final String QUEUE_DEFAULT = "default"; + private static final String QUEUE_DEFAULT_FULL = CapacitySchedulerConfiguration.ROOT + + CapacitySchedulerConfiguration.DOT + QUEUE_DEFAULT; + private static final String QUEUE_DEDICATED = "dedicated"; + public static final String QUEUE_DEDICATED_FULL = CapacitySchedulerConfiguration.ROOT + + CapacitySchedulerConfiguration.DOT + QUEUE_DEDICATED; + + // duration(milliseconds), 1mins + public static final long DURATION = 60*1000; + + // Containers 4 + public static final int NUM_CONTAINERS = 4; + private void validateRunning() throws ConnectException { if (!isRunning) { throw new ConnectException("RM is stopped"); @@ -92,7 +180,23 @@ public Response submitApplication(ApplicationSubmissionContextInfo newApp, ApplicationId appId = ApplicationId.fromString(newApp.getApplicationId()); LOG.info("Application submitted: " + appId); - applicationMap.add(appId); + + // Initialize appReport + ApplicationReport appReport = ApplicationReport.newInstance( + appId, ApplicationAttemptId.newInstance(appId, 1), null, newApp.getQueue(), null, null, 0, + null, YarnApplicationState.ACCEPTED, "", null, 0, 0, null, null, null, 0, + newApp.getApplicationType(), null, null, false, Priority.newInstance(newApp.getPriority()), + null, null); + + // Initialize appTimeoutsMap + HashMap appTimeoutsMap = new HashMap<>(); + ApplicationTimeoutType timeoutType = ApplicationTimeoutType.LIFETIME; + ApplicationTimeout appTimeOut = + ApplicationTimeout.newInstance(ApplicationTimeoutType.LIFETIME, "UNLIMITED", 10); + appTimeoutsMap.put(timeoutType, appTimeOut); + appReport.setApplicationTimeouts(appTimeoutsMap); + + applicationMap.put(appId, appReport); return Response.status(Status.ACCEPTED).header(HttpHeaders.LOCATION, "") .entity(getSubClusterId()).build(); } @@ -105,7 +209,7 @@ public AppInfo getApp(HttpServletRequest hsr, String appId, } ApplicationId applicationId = ApplicationId.fromString(appId); - if (!applicationMap.contains(applicationId)) { + if (!applicationMap.containsKey(applicationId)) { throw new NotFoundException("app with id: " + appId + " not found"); } @@ -140,7 +244,7 @@ public Response updateAppState(AppState targetState, HttpServletRequest hsr, validateRunning(); ApplicationId applicationId = ApplicationId.fromString(appId); - if (!applicationMap.remove(applicationId)) { + if (applicationMap.remove(applicationId) == null) { throw new ApplicationNotFoundException( "Trying to kill an absent application: " + appId); } @@ -213,7 +317,7 @@ public AppState getAppState(HttpServletRequest hsr, String appId) } ApplicationId applicationId = ApplicationId.fromString(appId); - if (!applicationMap.contains(applicationId)) { + if (!applicationMap.containsKey(applicationId)) { throw new NotFoundException("app with id: " + appId + " not found"); } @@ -231,4 +335,559 @@ public boolean isRunning() { public void setRunning(boolean runningMode) { this.isRunning = runningMode; } + + @Override + public ContainersInfo getContainers(HttpServletRequest req, HttpServletResponse res, + String appId, String appAttemptId) { + if (!isRunning) { + throw new RuntimeException("RM is stopped"); + } + + // We avoid to check if the Application exists in the system because we need + // to validate that each subCluster returns 1 container. + ContainersInfo containers = new ContainersInfo(); + + int subClusterId = Integer.valueOf(getSubClusterId().getId()); + + ContainerId containerId = ContainerId.newContainerId( + ApplicationAttemptId.fromString(appAttemptId), subClusterId); + Resource allocatedResource = + Resource.newInstance(subClusterId, subClusterId); + + NodeId assignedNode = NodeId.newInstance("Node", subClusterId); + Priority priority = Priority.newInstance(subClusterId); + long creationTime = subClusterId; + long finishTime = subClusterId; + String diagnosticInfo = "Diagnostic " + subClusterId; + String logUrl = "Log " + subClusterId; + int containerExitStatus = subClusterId; + ContainerState containerState = ContainerState.COMPLETE; + String nodeHttpAddress = "HttpAddress " + subClusterId; + + ContainerReport containerReport = ContainerReport.newInstance( + containerId, allocatedResource, assignedNode, priority, + creationTime, finishTime, diagnosticInfo, logUrl, + containerExitStatus, containerState, nodeHttpAddress); + + ContainerInfo container = new ContainerInfo(containerReport); + containers.add(container); + + return containers; + } + + @Override + public NodeToLabelsInfo getNodeToLabels(HttpServletRequest hsr) throws IOException { + if (!isRunning) { + throw new RuntimeException("RM is stopped"); + } + NodeLabelsInfo cpuNode = new NodeLabelsInfo(Collections.singleton("CPU")); + NodeLabelsInfo gpuNode = new NodeLabelsInfo(Collections.singleton("GPU")); + + HashMap nodeLabels = new HashMap<>(); + nodeLabels.put("node1", cpuNode); + nodeLabels.put("node2", gpuNode); + return new NodeToLabelsInfo(nodeLabels); + } + + @Override + public LabelsToNodesInfo getLabelsToNodes(Set labels) throws IOException { + if (!isRunning) { + throw new RuntimeException("RM is stopped"); + } + + Map labelsToNodes = new HashMap<>(); + + NodeLabel labelX = NodeLabel.newInstance("x", false); + NodeLabelInfo nodeLabelInfoX = new NodeLabelInfo(labelX); + ArrayList hostsX = new ArrayList<>(Arrays.asList("host1A", "host1B")); + Resource resourceX = Resource.newInstance(20*1024, 10); + NodeIDsInfo nodeIDsInfoX = new NodeIDsInfo(hostsX, resourceX); + labelsToNodes.put(nodeLabelInfoX, nodeIDsInfoX); + + NodeLabel labelY = NodeLabel.newInstance("y", false); + NodeLabelInfo nodeLabelInfoY = new NodeLabelInfo(labelY); + ArrayList hostsY = new ArrayList<>(Arrays.asList("host2A", "host2B")); + Resource resourceY = Resource.newInstance(40*1024, 20); + NodeIDsInfo nodeIDsInfoY = new NodeIDsInfo(hostsY, resourceY); + labelsToNodes.put(nodeLabelInfoY, nodeIDsInfoY); + + NodeLabel labelZ = NodeLabel.newInstance("z", false); + NodeLabelInfo nodeLabelInfoZ = new NodeLabelInfo(labelZ); + ArrayList hostsZ = new ArrayList<>(Arrays.asList("host3A", "host3B")); + Resource resourceZ = Resource.newInstance(80*1024, 40); + NodeIDsInfo nodeIDsInfoZ = new NodeIDsInfo(hostsZ, resourceZ); + labelsToNodes.put(nodeLabelInfoZ, nodeIDsInfoZ); + + return new LabelsToNodesInfo(labelsToNodes); + } + + @Override + public NodeLabelsInfo getClusterNodeLabels(HttpServletRequest hsr) throws IOException { + if (!isRunning) { + throw new RuntimeException("RM is stopped"); + } + NodeLabel labelCpu = NodeLabel.newInstance("cpu", false); + NodeLabel labelGpu = NodeLabel.newInstance("gpu", false); + return new NodeLabelsInfo(Sets.newHashSet(labelCpu, labelGpu)); + } + + @Override + public NodeLabelsInfo getLabelsOnNode(HttpServletRequest hsr, String nodeId) throws IOException { + if (!isRunning) { + throw new RuntimeException("RM is stopped"); + } + + if (StringUtils.equalsIgnoreCase(nodeId, "node1")) { + NodeLabel labelCpu = NodeLabel.newInstance("x", false); + NodeLabel labelGpu = NodeLabel.newInstance("y", false); + return new NodeLabelsInfo(Sets.newHashSet(labelCpu, labelGpu)); + } else { + return null; + } + } + + @Override + public ContainerInfo getContainer(HttpServletRequest req, HttpServletResponse res, + String appId, String appAttemptId, String containerId) { + if (!isRunning) { + throw new RuntimeException("RM is stopped"); + } + + ContainerId newContainerId = ContainerId.newContainerId( + ApplicationAttemptId.fromString(appAttemptId), Integer.valueOf(containerId)); + + Resource allocatedResource = Resource.newInstance(1024, 2); + + int subClusterId = Integer.valueOf(getSubClusterId().getId()); + NodeId assignedNode = NodeId.newInstance("Node", subClusterId); + Priority priority = Priority.newInstance(subClusterId); + long creationTime = subClusterId; + long finishTime = subClusterId; + String diagnosticInfo = "Diagnostic " + subClusterId; + String logUrl = "Log " + subClusterId; + int containerExitStatus = subClusterId; + ContainerState containerState = ContainerState.COMPLETE; + String nodeHttpAddress = "HttpAddress " + subClusterId; + + ContainerReport containerReport = ContainerReport.newInstance( + newContainerId, allocatedResource, assignedNode, priority, + creationTime, finishTime, diagnosticInfo, logUrl, + containerExitStatus, containerState, nodeHttpAddress); + + return new ContainerInfo(containerReport); + } + + @Override + public Response signalToContainer(String containerId, String command, + HttpServletRequest req) throws AuthorizationException { + if (!isRunning) { + throw new RuntimeException("RM is stopped"); + } + + if (!EnumUtils.isValidEnum(SignalContainerCommand.class, command.toUpperCase())) { + String errMsg = "Invalid command: " + command.toUpperCase() + ", valid commands are: " + + Arrays.asList(SignalContainerCommand.values()); + return Response.status(Status.BAD_REQUEST).entity(errMsg).build(); + } + + return Response.status(Status.OK).build(); + } + + @Override + public AppAttemptInfo getAppAttempt(HttpServletRequest req, HttpServletResponse res, + String appId, String appAttemptId) { + if (!isRunning) { + throw new RuntimeException("RM is stopped"); + } + + ApplicationId applicationId = ApplicationId.fromString(appId); + if (!applicationMap.containsKey(applicationId)) { + throw new NotFoundException("app with id: " + appId + " not found"); + } + + ApplicationReport newApplicationReport = ApplicationReport.newInstance( + applicationId, ApplicationAttemptId.newInstance(applicationId, Integer.parseInt(appAttemptId)), + "user", "queue", "appname", "host", 124, null, + YarnApplicationState.RUNNING, "diagnostics", "url", 1, 2, 3, 4, + FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.53789f, "YARN", null); + + ApplicationAttemptReport attempt = ApplicationAttemptReport.newInstance( + ApplicationAttemptId.newInstance(applicationId, Integer.parseInt(appAttemptId)), + "host", 124, "url", "oUrl", "diagnostics", + YarnApplicationAttemptState.FINISHED, ContainerId.newContainerId( + newApplicationReport.getCurrentApplicationAttemptId(), 1)); + + return new AppAttemptInfo(attempt); + } + + @Override + public AppAttemptsInfo getAppAttempts(HttpServletRequest hsr, String appId) { + if (!isRunning) { + throw new RuntimeException("RM is stopped"); + } + + ApplicationId applicationId = ApplicationId.fromString(appId); + if (!applicationMap.containsKey(applicationId)) { + throw new NotFoundException("app with id: " + appId + " not found"); + } + + AppAttemptsInfo infos = new AppAttemptsInfo(); + infos.add(TestRouterWebServiceUtil.generateAppAttemptInfo(0)); + infos.add(TestRouterWebServiceUtil.generateAppAttemptInfo(1)); + return infos; + } + + @Override + public AppTimeoutInfo getAppTimeout(HttpServletRequest hsr, + String appId, String type) throws AuthorizationException { + + if (!isRunning) { + throw new RuntimeException("RM is stopped"); + } + + ApplicationId applicationId = ApplicationId.fromString(appId); + if (!applicationMap.containsKey(applicationId)) { + throw new NotFoundException("app with id: " + appId + " not found"); + } + + ApplicationReport appReport = applicationMap.get(applicationId); + Map timeouts = appReport.getApplicationTimeouts(); + ApplicationTimeoutType paramType = ApplicationTimeoutType.valueOf(type); + + if (paramType == null) { + throw new NotFoundException("application timeout type not found"); + } + + if (!timeouts.containsKey(paramType)) { + throw new NotFoundException("timeout with id: " + appId + " not found"); + } + + ApplicationTimeout applicationTimeout = timeouts.get(paramType); + + AppTimeoutInfo timeoutInfo = new AppTimeoutInfo(); + timeoutInfo.setExpiryTime(applicationTimeout.getExpiryTime()); + timeoutInfo.setTimeoutType(applicationTimeout.getTimeoutType()); + timeoutInfo.setRemainingTime(applicationTimeout.getRemainingTime()); + + return timeoutInfo; + } + + @Override + public AppTimeoutsInfo getAppTimeouts(HttpServletRequest hsr, String appId) + throws AuthorizationException { + + if (!isRunning) { + throw new RuntimeException("RM is stopped"); + } + + ApplicationId applicationId = ApplicationId.fromString(appId); + + if (!applicationMap.containsKey(applicationId)) { + throw new NotFoundException("app with id: " + appId + " not found"); + } + + ApplicationReport appReport = applicationMap.get(applicationId); + Map timeouts = appReport.getApplicationTimeouts(); + + AppTimeoutsInfo timeoutsInfo = new AppTimeoutsInfo(); + + for (ApplicationTimeout timeout : timeouts.values()) { + AppTimeoutInfo timeoutInfo = new AppTimeoutInfo(); + timeoutInfo.setExpiryTime(timeout.getExpiryTime()); + timeoutInfo.setTimeoutType(timeout.getTimeoutType()); + timeoutInfo.setRemainingTime(timeout.getRemainingTime()); + timeoutsInfo.add(timeoutInfo); + } + + return timeoutsInfo; + } + + @Override + public Response updateApplicationTimeout(AppTimeoutInfo appTimeout, HttpServletRequest hsr, + String appId) throws AuthorizationException, + YarnException, InterruptedException, IOException { + + if (!isRunning) { + throw new RuntimeException("RM is stopped"); + } + + ApplicationId applicationId = ApplicationId.fromString(appId); + + if (!applicationMap.containsKey(applicationId)) { + throw new NotFoundException("app with id: " + appId + " not found"); + } + + ApplicationReport appReport = applicationMap.get(applicationId); + Map timeouts = appReport.getApplicationTimeouts(); + + ApplicationTimeoutType paramTimeoutType = appTimeout.getTimeoutType(); + if (!timeouts.containsKey(paramTimeoutType)) { + throw new NotFoundException("TimeOutType with id: " + appId + " not found"); + } + + ApplicationTimeout applicationTimeout = timeouts.get(paramTimeoutType); + applicationTimeout.setTimeoutType(appTimeout.getTimeoutType()); + applicationTimeout.setExpiryTime(appTimeout.getExpireTime()); + applicationTimeout.setRemainingTime(appTimeout.getRemainingTimeInSec()); + + AppTimeoutInfo result = new AppTimeoutInfo(applicationTimeout); + + return Response.status(Status.OK).entity(result).build(); + } + + @Override + public Response updateApplicationPriority(AppPriority targetPriority, HttpServletRequest hsr, + String appId) throws YarnException, InterruptedException, IOException { + if (!isRunning) { + throw new RuntimeException("RM is stopped"); + } + + ApplicationId applicationId = ApplicationId.fromString(appId); + if (targetPriority == null) { + return Response.status(Status.BAD_REQUEST).build(); + } + + if (!applicationMap.containsKey(applicationId)) { + throw new NotFoundException("app with id: " + appId + " not found"); + } + + ApplicationReport appReport = applicationMap.get(applicationId); + Priority newPriority = Priority.newInstance(targetPriority.getPriority()); + appReport.setPriority(newPriority); + + return Response.status(Status.OK).entity(targetPriority).build(); + } + + @Override + public AppPriority getAppPriority(HttpServletRequest hsr, String appId) + throws AuthorizationException { + if (!isRunning) { + throw new RuntimeException("RM is stopped"); + } + + ApplicationId applicationId = ApplicationId.fromString(appId); + + if (!applicationMap.containsKey(applicationId)) { + throw new NotFoundException("app with id: " + appId + " not found"); + } + ApplicationReport appReport = applicationMap.get(applicationId); + Priority priority = appReport.getPriority(); + + return new AppPriority(priority.getPriority()); + } + + @Override + public AppQueue getAppQueue(HttpServletRequest hsr, String appId) + throws AuthorizationException { + if (!isRunning) { + throw new RuntimeException("RM is stopped"); + } + ApplicationId applicationId = ApplicationId.fromString(appId); + if (!applicationMap.containsKey(applicationId)) { + throw new NotFoundException("app with id: " + appId + " not found"); + } + String queue = applicationMap.get(applicationId).getQueue(); + return new AppQueue(queue); + } + + @Override + public Response updateAppQueue(AppQueue targetQueue, HttpServletRequest hsr, String appId) + throws AuthorizationException, YarnException, InterruptedException, IOException { + if (!isRunning) { + throw new RuntimeException("RM is stopped"); + } + ApplicationId applicationId = ApplicationId.fromString(appId); + if (!applicationMap.containsKey(applicationId)) { + throw new NotFoundException("app with id: " + appId + " not found"); + } + if (targetQueue == null || StringUtils.isBlank(targetQueue.getQueue())) { + return Response.status(Status.BAD_REQUEST).build(); + } + + ApplicationReport appReport = applicationMap.get(applicationId); + String originalQueue = appReport.getQueue(); + appReport.setQueue(targetQueue.getQueue()); + applicationMap.put(applicationId, appReport); + LOG.info("Update applicationId = {} from originalQueue = {} to targetQueue = {}.", + appId, originalQueue, targetQueue); + + AppQueue targetAppQueue = new AppQueue(targetQueue.getQueue()); + return Response.status(Status.OK).entity(targetAppQueue).build(); + } + + public void updateApplicationState(YarnApplicationState appState, String appId) + throws AuthorizationException, YarnException, InterruptedException, IOException { + validateRunning(); + ApplicationId applicationId = ApplicationId.fromString(appId); + if (!applicationMap.containsKey(applicationId)) { + throw new NotFoundException("app with id: " + appId + " not found"); + } + ApplicationReport appReport = applicationMap.get(applicationId); + appReport.setYarnApplicationState(appState); + } + + @Override + public ApplicationStatisticsInfo getAppStatistics( + HttpServletRequest hsr, Set stateQueries, Set typeQueries) { + if (!isRunning) { + throw new RuntimeException("RM is stopped"); + } + + Map itemInfoMap = new HashMap<>(); + + for (ApplicationReport appReport : applicationMap.values()) { + + YarnApplicationState appState = appReport.getYarnApplicationState(); + String appType = appReport.getApplicationType(); + + if (stateQueries.contains(appState.name()) && typeQueries.contains(appType)) { + String itemInfoMapKey = appState.toString() + "_" + appType; + StatisticsItemInfo itemInfo = itemInfoMap.getOrDefault(itemInfoMapKey, null); + if (itemInfo == null) { + itemInfo = new StatisticsItemInfo(appState, appType, 1); + } else { + long newCount = itemInfo.getCount() + 1; + itemInfo.setCount(newCount); + } + itemInfoMap.put(itemInfoMapKey, itemInfo); + } + } + + return new ApplicationStatisticsInfo(itemInfoMap.values()); + } + + @Override + public AppActivitiesInfo getAppActivities( + HttpServletRequest hsr, String appId, String time, Set requestPriorities, + Set allocationRequestIds, String groupBy, String limit, Set actions, + boolean summarize) { + if (!isRunning) { + throw new RuntimeException("RM is stopped"); + } + + ApplicationId applicationId = ApplicationId.fromString(appId); + if (!applicationMap.containsKey(applicationId)) { + throw new NotFoundException("app with id: " + appId + " not found"); + } + + SchedulerNode schedulerNode = TestUtils.getMockNode("host0", "rack", 1, 10240); + + RMContext rmContext = Mockito.mock(RMContext.class); + Mockito.when(rmContext.getYarnConfiguration()).thenReturn(this.getConf()); + ResourceScheduler scheduler = Mockito.mock(ResourceScheduler.class); + Mockito.when(scheduler.getMinimumResourceCapability()).thenReturn(Resources.none()); + Mockito.when(rmContext.getScheduler()).thenReturn(scheduler); + LeafQueue mockQueue = Mockito.mock(LeafQueue.class); + Map rmApps = new ConcurrentHashMap<>(); + Mockito.doReturn(rmApps).when(rmContext).getRMApps(); + + FiCaSchedulerNode node = (FiCaSchedulerNode) schedulerNode; + ApplicationAttemptId appAttemptId = ApplicationAttemptId.newInstance(applicationId, 0); + RMApp mockApp = Mockito.mock(RMApp.class); + Mockito.doReturn(appAttemptId.getApplicationId()).when(mockApp).getApplicationId(); + Mockito.doReturn(FinalApplicationStatus.UNDEFINED).when(mockApp).getFinalApplicationStatus(); + rmApps.put(appAttemptId.getApplicationId(), mockApp); + FiCaSchedulerApp app = new FiCaSchedulerApp(appAttemptId, "user", mockQueue, + mock(ActiveUsersManager.class), rmContext); + + ActivitiesManager newActivitiesManager = new ActivitiesManager(rmContext); + newActivitiesManager.turnOnAppActivitiesRecording(app.getApplicationId(), 3); + + int numActivities = 10; + for (int i = 0; i < numActivities; i++) { + ActivitiesLogger.APP.startAppAllocationRecording(newActivitiesManager, node, + SystemClock.getInstance().getTime(), app); + ActivitiesLogger.APP.recordAppActivityWithoutAllocation(newActivitiesManager, node, app, + new SchedulerRequestKey(Priority.newInstance(0), 0, null), + ActivityDiagnosticConstant.NODE_IS_BLACKLISTED, ActivityState.REJECTED, + ActivityLevel.NODE); + ActivitiesLogger.APP.finishSkippedAppAllocationRecording(newActivitiesManager, + app.getApplicationId(), ActivityState.SKIPPED, ActivityDiagnosticConstant.EMPTY); + } + + Set prioritiesInt = + requestPriorities.stream().map(pri -> Integer.parseInt(pri)).collect(Collectors.toSet()); + Set allocationReqIds = + allocationRequestIds.stream().map(id -> Long.parseLong(id)).collect(Collectors.toSet()); + AppActivitiesInfo appActivitiesInfo = newActivitiesManager. + getAppActivitiesInfo(app.getApplicationId(), prioritiesInt, allocationReqIds, null, + Integer.parseInt(limit), summarize, 3); + + return appActivitiesInfo; + } + + @Override + public Response listReservation(String queue, String reservationId, long startTime, long endTime, + boolean includeResourceAllocations, HttpServletRequest hsr) throws Exception { + + if (!isRunning) { + throw new RuntimeException("RM is stopped"); + } + + if (!StringUtils.equals(queue, QUEUE_DEDICATED_FULL)) { + throw new RuntimeException("The specified queue: " + queue + + " is not managed by reservation system." + + " Please try again with a valid reservable queue."); + } + + MockRM mockRM = setupResourceManager(); + + ReservationId reservationID = ReservationId.parseReservationId(reservationId); + ReservationSystem reservationSystem = mockRM.getReservationSystem(); + reservationSystem.synchronizePlan(QUEUE_DEDICATED_FULL, true); + + // Generate reserved resources + ClientRMService clientService = mockRM.getClientRMService(); + + // arrival time from which the resource(s) can be allocated. + long arrival = Time.now(); + + // deadline by when the resource(s) must be allocated. + // The reason for choosing 1.05 is because this gives an integer + // DURATION * 0.05 = 3000(ms) + // deadline = arrival + 3000ms + long deadline = (long) (arrival + 1.05 * DURATION); + + // In this test of reserved resources, we will apply for 4 containers (1 core, 1GB memory) + // arrival = Time.now(), and make sure deadline - arrival > duration, + // the current setting is greater than 3000ms + ReservationSubmissionRequest submissionRequest = + ReservationSystemTestUtil.createSimpleReservationRequest( + reservationID, NUM_CONTAINERS, arrival, deadline, DURATION); + clientService.submitReservation(submissionRequest); + + // listReservations + ReservationListRequest request = ReservationListRequest.newInstance( + queue, reservationID.toString(), startTime, endTime, includeResourceAllocations); + ReservationListResponse resRespInfo = clientService.listReservations(request); + ReservationListInfo resResponse = + new ReservationListInfo(resRespInfo, includeResourceAllocations); + + if (mockRM != null) { + mockRM.stop(); + } + + return Response.status(Status.OK).entity(resResponse).build(); + } + + private MockRM setupResourceManager() throws Exception { + DefaultMetricsSystem.setMiniClusterMode(true); + + CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration(); + + // Define default queue + conf.setCapacity(QUEUE_DEFAULT_FULL, 20); + // Define dedicated queues + conf.setQueues(CapacitySchedulerConfiguration.ROOT, + new String[] {QUEUE_DEFAULT, QUEUE_DEDICATED}); + conf.setCapacity(QUEUE_DEDICATED_FULL, 80); + conf.setReservable(QUEUE_DEDICATED_FULL, true); + + conf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class, ResourceScheduler.class); + conf.setBoolean(YarnConfiguration.RM_RESERVATION_SYSTEM_ENABLE, true); + MockRM rm = new MockRM(conf); + rm.start(); + rm.registerNode("127.0.0.1:5678", 100*1024, 100); + return rm; + } } \ No newline at end of file diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/webapp/MockRESTRequestInterceptor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/webapp/MockRESTRequestInterceptor.java index 68b8db6f334c1..5951676a6d8e9 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/webapp/MockRESTRequestInterceptor.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/webapp/MockRESTRequestInterceptor.java @@ -62,6 +62,7 @@ import org.apache.hadoop.yarn.server.webapp.dao.AppAttemptInfo; import org.apache.hadoop.yarn.server.webapp.dao.ContainerInfo; import org.apache.hadoop.yarn.server.webapp.dao.ContainersInfo; +import org.apache.hadoop.yarn.webapp.dao.SchedConfUpdateInfo; /** * This class mocks the RESTRequestInterceptor. @@ -373,4 +374,16 @@ public Response signalToContainer(String containerId, String command, HttpServletRequest req) { return Response.status(Status.OK).build(); } + + @Override + public Response updateSchedulerConfiguration(SchedConfUpdateInfo mutationInfo, + HttpServletRequest hsr) throws AuthorizationException, InterruptedException { + return Response.status(Status.OK).build(); + } + + @Override + public Response getSchedulerConfiguration(HttpServletRequest hsr) + throws AuthorizationException { + return Response.status(Status.OK).build(); + } } \ No newline at end of file diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/webapp/PassThroughRESTRequestInterceptor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/webapp/PassThroughRESTRequestInterceptor.java index 31f8b8b990fd9..84a6de3205f5e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/webapp/PassThroughRESTRequestInterceptor.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/webapp/PassThroughRESTRequestInterceptor.java @@ -60,10 +60,11 @@ import org.apache.hadoop.yarn.server.webapp.dao.AppAttemptInfo; import org.apache.hadoop.yarn.server.webapp.dao.ContainerInfo; import org.apache.hadoop.yarn.server.webapp.dao.ContainersInfo; +import org.apache.hadoop.yarn.webapp.dao.SchedConfUpdateInfo; /** - * Mock intercepter that does not do anything other than forwarding it to the - * next intercepter in the chain. + * Mock interceptor that does not do anything other than forwarding it to the + * next interceptor in the chain. */ public class PassThroughRESTRequestInterceptor extends AbstractRESTRequestInterceptor { @@ -379,4 +380,17 @@ public Response signalToContainer(String containerId, String command, HttpServletRequest req) throws AuthorizationException { return getNextInterceptor().signalToContainer(containerId, command, req); } + + @Override + public Response updateSchedulerConfiguration(SchedConfUpdateInfo mutationInfo, + HttpServletRequest hsr) + throws AuthorizationException, InterruptedException { + return getNextInterceptor().updateSchedulerConfiguration(mutationInfo, hsr); + } + + @Override + public Response getSchedulerConfiguration(HttpServletRequest hsr) + throws AuthorizationException { + return getNextInterceptor().getSchedulerConfiguration(hsr); + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/webapp/TestFederationInterceptorREST.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/webapp/TestFederationInterceptorREST.java index b3a7e9060c53b..c617f8612e91a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/webapp/TestFederationInterceptorREST.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/webapp/TestFederationInterceptorREST.java @@ -21,12 +21,24 @@ import java.io.IOException; import java.util.ArrayList; import java.util.List; +import java.util.HashMap; +import java.util.Map; +import java.util.Set; +import java.util.HashSet; +import java.util.Collections; import javax.ws.rs.core.Response; +import javax.ws.rs.core.Response.Status; +import org.apache.hadoop.util.Time; import org.apache.hadoop.yarn.api.records.ApplicationId; +import org.apache.hadoop.yarn.api.records.ReservationId; import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.api.records.ResourceOption; +import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; +import org.apache.hadoop.yarn.api.records.NodeLabel; +import org.apache.hadoop.yarn.api.records.ApplicationTimeoutType; +import org.apache.hadoop.yarn.api.records.YarnApplicationState; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.server.federation.policies.manager.UniformBroadcastPolicyManager; @@ -35,6 +47,11 @@ import org.apache.hadoop.yarn.server.federation.store.records.SubClusterInfo; import org.apache.hadoop.yarn.server.federation.store.records.SubClusterRegisterRequest; import org.apache.hadoop.yarn.server.federation.store.records.SubClusterState; +import org.apache.hadoop.yarn.server.federation.store.records.GetApplicationHomeSubClusterRequest; +import org.apache.hadoop.yarn.server.federation.store.records.GetApplicationHomeSubClusterResponse; +import org.apache.hadoop.yarn.server.federation.store.records.ApplicationHomeSubCluster; +import org.apache.hadoop.yarn.server.federation.store.records.ReservationHomeSubCluster; +import org.apache.hadoop.yarn.server.federation.store.records.AddReservationHomeSubClusterRequest; import org.apache.hadoop.yarn.server.federation.utils.FederationStateStoreFacade; import org.apache.hadoop.yarn.server.federation.utils.FederationStateStoreTestUtil; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppInfo; @@ -47,18 +64,44 @@ import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.NodesInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ResourceInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ResourceOptionInfo; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.NodeToLabelsInfo; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.NodeLabelsInfo; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.NodeLabelInfo; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.LabelsToNodesInfo; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppAttemptsInfo; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppAttemptInfo; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppTimeoutInfo; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppTimeoutsInfo; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.StatisticsItemInfo; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppPriority; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppQueue; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ReservationListInfo; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ReservationInfo; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.NodeIDsInfo; +import org.apache.hadoop.yarn.server.router.webapp.cache.RouterAppInfoCacheKey; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ApplicationStatisticsInfo; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppActivitiesInfo; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ReservationDefinitionInfo; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ReservationRequestsInfo; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ReservationRequestInfo; +import org.apache.hadoop.yarn.server.webapp.dao.ContainerInfo; +import org.apache.hadoop.yarn.server.webapp.dao.ContainersInfo; +import org.apache.hadoop.yarn.util.LRUCacheHashMap; import org.apache.hadoop.yarn.util.MonotonicClock; +import org.apache.hadoop.yarn.util.Times; import org.junit.Assert; import org.junit.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import static org.apache.hadoop.yarn.server.router.webapp.MockDefaultRequestInterceptorREST.QUEUE_DEDICATED_FULL; + /** * Extends the {@code BaseRouterClientRMTest} and overrides methods in order to * use the {@code RouterClientRMService} pipeline test cases for testing the * {@code FederationInterceptor} class. The tests for * {@code RouterClientRMService} has been written cleverly so that it can be - * reused to validate different request intercepter chains. + * reused to validate different request interceptor chains. */ public class TestFederationInterceptorREST extends BaseRouterWebServicesTest { private static final Logger LOG = @@ -116,8 +159,8 @@ protected YarnConfiguration createConfiguration() { String mockPassThroughInterceptorClass = PassThroughRESTRequestInterceptor.class.getName(); - // Create a request intercepter pipeline for testing. The last one in the - // chain is the federation intercepter that calls the mock resource manager. + // Create a request interceptor pipeline for testing. The last one in the + // chain is the federation interceptor that calls the mock resource manager. // The others in the chain will simply forward it to the next one in the // chain conf.set(YarnConfiguration.ROUTER_CLIENTRM_INTERCEPTOR_CLASS_PIPELINE, @@ -130,6 +173,10 @@ protected YarnConfiguration createConfiguration() { // Disable StateStoreFacade cache conf.setInt(YarnConfiguration.FEDERATION_CACHE_TIME_TO_LIVE_SECS, 0); + // Open AppsInfo Cache + conf.setBoolean(YarnConfiguration.ROUTER_APPSINFO_ENABLED, true); + conf.setInt(YarnConfiguration.ROUTER_APPSINFO_CACHED_COUNT, 10); + return conf; } @@ -160,7 +207,7 @@ public void testSubmitApplication() throws YarnException, IOException, InterruptedException { ApplicationId appId = - ApplicationId.newInstance(System.currentTimeMillis(), 1); + ApplicationId.newInstance(Time.now(), 1); ApplicationSubmissionContextInfo context = new ApplicationSubmissionContextInfo(); @@ -187,7 +234,7 @@ public void testSubmitApplicationMultipleSubmission() throws YarnException, IOException, InterruptedException { ApplicationId appId = - ApplicationId.newInstance(System.currentTimeMillis(), 1); + ApplicationId.newInstance(Time.now(), 1); ApplicationSubmissionContextInfo context = new ApplicationSubmissionContextInfo(); context.setApplicationId(appId.toString()); @@ -236,7 +283,7 @@ public void testSubmitApplicationEmptyRequest() } /** - * This test validates the correctness of SubmitApplication in case of of + * This test validates the correctness of SubmitApplication in case of * application in wrong format. */ @Test @@ -259,7 +306,7 @@ public void testForceKillApplication() throws YarnException, IOException, InterruptedException { ApplicationId appId = - ApplicationId.newInstance(System.currentTimeMillis(), 1); + ApplicationId.newInstance(Time.now(), 1); ApplicationSubmissionContextInfo context = new ApplicationSubmissionContextInfo(); context.setApplicationId(appId.toString()); @@ -286,7 +333,7 @@ public void testForceKillApplicationNotExists() throws YarnException, IOException, InterruptedException { ApplicationId appId = - ApplicationId.newInstance(System.currentTimeMillis(), 1); + ApplicationId.newInstance(Time.now(), 1); AppState appState = new AppState("KILLED"); Response response = @@ -317,7 +364,7 @@ public void testForceKillApplicationWrongFormat() public void testForceKillApplicationEmptyRequest() throws YarnException, IOException, InterruptedException { ApplicationId appId = - ApplicationId.newInstance(System.currentTimeMillis(), 1); + ApplicationId.newInstance(Time.now(), 1); ApplicationSubmissionContextInfo context = new ApplicationSubmissionContextInfo(); @@ -341,7 +388,7 @@ public void testGetApplicationReport() throws YarnException, IOException, InterruptedException { ApplicationId appId = - ApplicationId.newInstance(System.currentTimeMillis(), 1); + ApplicationId.newInstance(Time.now(), 1); ApplicationSubmissionContextInfo context = new ApplicationSubmissionContextInfo(); context.setApplicationId(appId.toString()); @@ -478,7 +525,7 @@ public void testGetApplicationState() throws YarnException, IOException, InterruptedException { ApplicationId appId = - ApplicationId.newInstance(System.currentTimeMillis(), 1); + ApplicationId.newInstance(Time.now(), 1); ApplicationSubmissionContextInfo context = new ApplicationSubmissionContextInfo(); context.setApplicationId(appId.toString()); @@ -505,7 +552,7 @@ public void testGetApplicationStateNotExists() throws YarnException, IOException, InterruptedException { ApplicationId appId = - ApplicationId.newInstance(System.currentTimeMillis(), 1); + ApplicationId.newInstance(Time.now(), 1); AppState response = interceptor.getAppState(null, appId.toString()); @@ -560,4 +607,524 @@ SubClusterState.SC_RUNNING, new MonotonicClock().getTime(), SubClusterRegisterRequest.newInstance(subClusterInfo)); } + @Test + public void testGetContainers() + throws YarnException, IOException, InterruptedException { + + ApplicationId appId = ApplicationId.newInstance(Time.now(), 1); + ApplicationSubmissionContextInfo context = + new ApplicationSubmissionContextInfo(); + context.setApplicationId(appId.toString()); + + // Submit the application we want the report later + Response response = interceptor.submitApplication(context, null); + + Assert.assertNotNull(response); + Assert.assertNotNull(stateStoreUtil.queryApplicationHomeSC(appId)); + + ApplicationAttemptId appAttempt = ApplicationAttemptId.newInstance(appId, 1); + + ContainersInfo responseGet = interceptor.getContainers( + null, null, appId.toString(), appAttempt.toString()); + + Assert.assertEquals(4, responseGet.getContainers().size()); + } + + @Test + public void testGetContainersNotExists() { + ApplicationId appId = ApplicationId.newInstance(Time.now(), 1); + ContainersInfo response = interceptor.getContainers(null, null, appId.toString(), null); + Assert.assertTrue(response.getContainers().isEmpty()); + } + + @Test + public void testGetContainersWrongFormat() { + ContainersInfo response = interceptor.getContainers(null, null, "Application_wrong_id", null); + + Assert.assertNotNull(response); + Assert.assertTrue(response.getContainers().isEmpty()); + + ApplicationId appId = ApplicationId.newInstance(Time.now(), 1); + response = interceptor.getContainers(null, null, appId.toString(), "AppAttempt_wrong_id"); + + Assert.assertTrue(response.getContainers().isEmpty()); + } + + @Test + public void testGetNodeToLabels() throws IOException { + NodeToLabelsInfo info = interceptor.getNodeToLabels(null); + HashMap map = info.getNodeToLabels(); + Assert.assertNotNull(map); + Assert.assertEquals(2, map.size()); + + NodeLabelsInfo node1Value = map.getOrDefault("node1", null); + Assert.assertNotNull(node1Value); + Assert.assertEquals(1, node1Value.getNodeLabelsName().size()); + Assert.assertEquals("CPU", node1Value.getNodeLabelsName().get(0)); + + NodeLabelsInfo node2Value = map.getOrDefault("node2", null); + Assert.assertNotNull(node2Value); + Assert.assertEquals(1, node2Value.getNodeLabelsName().size()); + Assert.assertEquals("GPU", node2Value.getNodeLabelsName().get(0)); + } + + @Test + public void testGetLabelsToNodes() throws Exception { + LabelsToNodesInfo labelsToNodesInfo = interceptor.getLabelsToNodes(null); + Map map = labelsToNodesInfo.getLabelsToNodes(); + Assert.assertNotNull(map); + Assert.assertEquals(3, map.size()); + + NodeLabel labelX = NodeLabel.newInstance("x", false); + NodeLabelInfo nodeLabelInfoX = new NodeLabelInfo(labelX); + NodeIDsInfo nodeIDsInfoX = map.get(nodeLabelInfoX); + Assert.assertNotNull(nodeIDsInfoX); + Assert.assertEquals(2, nodeIDsInfoX.getNodeIDs().size()); + Resource resourceX = + nodeIDsInfoX.getPartitionInfo().getResourceAvailable().getResource(); + Assert.assertNotNull(resourceX); + Assert.assertEquals(4*10, resourceX.getVirtualCores()); + Assert.assertEquals(4*20*1024, resourceX.getMemorySize()); + + NodeLabel labelY = NodeLabel.newInstance("y", false); + NodeLabelInfo nodeLabelInfoY = new NodeLabelInfo(labelY); + NodeIDsInfo nodeIDsInfoY = map.get(nodeLabelInfoY); + Assert.assertNotNull(nodeIDsInfoY); + Assert.assertEquals(2, nodeIDsInfoY.getNodeIDs().size()); + Resource resourceY = + nodeIDsInfoY.getPartitionInfo().getResourceAvailable().getResource(); + Assert.assertNotNull(resourceY); + Assert.assertEquals(4*20, resourceY.getVirtualCores()); + Assert.assertEquals(4*40*1024, resourceY.getMemorySize()); + } + + @Test + public void testGetClusterNodeLabels() throws Exception { + NodeLabelsInfo nodeLabelsInfo = interceptor.getClusterNodeLabels(null); + Assert.assertNotNull(nodeLabelsInfo); + Assert.assertEquals(2, nodeLabelsInfo.getNodeLabelsName().size()); + + List nodeLabelsName = nodeLabelsInfo.getNodeLabelsName(); + Assert.assertNotNull(nodeLabelsName); + Assert.assertTrue(nodeLabelsName.contains("cpu")); + Assert.assertTrue(nodeLabelsName.contains("gpu")); + + ArrayList nodeLabelInfos = nodeLabelsInfo.getNodeLabelsInfo(); + Assert.assertNotNull(nodeLabelInfos); + Assert.assertEquals(2, nodeLabelInfos.size()); + NodeLabelInfo cpuNodeLabelInfo = new NodeLabelInfo("cpu", false); + Assert.assertTrue(nodeLabelInfos.contains(cpuNodeLabelInfo)); + NodeLabelInfo gpuNodeLabelInfo = new NodeLabelInfo("gpu", false); + Assert.assertTrue(nodeLabelInfos.contains(gpuNodeLabelInfo)); + } + + @Test + public void testGetLabelsOnNode() throws Exception { + NodeLabelsInfo nodeLabelsInfo = interceptor.getLabelsOnNode(null, "node1"); + Assert.assertNotNull(nodeLabelsInfo); + Assert.assertEquals(2, nodeLabelsInfo.getNodeLabelsName().size()); + + List nodeLabelsName = nodeLabelsInfo.getNodeLabelsName(); + Assert.assertNotNull(nodeLabelsName); + Assert.assertTrue(nodeLabelsName.contains("x")); + Assert.assertTrue(nodeLabelsName.contains("y")); + + // null request + NodeLabelsInfo nodeLabelsInfo2 = interceptor.getLabelsOnNode(null, "node2"); + Assert.assertNotNull(nodeLabelsInfo2); + Assert.assertEquals(0, nodeLabelsInfo2.getNodeLabelsName().size()); + } + + @Test + public void testGetContainer() + throws IOException, InterruptedException, YarnException { + // Submit application to multiSubCluster + ApplicationId appId = ApplicationId.newInstance(Time.now(), 1); + ApplicationSubmissionContextInfo context = new ApplicationSubmissionContextInfo(); + context.setApplicationId(appId.toString()); + + Assert.assertNotNull(interceptor.submitApplication(context, null)); + + ApplicationAttemptId appAttemptId = + ApplicationAttemptId.newInstance(appId, 1); + + ContainerInfo containerInfo = interceptor.getContainer(null, null, + appId.toString(), appAttemptId.toString(), "0"); + Assert.assertNotNull(containerInfo); + } + + @Test + public void testGetAppAttempts() + throws IOException, InterruptedException, YarnException { + // Submit application to multiSubCluster + ApplicationId appId = ApplicationId.newInstance(Time.now(), 1); + ApplicationSubmissionContextInfo context = new ApplicationSubmissionContextInfo(); + context.setApplicationId(appId.toString()); + + Assert.assertNotNull(interceptor.submitApplication(context, null)); + + AppAttemptsInfo appAttemptsInfo = interceptor.getAppAttempts(null, appId.toString()); + Assert.assertNotNull(appAttemptsInfo); + + ArrayList attemptLists = appAttemptsInfo.getAttempts(); + Assert.assertNotNull(appAttemptsInfo); + Assert.assertEquals(2, attemptLists.size()); + + AppAttemptInfo attemptInfo1 = attemptLists.get(0); + Assert.assertNotNull(attemptInfo1); + Assert.assertEquals(0, attemptInfo1.getAttemptId()); + Assert.assertEquals("AppAttemptId_0", attemptInfo1.getAppAttemptId()); + Assert.assertEquals("LogLink_0", attemptInfo1.getLogsLink()); + Assert.assertEquals(1659621705L, attemptInfo1.getFinishedTime()); + + AppAttemptInfo attemptInfo2 = attemptLists.get(1); + Assert.assertNotNull(attemptInfo2); + Assert.assertEquals(0, attemptInfo2.getAttemptId()); + Assert.assertEquals("AppAttemptId_1", attemptInfo2.getAppAttemptId()); + Assert.assertEquals("LogLink_1", attemptInfo2.getLogsLink()); + Assert.assertEquals(1659621705L, attemptInfo2.getFinishedTime()); + } + + @Test + public void testGetAppAttempt() + throws IOException, InterruptedException, YarnException { + + // Generate ApplicationId information + ApplicationId appId = ApplicationId.newInstance(Time.now(), 1); + ApplicationSubmissionContextInfo context = new ApplicationSubmissionContextInfo(); + context.setApplicationId(appId.toString()); + + // Generate ApplicationAttemptId information + Assert.assertNotNull(interceptor.submitApplication(context, null)); + ApplicationAttemptId expectAppAttemptId = ApplicationAttemptId.newInstance(appId, 1); + + org.apache.hadoop.yarn.server.webapp.dao.AppAttemptInfo + appAttemptInfo = interceptor.getAppAttempt(null, null, appId.toString(), "1"); + + Assert.assertNotNull(appAttemptInfo); + Assert.assertEquals(expectAppAttemptId.toString(), appAttemptInfo.getAppAttemptId()); + Assert.assertEquals("url", appAttemptInfo.getTrackingUrl()); + Assert.assertEquals("oUrl", appAttemptInfo.getOriginalTrackingUrl()); + Assert.assertEquals(124, appAttemptInfo.getRpcPort()); + Assert.assertEquals("host", appAttemptInfo.getHost()); + } + + @Test + public void testGetAppTimeout() throws IOException, InterruptedException, YarnException { + + // Generate ApplicationId information + ApplicationId appId = ApplicationId.newInstance(Time.now(), 1); + ApplicationSubmissionContextInfo context = new ApplicationSubmissionContextInfo(); + context.setApplicationId(appId.toString()); + + // Generate ApplicationAttemptId information + Assert.assertNotNull(interceptor.submitApplication(context, null)); + + ApplicationTimeoutType appTimeoutType = ApplicationTimeoutType.LIFETIME; + AppTimeoutInfo appTimeoutInfo = + interceptor.getAppTimeout(null, appId.toString(), appTimeoutType.toString()); + Assert.assertNotNull(appTimeoutInfo); + Assert.assertEquals(10, appTimeoutInfo.getRemainingTimeInSec()); + Assert.assertEquals("UNLIMITED", appTimeoutInfo.getExpireTime()); + Assert.assertEquals(appTimeoutType, appTimeoutInfo.getTimeoutType()); + } + + @Test + public void testGetAppTimeouts() throws IOException, InterruptedException, YarnException { + + // Generate ApplicationId information + ApplicationId appId = ApplicationId.newInstance(Time.now(), 1); + ApplicationSubmissionContextInfo context = new ApplicationSubmissionContextInfo(); + context.setApplicationId(appId.toString()); + + // Generate ApplicationAttemptId information + Assert.assertNotNull(interceptor.submitApplication(context, null)); + + AppTimeoutsInfo appTimeoutsInfo = interceptor.getAppTimeouts(null, appId.toString()); + Assert.assertNotNull(appTimeoutsInfo); + + List timeouts = appTimeoutsInfo.getAppTimeouts(); + Assert.assertNotNull(timeouts); + Assert.assertEquals(1, timeouts.size()); + + AppTimeoutInfo resultAppTimeout = timeouts.get(0); + Assert.assertNotNull(resultAppTimeout); + Assert.assertEquals(10, resultAppTimeout.getRemainingTimeInSec()); + Assert.assertEquals("UNLIMITED", resultAppTimeout.getExpireTime()); + Assert.assertEquals(ApplicationTimeoutType.LIFETIME, resultAppTimeout.getTimeoutType()); + } + + @Test + public void testUpdateApplicationTimeout() throws IOException, InterruptedException, + YarnException { + + // Generate ApplicationId information + ApplicationId appId = ApplicationId.newInstance(Time.now(), 1); + ApplicationSubmissionContextInfo context = new ApplicationSubmissionContextInfo(); + context.setApplicationId(appId.toString()); + + // Generate ApplicationAttemptId information + Assert.assertNotNull(interceptor.submitApplication(context, null)); + + long newLifetime = 10L; + // update 10L seconds more to timeout + String timeout = Times.formatISO8601(Time.now() + newLifetime * 1000); + AppTimeoutInfo paramAppTimeOut = new AppTimeoutInfo(); + paramAppTimeOut.setExpiryTime(timeout); + // RemainingTime = Math.max((timeoutInMillis - System.currentTimeMillis()) / 1000, 0)) + paramAppTimeOut.setRemainingTime(newLifetime); + paramAppTimeOut.setTimeoutType(ApplicationTimeoutType.LIFETIME); + + Response response = + interceptor.updateApplicationTimeout(paramAppTimeOut, null, appId.toString()); + Assert.assertNotNull(response); + AppTimeoutInfo entity = (AppTimeoutInfo) response.getEntity(); + Assert.assertNotNull(entity); + Assert.assertEquals(paramAppTimeOut.getExpireTime(), entity.getExpireTime()); + Assert.assertEquals(paramAppTimeOut.getTimeoutType(), entity.getTimeoutType()); + Assert.assertEquals(paramAppTimeOut.getRemainingTimeInSec(), entity.getRemainingTimeInSec()); + } + + @Test + public void testUpdateApplicationPriority() throws IOException, InterruptedException, + YarnException { + + // Submit application to multiSubCluster + ApplicationId appId = ApplicationId.newInstance(Time.now(), 1); + ApplicationSubmissionContextInfo context = new ApplicationSubmissionContextInfo(); + context.setApplicationId(appId.toString()); + context.setPriority(20); + + // Submit the application we are going to kill later + Assert.assertNotNull(interceptor.submitApplication(context, null)); + + int iPriority = 10; + // Set Priority for application + Response response = interceptor.updateApplicationPriority( + new AppPriority(iPriority), null, appId.toString()); + + Assert.assertNotNull(response); + AppPriority entity = (AppPriority) response.getEntity(); + Assert.assertNotNull(entity); + Assert.assertEquals(iPriority, entity.getPriority()); + } + + @Test + public void testGetAppPriority() throws IOException, InterruptedException, + YarnException { + + // Submit application to multiSubCluster + ApplicationId appId = ApplicationId.newInstance(Time.now(), 1); + int priority = 40; + ApplicationSubmissionContextInfo context = new ApplicationSubmissionContextInfo(); + context.setApplicationId(appId.toString()); + context.setPriority(priority); + + // Submit the application we are going to kill later + Assert.assertNotNull(interceptor.submitApplication(context, null)); + + // Set Priority for application + AppPriority appPriority = interceptor.getAppPriority(null, appId.toString()); + Assert.assertNotNull(appPriority); + Assert.assertEquals(priority, appPriority.getPriority()); + } + + @Test + public void testUpdateAppQueue() throws IOException, InterruptedException, + YarnException { + + String oldQueue = "oldQueue"; + String newQueue = "newQueue"; + + // Submit application to multiSubCluster + ApplicationId appId = ApplicationId.newInstance(Time.now(), 1); + ApplicationSubmissionContextInfo context = new ApplicationSubmissionContextInfo(); + context.setApplicationId(appId.toString()); + context.setQueue(oldQueue); + + // Submit the application + Assert.assertNotNull(interceptor.submitApplication(context, null)); + + // Set New Queue for application + Response response = interceptor.updateAppQueue(new AppQueue(newQueue), + null, appId.toString()); + + Assert.assertNotNull(response); + AppQueue appQueue = (AppQueue) response.getEntity(); + Assert.assertEquals(newQueue, appQueue.getQueue()); + + // Get AppQueue by application + AppQueue queue = interceptor.getAppQueue(null, appId.toString()); + Assert.assertNotNull(queue); + Assert.assertEquals(newQueue, queue.getQueue()); + } + + @Test + public void testGetAppQueue() throws IOException, InterruptedException, YarnException { + String queueName = "queueName"; + + // Submit application to multiSubCluster + ApplicationId appId = ApplicationId.newInstance(Time.now(), 1); + ApplicationSubmissionContextInfo context = new ApplicationSubmissionContextInfo(); + context.setApplicationId(appId.toString()); + context.setQueue(queueName); + + Assert.assertNotNull(interceptor.submitApplication(context, null)); + + // Get Queue by application + AppQueue queue = interceptor.getAppQueue(null, appId.toString()); + Assert.assertNotNull(queue); + Assert.assertEquals(queueName, queue.getQueue()); + } + + @Test + public void testGetAppsInfoCache() throws IOException, InterruptedException, YarnException { + + AppsInfo responseGet = interceptor.getApps( + null, null, null, null, null, null, null, null, null, null, null, null, null, null, null); + Assert.assertNotNull(responseGet); + + RouterAppInfoCacheKey cacheKey = RouterAppInfoCacheKey.newInstance( + null, null, null, null, null, null, null, null, null, null, null, null, null, null, null); + + LRUCacheHashMap appsInfoCache = + interceptor.getAppInfosCaches(); + Assert.assertNotNull(appsInfoCache); + Assert.assertTrue(!appsInfoCache.isEmpty()); + Assert.assertEquals(1, appsInfoCache.size()); + Assert.assertTrue(appsInfoCache.containsKey(cacheKey)); + + AppsInfo cacheResult = appsInfoCache.get(cacheKey); + Assert.assertNotNull(cacheResult); + Assert.assertEquals(responseGet, cacheResult); + } + + @Test + public void testGetAppStatistics() throws IOException, InterruptedException, YarnException { + AppState appStateRUNNING = new AppState(YarnApplicationState.RUNNING.name()); + + // Submit application to multiSubCluster + ApplicationId appId = ApplicationId.newInstance(Time.now(), 1); + ApplicationSubmissionContextInfo context = new ApplicationSubmissionContextInfo(); + context.setApplicationId(appId.toString()); + context.setApplicationType("MapReduce"); + context.setQueue("queue"); + + Assert.assertNotNull(interceptor.submitApplication(context, null)); + + GetApplicationHomeSubClusterRequest request = + GetApplicationHomeSubClusterRequest.newInstance(appId); + GetApplicationHomeSubClusterResponse response = + stateStore.getApplicationHomeSubCluster(request); + + Assert.assertNotNull(response); + ApplicationHomeSubCluster homeSubCluster = response.getApplicationHomeSubCluster(); + + DefaultRequestInterceptorREST interceptorREST = + interceptor.getInterceptorForSubCluster(homeSubCluster.getHomeSubCluster()); + + MockDefaultRequestInterceptorREST mockInterceptorREST = + (MockDefaultRequestInterceptorREST) interceptorREST; + mockInterceptorREST.updateApplicationState(YarnApplicationState.RUNNING, + appId.toString()); + + Set stateQueries = new HashSet<>(); + stateQueries.add(YarnApplicationState.RUNNING.name()); + + Set typeQueries = new HashSet<>(); + typeQueries.add("MapReduce"); + + ApplicationStatisticsInfo response2 = + interceptor.getAppStatistics(null, stateQueries, typeQueries); + + Assert.assertNotNull(response2); + Assert.assertFalse(response2.getStatItems().isEmpty()); + + StatisticsItemInfo result = response2.getStatItems().get(0); + Assert.assertEquals(1, result.getCount()); + Assert.assertEquals(YarnApplicationState.RUNNING, result.getState()); + Assert.assertEquals("MapReduce", result.getType()); + } + + @Test + public void testGetAppActivities() throws IOException, InterruptedException { + // Submit application to multiSubCluster + ApplicationId appId = ApplicationId.newInstance(Time.now(), 1); + ApplicationSubmissionContextInfo context = new ApplicationSubmissionContextInfo(); + context.setApplicationId(appId.toString()); + context.setApplicationType("MapReduce"); + context.setQueue("queue"); + + Assert.assertNotNull(interceptor.submitApplication(context, null)); + Set prioritiesSet = Collections.singleton("0"); + Set allocationRequestIdsSet = Collections.singleton("0"); + + AppActivitiesInfo appActivitiesInfo = + interceptor.getAppActivities(null, appId.toString(), String.valueOf(Time.now()), + prioritiesSet, allocationRequestIdsSet, null, "-1", null, false); + + Assert.assertNotNull(appActivitiesInfo); + Assert.assertEquals(appId.toString(), appActivitiesInfo.getApplicationId()); + Assert.assertEquals(10, appActivitiesInfo.getAllocations().size()); + } + + @Test + public void testListReservation() throws Exception { + + // Add ReservationId In stateStore + ReservationId reservationId = ReservationId.newInstance(Time.now(), 1); + SubClusterId homeSubClusterId = subClusters.get(0); + ReservationHomeSubCluster reservationHomeSubCluster = + ReservationHomeSubCluster.newInstance(reservationId, homeSubClusterId); + AddReservationHomeSubClusterRequest request = + AddReservationHomeSubClusterRequest.newInstance(reservationHomeSubCluster); + stateStore.addReservationHomeSubCluster(request); + + // Call the listReservation method + String applyReservationId = reservationId.toString(); + Response listReservationResponse = interceptor.listReservation( + QUEUE_DEDICATED_FULL, applyReservationId, -1, -1, false, null); + Assert.assertNotNull(listReservationResponse); + Assert.assertNotNull(listReservationResponse.getStatus()); + Status status = Status.fromStatusCode(listReservationResponse.getStatus()); + Assert.assertEquals(Status.OK, status); + + Object entity = listReservationResponse.getEntity(); + Assert.assertNotNull(entity); + Assert.assertNotNull(entity instanceof ReservationListInfo); + + ReservationListInfo listInfo = (ReservationListInfo) entity; + Assert.assertNotNull(listInfo); + + List reservationInfoList = listInfo.getReservations(); + Assert.assertNotNull(reservationInfoList); + Assert.assertEquals(1, reservationInfoList.size()); + + ReservationInfo reservationInfo = reservationInfoList.get(0); + Assert.assertNotNull(reservationInfo); + Assert.assertEquals(applyReservationId, reservationInfo.getReservationId()); + + ReservationDefinitionInfo definitionInfo = reservationInfo.getReservationDefinition(); + Assert.assertNotNull(definitionInfo); + + ReservationRequestsInfo reservationRequestsInfo = definitionInfo.getReservationRequests(); + Assert.assertNotNull(reservationRequestsInfo); + + ArrayList reservationRequestInfoList = + reservationRequestsInfo.getReservationRequest(); + Assert.assertNotNull(reservationRequestInfoList); + Assert.assertEquals(1, reservationRequestInfoList.size()); + + ReservationRequestInfo reservationRequestInfo = reservationRequestInfoList.get(0); + Assert.assertNotNull(reservationRequestInfo); + Assert.assertEquals(4, reservationRequestInfo.getNumContainers()); + + ResourceInfo resourceInfo = reservationRequestInfo.getCapability(); + Assert.assertNotNull(resourceInfo); + + int vCore = resourceInfo.getvCores(); + long memory = resourceInfo.getMemorySize(); + Assert.assertEquals(1, vCore); + Assert.assertEquals(1024, memory); + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/webapp/TestFederationInterceptorRESTRetry.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/webapp/TestFederationInterceptorRESTRetry.java index e563d5a58204e..b2f421e25ae30 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/webapp/TestFederationInterceptorRESTRetry.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/webapp/TestFederationInterceptorRESTRetry.java @@ -139,8 +139,8 @@ protected YarnConfiguration createConfiguration() { String mockPassThroughInterceptorClass = PassThroughClientRequestInterceptor.class.getName(); - // Create a request intercepter pipeline for testing. The last one in the - // chain is the federation intercepter that calls the mock resource manager. + // Create a request interceptor pipeline for testing. The last one in the + // chain is the federation interceptor that calls the mock resource manager. // The others in the chain will simply forward it to the next one in the // chain conf.set(YarnConfiguration.ROUTER_CLIENTRM_INTERCEPTOR_CLASS_PIPELINE, diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/webapp/TestRouterWebServiceUtil.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/webapp/TestRouterWebServiceUtil.java index edf3804d82eda..96a6881adc6d0 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/webapp/TestRouterWebServiceUtil.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/webapp/TestRouterWebServiceUtil.java @@ -30,12 +30,18 @@ import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.NodeInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.NodesInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ResourceRequestInfo; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppAttemptInfo; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ApplicationStatisticsInfo; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.StatisticsItemInfo; import org.apache.hadoop.yarn.server.uam.UnmanagedApplicationManager; import org.junit.Assert; import org.junit.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + /** * Test class to validate RouterWebServiceUtil methods. */ @@ -579,4 +585,97 @@ private void setUpClusterMetrics(ClusterMetricsInfo metrics, long seed) { metrics.setActiveNodes(rand.nextInt(1000)); metrics.setShutdownNodes(rand.nextInt(1000)); } + + public static AppAttemptInfo generateAppAttemptInfo(int attemptId) { + AppAttemptInfo appAttemptInfo = mock(AppAttemptInfo.class); + when(appAttemptInfo.getAppAttemptId()).thenReturn("AppAttemptId_" + attemptId); + when(appAttemptInfo.getAttemptId()).thenReturn(0); + when(appAttemptInfo.getFinishedTime()).thenReturn(1659621705L); + when(appAttemptInfo.getLogsLink()).thenReturn("LogLink_" + attemptId); + return appAttemptInfo; + } + + @Test + public void testMergeApplicationStatisticsInfo() { + ApplicationStatisticsInfo infoA = new ApplicationStatisticsInfo(); + ApplicationStatisticsInfo infoB = new ApplicationStatisticsInfo(); + + StatisticsItemInfo item1 = new StatisticsItemInfo(YarnApplicationState.ACCEPTED, "*", 10); + StatisticsItemInfo item2 = new StatisticsItemInfo(YarnApplicationState.ACCEPTED, "*", 20); + + infoA.add(item1); + infoB.add(item2); + + List lists = new ArrayList<>(); + lists.add(infoA); + lists.add(infoB); + + ApplicationStatisticsInfo mergeInfo = + RouterWebServiceUtil.mergeApplicationStatisticsInfo(lists); + ArrayList statItem = mergeInfo.getStatItems(); + + Assert.assertNotNull(statItem); + Assert.assertEquals(1, statItem.size()); + + StatisticsItemInfo first = statItem.get(0); + + Assert.assertEquals(item1.getCount() + item2.getCount(), first.getCount()); + Assert.assertEquals(item1.getType(), first.getType()); + Assert.assertEquals(item1.getState(), first.getState()); + } + + @Test + public void testMergeDiffApplicationStatisticsInfo() { + ApplicationStatisticsInfo infoA = new ApplicationStatisticsInfo(); + StatisticsItemInfo item1 = new StatisticsItemInfo(YarnApplicationState.ACCEPTED, "*", 10); + StatisticsItemInfo item2 = + new StatisticsItemInfo(YarnApplicationState.NEW_SAVING, "test1", 20); + infoA.add(item1); + infoA.add(item2); + + ApplicationStatisticsInfo infoB = new ApplicationStatisticsInfo(); + StatisticsItemInfo item3 = + new StatisticsItemInfo(YarnApplicationState.NEW_SAVING, "test1", 30); + StatisticsItemInfo item4 = new StatisticsItemInfo(YarnApplicationState.FINISHED, "test3", 40); + infoB.add(item3); + infoB.add(item4); + + List lists = new ArrayList<>(); + lists.add(infoA); + lists.add(infoB); + + ApplicationStatisticsInfo mergeInfo = + RouterWebServiceUtil.mergeApplicationStatisticsInfo(lists); + + Assert.assertEquals(3, mergeInfo.getStatItems().size()); + List mergeInfoStatItems = mergeInfo.getStatItems(); + + StatisticsItemInfo item1Result = null; + StatisticsItemInfo item2Result = null; + StatisticsItemInfo item3Result = null; + + for (StatisticsItemInfo item : mergeInfoStatItems) { + // ACCEPTED + if (item.getState() == YarnApplicationState.ACCEPTED) { + item1Result = item; + } + + // NEW_SAVING + if (item.getState() == YarnApplicationState.NEW_SAVING) { + item2Result = item; + } + + // FINISHED + if (item.getState() == YarnApplicationState.FINISHED) { + item3Result = item; + } + } + + Assert.assertEquals(YarnApplicationState.ACCEPTED, item1Result.getState()); + Assert.assertEquals(item1.getCount(), item1Result.getCount()); + Assert.assertEquals(YarnApplicationState.NEW_SAVING, item2Result.getState()); + Assert.assertEquals((item2.getCount() + item3.getCount()), item2Result.getCount()); + Assert.assertEquals(YarnApplicationState.FINISHED, item3Result.getState()); + Assert.assertEquals(item4.getCount(), item3Result.getCount()); + } } \ No newline at end of file diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/webapp/TestRouterWebServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/webapp/TestRouterWebServices.java index 7491cbc2a9fc7..5bf8db04b06cd 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/webapp/TestRouterWebServices.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/webapp/TestRouterWebServices.java @@ -200,6 +200,12 @@ public void testRouterWebServicesE2E() throws Exception { ContainerInfo containerInfo = getContainer(user); Assert.assertNotNull(containerInfo); + + Response response19 = updateSchedulerConfiguration(user); + Assert.assertNotNull(response19); + + Response response20 = getSchedulerConfiguration(user); + Assert.assertNotNull(response20); } /** diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/webapp/TestRouterWebServicesREST.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/webapp/TestRouterWebServicesREST.java index b345ebdd90202..1e6d331bdc31a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/webapp/TestRouterWebServicesREST.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/webapp/TestRouterWebServicesREST.java @@ -142,6 +142,8 @@ import com.sun.jersey.api.client.WebResource.Builder; import net.jcip.annotations.NotThreadSafe; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import javax.servlet.http.HttpServletRequest; @@ -157,6 +159,8 @@ public class TestRouterWebServicesREST { /** The number of concurrent submissions for multi-thread test. */ private static final int NUM_THREADS_TESTS = 100; + private static final Logger LOG = + LoggerFactory.getLogger(TestRouterWebServicesREST.class); private static String userName = "test"; @@ -196,7 +200,7 @@ public Boolean get() { } return false; } - }, 1000, 10 * 1000); + }, 1000, 20 * 1000); } catch (Exception e) { fail("Web app not running"); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/resources/capacity-scheduler.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/resources/capacity-scheduler.xml index ffe1a5d4ca48c..a8487f3f3c00a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/resources/capacity-scheduler.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/resources/capacity-scheduler.xml @@ -45,7 +45,7 @@ yarn.scheduler.capacity.root.queues - default,decided + default,decided,target The queues at the this level (root is the root queue). @@ -53,10 +53,22 @@ yarn.scheduler.capacity.root.default.capacity - 100 + 50 Default queue target capacity. + + yarn.scheduler.capacity.root.target.capacity + 10 + target queue capacity. + + + + yarn.scheduler.capacity.root.decided.capacity + 40 + decided queue capacity. + + yarn.scheduler.capacity.root.default.user-limit-factor 1 @@ -81,6 +93,14 @@ + + yarn.scheduler.capacity.root.target.state + RUNNING + + The state of the target queue. State can be one of RUNNING or STOPPED. + + + yarn.scheduler.capacity.root.default.acl_submit_applications * @@ -89,6 +109,14 @@ + + yarn.scheduler.capacity.root.target.acl_submit_applications + * + + The ACL of who can submit jobs to the target queue. + + + yarn.scheduler.capacity.root.default.acl_administer_queue * @@ -97,6 +125,14 @@ + + yarn.scheduler.capacity.root.target.acl_administer_queue + * + + The ACL of who can administer jobs on the target queue. + + + yarn.scheduler.capacity.root.decided.reservable true diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/resources/profiles/sample-profiles-1.json b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/resources/profiles/sample-profiles-1.json new file mode 100644 index 0000000000000..8485ab6f3d550 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/resources/profiles/sample-profiles-1.json @@ -0,0 +1,24 @@ +{ + "___asflicense__": [ + "", + "Licensed to the Apache Software Foundation (ASF) under one", + "or more contributor license agreements. See the NOTICE file", + "distributed with this work for additional information", + "regarding copyright ownership. The ASF licenses this file", + "to you under the Apache License, Version 2.0 (the", + "\"License\"); you may not use this file except in compliance", + "with the License. You may obtain a copy of the License at", + "", + " http://www.apache.org/licenses/LICENSE-2.0", + "", + "Unless required by applicable law or agreed to in writing, software", + "distributed under the License is distributed on an \"AS IS\" BASIS,", + "WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.", + "See the License for the specific language governing permissions and", + "limitations under the License." + ], + "default" : { + "memory-mb" : 2048, + "vcores" : 2 + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/resources/yarn-site.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/resources/yarn-site.xml index 310a1612486bf..84d4171c79cd4 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/resources/yarn-site.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/resources/yarn-site.xml @@ -31,4 +31,12 @@ yarn.cluster.max-application-priority 50 + + yarn.resourcemanager.resource-profiles.enabled + true + + + yarn.resourcemanager.resource-profiles.source-file + profiles/sample-profiles-1.json + diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-sharedcachemanager/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-sharedcachemanager/pom.xml index fd429a5f93e67..971fb0941a3ba 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-sharedcachemanager/pom.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-sharedcachemanager/pom.xml @@ -49,11 +49,6 @@ org.apache.hadoop hadoop-yarn-client - - junit - junit - test - org.mockito mockito-core @@ -71,6 +66,21 @@ test test-jar + + org.junit.jupiter + junit-jupiter-api + test + + + org.junit.jupiter + junit-jupiter-engine + test + + + org.junit.platform + junit-platform-launcher + test + diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-sharedcachemanager/src/test/java/org/apache/hadoop/yarn/server/sharedcachemanager/TestCleanerTask.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-sharedcachemanager/src/test/java/org/apache/hadoop/yarn/server/sharedcachemanager/TestCleanerTask.java index 23c80d9655dab..019ef67bd6a8f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-sharedcachemanager/src/test/java/org/apache/hadoop/yarn/server/sharedcachemanager/TestCleanerTask.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-sharedcachemanager/src/test/java/org/apache/hadoop/yarn/server/sharedcachemanager/TestCleanerTask.java @@ -18,25 +18,26 @@ package org.apache.hadoop.yarn.server.sharedcachemanager; -import static org.mockito.ArgumentMatchers.anyBoolean; -import static org.mockito.ArgumentMatchers.eq; -import static org.mockito.ArgumentMatchers.isA; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.never; -import static org.mockito.Mockito.spy; -import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.when; - import java.util.concurrent.locks.Lock; import java.util.concurrent.locks.ReentrantLock; +import org.junit.jupiter.api.Test; + import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.server.sharedcachemanager.metrics.CleanerMetrics; import org.apache.hadoop.yarn.server.sharedcachemanager.store.SCMStore; -import org.junit.Test; + +import static org.mockito.ArgumentMatchers.anyBoolean; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.ArgumentMatchers.isA; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; public class TestCleanerTask { private static final String ROOT = @@ -47,7 +48,7 @@ public class TestCleanerTask { YarnConfiguration.DEFAULT_SHARED_CACHE_NESTED_LEVEL; @Test - public void testNonExistentRoot() throws Exception { + void testNonExistentRoot() throws Exception { FileSystem fs = mock(FileSystem.class); CleanerMetrics metrics = mock(CleanerMetrics.class); SCMStore store = mock(SCMStore.class); @@ -64,7 +65,7 @@ public void testNonExistentRoot() throws Exception { } @Test - public void testProcessFreshResource() throws Exception { + void testProcessFreshResource() throws Exception { FileSystem fs = mock(FileSystem.class); CleanerMetrics metrics = mock(CleanerMetrics.class); SCMStore store = mock(SCMStore.class); @@ -89,7 +90,7 @@ public void testProcessFreshResource() throws Exception { } @Test - public void testProcessEvictableResource() throws Exception { + void testProcessEvictableResource() throws Exception { FileSystem fs = mock(FileSystem.class); CleanerMetrics metrics = mock(CleanerMetrics.class); SCMStore store = mock(SCMStore.class); @@ -125,7 +126,7 @@ private CleanerTask createSpiedTask(FileSystem fs, SCMStore store, } @Test - public void testResourceIsInUseHasAnActiveApp() throws Exception { + void testResourceIsInUseHasAnActiveApp() throws Exception { FileSystem fs = mock(FileSystem.class); CleanerMetrics metrics = mock(CleanerMetrics.class); SCMStore store = mock(SCMStore.class); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-sharedcachemanager/src/test/java/org/apache/hadoop/yarn/server/sharedcachemanager/TestClientSCMProtocolService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-sharedcachemanager/src/test/java/org/apache/hadoop/yarn/server/sharedcachemanager/TestClientSCMProtocolService.java index ca4bdce7cf19d..314332798ab97 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-sharedcachemanager/src/test/java/org/apache/hadoop/yarn/server/sharedcachemanager/TestClientSCMProtocolService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-sharedcachemanager/src/test/java/org/apache/hadoop/yarn/server/sharedcachemanager/TestClientSCMProtocolService.java @@ -18,15 +18,16 @@ package org.apache.hadoop.yarn.server.sharedcachemanager; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertNull; -import static org.mockito.Mockito.spy; - import java.io.File; import java.io.IOException; import java.net.InetSocketAddress; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.ipc.RPC; import org.apache.hadoop.security.UserGroupInformation; @@ -42,11 +43,11 @@ import org.apache.hadoop.yarn.server.sharedcachemanager.store.InMemorySCMStore; import org.apache.hadoop.yarn.server.sharedcachemanager.store.SCMStore; import org.apache.hadoop.yarn.server.sharedcachemanager.store.SharedCacheResourceReference; -import org.junit.After; -import org.junit.AfterClass; -import org.junit.Before; -import org.junit.BeforeClass; -import org.junit.Test; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.mockito.Mockito.spy; /** @@ -55,7 +56,7 @@ public class TestClientSCMProtocolService { private static File testDir = null; - @BeforeClass + @BeforeAll public static void setupTestDirs() throws IOException { testDir = new File("target", TestSharedCacheUploaderService.class.getCanonicalName()); @@ -64,7 +65,7 @@ public static void setupTestDirs() throws IOException { testDir = testDir.getAbsoluteFile(); } - @AfterClass + @AfterAll public static void cleanupTestDirs() throws IOException { if (testDir != null) { testDir.delete(); @@ -78,7 +79,7 @@ public static void cleanupTestDirs() throws IOException { private final RecordFactory recordFactory = RecordFactoryProvider .getRecordFactory(null); - @Before + @BeforeEach public void startUp() { Configuration conf = new Configuration(); conf.set(YarnConfiguration.SCM_STORE_CLASS, @@ -105,7 +106,7 @@ public void startUp() { conf); } - @After + @AfterEach public void cleanUp() { if (store != null) { store.stop(); @@ -124,19 +125,19 @@ public void cleanUp() { } @Test - public void testUse_MissingEntry() throws Exception { + void testUse_MissingEntry() throws Exception { long misses = ClientSCMMetrics.getInstance().getCacheMisses(); UseSharedCacheResourceRequest request = recordFactory.newRecordInstance(UseSharedCacheResourceRequest.class); request.setResourceKey("key1"); request.setAppId(createAppId(1, 1L)); assertNull(clientSCMProxy.use(request).getPath()); - assertEquals("Client SCM metrics aren't updated.", 1, ClientSCMMetrics - .getInstance().getCacheMisses() - misses); + assertEquals(1, ClientSCMMetrics + .getInstance().getCacheMisses() - misses, "Client SCM metrics aren't updated."); } @Test - public void testUse_ExistingEntry_NoAppIds() throws Exception { + void testUse_ExistingEntry_NoAppIds() throws Exception { // Pre-populate the SCM with one cache entry store.addResource("key1", "foo.jar"); @@ -150,13 +151,13 @@ public void testUse_ExistingEntry_NoAppIds() throws Exception { String expectedPath = testDir.getAbsolutePath() + "/k/e/y/key1/foo.jar"; assertEquals(expectedPath, clientSCMProxy.use(request).getPath()); assertEquals(1, store.getResourceReferences("key1").size()); - assertEquals("Client SCM metrics aren't updated.", 1, ClientSCMMetrics - .getInstance().getCacheHits() - hits); + assertEquals(1, ClientSCMMetrics + .getInstance().getCacheHits() - hits, "Client SCM metrics aren't updated."); } @Test - public void testUse_ExistingEntry_OneId() throws Exception { + void testUse_ExistingEntry_OneId() throws Exception { // Pre-populate the SCM with one cache entry store.addResource("key1", "foo.jar"); store.addResourceReference("key1", @@ -174,12 +175,12 @@ public void testUse_ExistingEntry_OneId() throws Exception { String expectedPath = testDir.getAbsolutePath() + "/k/e/y/key1/foo.jar"; assertEquals(expectedPath, clientSCMProxy.use(request).getPath()); assertEquals(2, store.getResourceReferences("key1").size()); - assertEquals("Client SCM metrics aren't updated.", 1, ClientSCMMetrics - .getInstance().getCacheHits() - hits); + assertEquals(1, ClientSCMMetrics + .getInstance().getCacheHits() - hits, "Client SCM metrics aren't updated."); } @Test - public void testUse_ExistingEntry_DupId() throws Exception { + void testUse_ExistingEntry_DupId() throws Exception { // Pre-populate the SCM with one cache entry store.addResource("key1", "foo.jar"); UserGroupInformation testUGI = UserGroupInformation.getCurrentUser(); @@ -201,12 +202,12 @@ public void testUse_ExistingEntry_DupId() throws Exception { assertEquals(expectedPath, clientSCMProxy.use(request).getPath()); assertEquals(1, store.getResourceReferences("key1").size()); - assertEquals("Client SCM metrics aren't updated.", 1, ClientSCMMetrics - .getInstance().getCacheHits() - hits); + assertEquals(1, ClientSCMMetrics + .getInstance().getCacheHits() - hits, "Client SCM metrics aren't updated."); } @Test - public void testRelease_ExistingEntry_NonExistantAppId() throws Exception { + void testRelease_ExistingEntry_NonExistantAppId() throws Exception { // Pre-populate the SCM with one cache entry store.addResource("key1", "foo.jar"); store.addResourceReference("key1", @@ -224,13 +225,14 @@ public void testRelease_ExistingEntry_NonExistantAppId() throws Exception { assertEquals(1, store.getResourceReferences("key1").size()); assertEquals( - "Client SCM metrics were updated when a release did not happen", 0, - ClientSCMMetrics.getInstance().getCacheReleases() - releases); + 0, + ClientSCMMetrics.getInstance().getCacheReleases() - releases, + "Client SCM metrics were updated when a release did not happen"); } @Test - public void testRelease_ExistingEntry_WithAppId() throws Exception { + void testRelease_ExistingEntry_WithAppId() throws Exception { // Pre-populate the SCM with one cache entry store.addResource("key1", "foo.jar"); UserGroupInformation testUGI = UserGroupInformation.getCurrentUser(); @@ -249,13 +251,13 @@ public void testRelease_ExistingEntry_WithAppId() throws Exception { clientSCMProxy.release(request); assertEquals(0, store.getResourceReferences("key1").size()); - assertEquals("Client SCM metrics aren't updated.", 1, ClientSCMMetrics - .getInstance().getCacheReleases() - releases); + assertEquals(1, ClientSCMMetrics + .getInstance().getCacheReleases() - releases, "Client SCM metrics aren't updated."); } @Test - public void testRelease_MissingEntry() throws Exception { + void testRelease_MissingEntry() throws Exception { long releases = ClientSCMMetrics.getInstance().getCacheReleases(); @@ -268,8 +270,9 @@ public void testRelease_MissingEntry() throws Exception { assertNotNull(store.getResourceReferences("key2")); assertEquals(0, store.getResourceReferences("key2").size()); assertEquals( - "Client SCM metrics were updated when a release did not happen.", 0, - ClientSCMMetrics.getInstance().getCacheReleases() - releases); + 0, + ClientSCMMetrics.getInstance().getCacheReleases() - releases, + "Client SCM metrics were updated when a release did not happen."); } private ApplicationId createAppId(int id, long timestamp) { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-sharedcachemanager/src/test/java/org/apache/hadoop/yarn/server/sharedcachemanager/TestRemoteAppChecker.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-sharedcachemanager/src/test/java/org/apache/hadoop/yarn/server/sharedcachemanager/TestRemoteAppChecker.java index 46546444ad568..29a4291dac67f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-sharedcachemanager/src/test/java/org/apache/hadoop/yarn/server/sharedcachemanager/TestRemoteAppChecker.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-sharedcachemanager/src/test/java/org/apache/hadoop/yarn/server/sharedcachemanager/TestRemoteAppChecker.java @@ -18,11 +18,8 @@ package org.apache.hadoop.yarn.server.sharedcachemanager; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; -import static org.mockito.Mockito.doReturn; -import static org.mockito.Mockito.doThrow; -import static org.mockito.Mockito.spy; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Test; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.yarn.api.records.ApplicationId; @@ -32,14 +29,18 @@ import org.apache.hadoop.yarn.client.api.YarnClient; import org.apache.hadoop.yarn.client.api.impl.YarnClientImpl; import org.apache.hadoop.yarn.exceptions.ApplicationNotFoundException; -import org.junit.After; -import org.junit.Test; + +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.spy; public class TestRemoteAppChecker { private RemoteAppChecker checker; - @After + @AfterEach public void cleanup() { if (checker != null) { checker.stop(); @@ -61,7 +62,7 @@ private YarnClient createCheckerWithMockedClient() { } @Test - public void testNonExistentApp() throws Exception { + void testNonExistentApp() throws Exception { YarnClient client = createCheckerWithMockedClient(); ApplicationId id = ApplicationId.newInstance(1, 1); @@ -76,7 +77,7 @@ public void testNonExistentApp() throws Exception { } @Test - public void testRunningApp() throws Exception { + void testRunningApp() throws Exception { YarnClient client = createCheckerWithMockedClient(); ApplicationId id = ApplicationId.newInstance(1, 1); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-sharedcachemanager/src/test/java/org/apache/hadoop/yarn/server/sharedcachemanager/TestSCMAdminProtocolService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-sharedcachemanager/src/test/java/org/apache/hadoop/yarn/server/sharedcachemanager/TestSCMAdminProtocolService.java index e183ffa5cfba1..9c55452a7f7ea 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-sharedcachemanager/src/test/java/org/apache/hadoop/yarn/server/sharedcachemanager/TestSCMAdminProtocolService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-sharedcachemanager/src/test/java/org/apache/hadoop/yarn/server/sharedcachemanager/TestSCMAdminProtocolService.java @@ -18,36 +18,37 @@ package org.apache.hadoop.yarn.server.sharedcachemanager; -import static org.junit.Assert.assertEquals; -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.ArgumentMatchers.isA; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.spy; -import static org.mockito.Mockito.times; -import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.doNothing; -import static org.mockito.Mockito.when; - import java.io.IOException; import java.net.InetSocketAddress; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.ipc.RPC; -import org.apache.hadoop.yarn.server.api.SCMAdminProtocol; -import org.apache.hadoop.yarn.server.api.protocolrecords.RunSharedCacheCleanerTaskRequest; -import org.apache.hadoop.yarn.server.api.protocolrecords.RunSharedCacheCleanerTaskResponse; -import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.RunSharedCacheCleanerTaskResponsePBImpl; import org.apache.hadoop.yarn.client.SCMAdmin; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.factories.RecordFactory; import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider; import org.apache.hadoop.yarn.ipc.YarnRPC; +import org.apache.hadoop.yarn.server.api.SCMAdminProtocol; +import org.apache.hadoop.yarn.server.api.protocolrecords.RunSharedCacheCleanerTaskRequest; +import org.apache.hadoop.yarn.server.api.protocolrecords.RunSharedCacheCleanerTaskResponse; +import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.RunSharedCacheCleanerTaskResponsePBImpl; import org.apache.hadoop.yarn.server.sharedcachemanager.store.InMemorySCMStore; import org.apache.hadoop.yarn.server.sharedcachemanager.store.SCMStore; -import org.junit.After; -import org.junit.Assert; -import org.junit.Before; -import org.junit.Test; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.isA; +import static org.mockito.Mockito.doNothing; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; /** * Basic unit tests for the SCM Admin Protocol Service and SCMAdmin. @@ -63,7 +64,7 @@ public class TestSCMAdminProtocolService { private final RecordFactory recordFactory = RecordFactoryProvider .getRecordFactory(null); - @Before + @BeforeEach public void startUp() { Configuration conf = new Configuration(); conf.set(YarnConfiguration.SCM_STORE_CLASS, @@ -95,7 +96,7 @@ protected SCMAdminProtocol createSCMAdminProtocol() throws IOException { }; } - @After + @AfterEach public void cleanUpTest() { if (service != null) { service.stop(); @@ -107,18 +108,18 @@ public void cleanUpTest() { } @Test - public void testRunCleanerTask() throws Exception { + void testRunCleanerTask() throws Exception { doNothing().when(cleaner).runCleanerTask(); RunSharedCacheCleanerTaskRequest request = recordFactory.newRecordInstance(RunSharedCacheCleanerTaskRequest.class); RunSharedCacheCleanerTaskResponse response = SCMAdminProxy.runCleanerTask(request); - Assert.assertTrue("cleaner task request isn't accepted", response.getAccepted()); + assertTrue(response.getAccepted(), "cleaner task request isn't accepted"); verify(service, times(1)).runCleanerTask(any(RunSharedCacheCleanerTaskRequest.class)); } @Test - public void testRunCleanerTaskCLI() throws Exception { - String[] args = { "-runCleanerTask" }; + void testRunCleanerTaskCLI() throws Exception { + String[] args = {"-runCleanerTask"}; RunSharedCacheCleanerTaskResponse rp = new RunSharedCacheCleanerTaskResponsePBImpl(); rp.setAccepted(true); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-sharedcachemanager/src/test/java/org/apache/hadoop/yarn/server/sharedcachemanager/TestSharedCacheUploaderService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-sharedcachemanager/src/test/java/org/apache/hadoop/yarn/server/sharedcachemanager/TestSharedCacheUploaderService.java index 048523e8457d4..38833680f697e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-sharedcachemanager/src/test/java/org/apache/hadoop/yarn/server/sharedcachemanager/TestSharedCacheUploaderService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-sharedcachemanager/src/test/java/org/apache/hadoop/yarn/server/sharedcachemanager/TestSharedCacheUploaderService.java @@ -18,17 +18,17 @@ package org.apache.hadoop.yarn.server.sharedcachemanager; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertTrue; -import static org.mockito.Mockito.spy; - import java.io.File; import java.io.IOException; import java.net.InetSocketAddress; import java.util.Collection; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.ipc.RPC; import org.apache.hadoop.yarn.conf.YarnConfiguration; @@ -41,11 +41,12 @@ import org.apache.hadoop.yarn.server.sharedcachemanager.store.InMemorySCMStore; import org.apache.hadoop.yarn.server.sharedcachemanager.store.SCMStore; import org.apache.hadoop.yarn.server.sharedcachemanager.store.SharedCacheResourceReference; -import org.junit.After; -import org.junit.AfterClass; -import org.junit.Before; -import org.junit.BeforeClass; -import org.junit.Test; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.Mockito.spy; /** @@ -54,7 +55,7 @@ public class TestSharedCacheUploaderService { private static File testDir = null; - @BeforeClass + @BeforeAll public static void setupTestDirs() throws IOException { testDir = new File("target", TestSharedCacheUploaderService.class.getCanonicalName()); @@ -63,7 +64,7 @@ public static void setupTestDirs() throws IOException { testDir = testDir.getAbsoluteFile(); } - @AfterClass + @AfterAll public static void cleanupTestDirs() throws IOException { if (testDir != null) { testDir.delete(); @@ -76,7 +77,7 @@ public static void cleanupTestDirs() throws IOException { private final RecordFactory recordFactory = RecordFactoryProvider .getRecordFactory(null); - @Before + @BeforeEach public void startUp() { Configuration conf = new Configuration(); conf.set(YarnConfiguration.SCM_STORE_CLASS, @@ -103,7 +104,7 @@ public void startUp() { SCMUploaderProtocol.class, scmAddress, conf); } - @After + @AfterEach public void cleanUp() { if (store != null) { store.stop(); @@ -119,7 +120,7 @@ public void cleanUp() { } @Test - public void testNotify_noEntry() throws Exception { + void testNotify_noEntry() throws Exception { long accepted = SharedCacheUploaderMetrics.getInstance().getAcceptedUploads(); @@ -134,14 +135,15 @@ public void testNotify_noEntry() throws Exception { assertEquals(0, set.size()); assertEquals( - "NM upload metrics aren't updated.", 1, + 1, SharedCacheUploaderMetrics.getInstance().getAcceptedUploads() - - accepted); + accepted, + "NM upload metrics aren't updated."); } @Test - public void testNotify_entryExists_differentName() throws Exception { + void testNotify_entryExists_differentName() throws Exception { long rejected = SharedCacheUploaderMetrics.getInstance().getRejectUploads(); @@ -157,14 +159,15 @@ public void testNotify_entryExists_differentName() throws Exception { assertNotNull(set); assertEquals(0, set.size()); assertEquals( - "NM upload metrics aren't updated.", 1, + 1, SharedCacheUploaderMetrics.getInstance().getRejectUploads() - - rejected); + rejected, + "NM upload metrics aren't updated."); } @Test - public void testNotify_entryExists_sameName() throws Exception { + void testNotify_entryExists_sameName() throws Exception { long accepted = SharedCacheUploaderMetrics.getInstance().getAcceptedUploads(); @@ -180,9 +183,10 @@ public void testNotify_entryExists_sameName() throws Exception { assertNotNull(set); assertEquals(0, set.size()); assertEquals( - "NM upload metrics aren't updated.", 1, + 1, SharedCacheUploaderMetrics.getInstance().getAcceptedUploads() - - accepted); + accepted, + "NM upload metrics aren't updated."); } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-sharedcachemanager/src/test/java/org/apache/hadoop/yarn/server/sharedcachemanager/metrics/TestCleanerMetrics.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-sharedcachemanager/src/test/java/org/apache/hadoop/yarn/server/sharedcachemanager/metrics/TestCleanerMetrics.java index cca3f4286b321..6829d9a6347de 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-sharedcachemanager/src/test/java/org/apache/hadoop/yarn/server/sharedcachemanager/metrics/TestCleanerMetrics.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-sharedcachemanager/src/test/java/org/apache/hadoop/yarn/server/sharedcachemanager/metrics/TestCleanerMetrics.java @@ -17,24 +17,25 @@ */ package org.apache.hadoop.yarn.server.sharedcachemanager.metrics; -import static org.junit.Assert.assertEquals; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; import org.apache.hadoop.conf.Configuration; -import org.junit.Before; -import org.junit.Test; + +import static org.junit.jupiter.api.Assertions.assertEquals; public class TestCleanerMetrics { Configuration conf = new Configuration(); CleanerMetrics cleanerMetrics; - @Before + @BeforeEach public void init() { cleanerMetrics = CleanerMetrics.getInstance(); } @Test - public void testMetricsOverMultiplePeriods() { + void testMetricsOverMultiplePeriods() { simulateACleanerRun(); assertMetrics(4, 4, 1, 1); simulateACleanerRun(); @@ -51,14 +52,14 @@ public void simulateACleanerRun() { void assertMetrics(int proc, int totalProc, int del, int totalDel) { assertEquals( - "Processed files in the last period are not measured correctly", proc, - cleanerMetrics.getProcessedFiles()); - assertEquals("Total processed files are not measured correctly", - totalProc, cleanerMetrics.getTotalProcessedFiles()); - assertEquals( - "Deleted files in the last period are not measured correctly", del, - cleanerMetrics.getDeletedFiles()); - assertEquals("Total deleted files are not measured correctly", - totalDel, cleanerMetrics.getTotalDeletedFiles()); + proc, + cleanerMetrics.getProcessedFiles(), + "Processed files in the last period are not measured correctly"); + assertEquals(totalProc, cleanerMetrics.getTotalProcessedFiles(), + "Total processed files are not measured correctly"); + assertEquals(del, cleanerMetrics.getDeletedFiles(), + "Deleted files in the last period are not measured correctly"); + assertEquals(totalDel, cleanerMetrics.getTotalDeletedFiles(), + "Total deleted files are not measured correctly"); } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-sharedcachemanager/src/test/java/org/apache/hadoop/yarn/server/sharedcachemanager/store/SCMStoreBaseTest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-sharedcachemanager/src/test/java/org/apache/hadoop/yarn/server/sharedcachemanager/store/SCMStoreBaseTest.java index 4e960b29c779d..2374bf3fdbe06 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-sharedcachemanager/src/test/java/org/apache/hadoop/yarn/server/sharedcachemanager/store/SCMStoreBaseTest.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-sharedcachemanager/src/test/java/org/apache/hadoop/yarn/server/sharedcachemanager/store/SCMStoreBaseTest.java @@ -18,9 +18,10 @@ package org.apache.hadoop.yarn.server.sharedcachemanager.store; +import org.junit.jupiter.api.Test; + import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.util.ReflectionUtils; -import org.junit.Test; /** * All test classes that test an SCMStore implementation must extend this class. @@ -33,7 +34,7 @@ public abstract class SCMStoreBaseTest { abstract Class getStoreClass(); @Test - public void TestZeroArgConstructor() throws Exception { + void TestZeroArgConstructor() throws Exception { // Test that the SCMStore implementation class is compatible with // ReflectionUtils#newInstance ReflectionUtils.newInstance(getStoreClass(), new Configuration()); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-sharedcachemanager/src/test/java/org/apache/hadoop/yarn/server/sharedcachemanager/store/TestInMemorySCMStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-sharedcachemanager/src/test/java/org/apache/hadoop/yarn/server/sharedcachemanager/store/TestInMemorySCMStore.java index 35adbdd34e4bc..5cb5332be4135 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-sharedcachemanager/src/test/java/org/apache/hadoop/yarn/server/sharedcachemanager/store/TestInMemorySCMStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-sharedcachemanager/src/test/java/org/apache/hadoop/yarn/server/sharedcachemanager/store/TestInMemorySCMStore.java @@ -18,16 +18,6 @@ package org.apache.hadoop.yarn.server.sharedcachemanager.store; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertNull; -import static org.junit.Assert.assertSame; -import static org.junit.Assert.assertTrue; -import static org.mockito.ArgumentMatchers.isA; -import static org.mockito.Mockito.doReturn; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.spy; - import java.util.ArrayList; import java.util.Collection; import java.util.Collections; @@ -41,6 +31,10 @@ import java.util.concurrent.ExecutorService; import java.util.concurrent.Future; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; @@ -48,9 +42,16 @@ import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.server.sharedcachemanager.AppChecker; import org.apache.hadoop.yarn.server.sharedcachemanager.DummyAppChecker; -import org.junit.After; -import org.junit.Before; -import org.junit.Test; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertSame; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.ArgumentMatchers.isA; +import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.spy; public class TestInMemorySCMStore extends SCMStoreBaseTest { @@ -62,13 +63,13 @@ Class getStoreClass() { return InMemorySCMStore.class; } - @Before + @BeforeEach public void setup() { this.checker = spy(new DummyAppChecker()); this.store = spy(new InMemorySCMStore(checker)); } - @After + @AfterEach public void cleanup() { if (this.store != null) { this.store.stop(); @@ -117,7 +118,7 @@ private void startStoreWithApps() throws Exception { } @Test - public void testAddResourceConcurrency() throws Exception { + void testAddResourceConcurrency() throws Exception { startEmptyStore(); final String key = "key1"; int count = 5; @@ -140,7 +141,7 @@ public String call() throws Exception { start.countDown(); // check the result; they should all agree with the value Set results = new HashSet(); - for (Future future: futures) { + for (Future future : futures) { results.add(future.get()); } assertSame(1, results.size()); @@ -148,7 +149,7 @@ public String call() throws Exception { } @Test - public void testAddResourceRefNonExistentResource() throws Exception { + void testAddResourceRefNonExistentResource() throws Exception { startEmptyStore(); String key = "key1"; ApplicationId id = createAppId(1, 1L); @@ -158,7 +159,7 @@ public void testAddResourceRefNonExistentResource() throws Exception { } @Test - public void testRemoveResourceEmptyRefs() throws Exception { + void testRemoveResourceEmptyRefs() throws Exception { startEmptyStore(); String key = "key1"; String fileName = "foo.jar"; @@ -169,7 +170,7 @@ public void testRemoveResourceEmptyRefs() throws Exception { } @Test - public void testAddResourceRefRemoveResource() throws Exception { + void testAddResourceRefRemoveResource() throws Exception { startEmptyStore(); String key = "key1"; ApplicationId id = createAppId(1, 1L); @@ -186,7 +187,7 @@ public void testAddResourceRefRemoveResource() throws Exception { } @Test - public void testAddResourceRefConcurrency() throws Exception { + void testAddResourceRefConcurrency() throws Exception { startEmptyStore(); final String key = "key1"; final String user = "user"; @@ -215,7 +216,7 @@ public String call() throws Exception { start.countDown(); // check the result Set results = new HashSet(); - for (Future future: futures) { + for (Future future : futures) { results.add(future.get()); } // they should all have the same file name @@ -228,7 +229,7 @@ public String call() throws Exception { } @Test - public void testAddResourceRefAddResourceConcurrency() throws Exception { + void testAddResourceRefAddResourceConcurrency() throws Exception { startEmptyStore(); final String key = "key1"; final String fileName = "foo.jar"; @@ -265,7 +266,7 @@ public String call() throws Exception { } @Test - public void testRemoveRef() throws Exception { + void testRemoveRef() throws Exception { startEmptyStore(); String key = "key1"; String fileName = "foo.jar"; @@ -287,7 +288,7 @@ public void testRemoveRef() throws Exception { } @Test - public void testBootstrapping() throws Exception { + void testBootstrapping() throws Exception { Map initialCachedResources = startStoreWithResources(); int count = initialCachedResources.size(); ApplicationId id = createAppId(1, 1L); @@ -306,7 +307,7 @@ public void testBootstrapping() throws Exception { } @Test - public void testEvictableWithInitialApps() throws Exception { + void testEvictableWithInitialApps() throws Exception { startStoreWithApps(); assertFalse(store.isResourceEvictable("key", mock(FileStatus.class))); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/pom.xml index f6f56f818a236..2de2c13f16b51 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/pom.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/pom.xml @@ -132,10 +132,6 @@ test test-jar - - javax.ws.rs - javax.ws.rs-api - diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timeline-pluginstorage/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timeline-pluginstorage/pom.xml index 0954c566427c4..1b80d25830796 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timeline-pluginstorage/pom.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timeline-pluginstorage/pom.xml @@ -71,7 +71,7 @@ test-jar test - + org.apache.hadoop hadoop-yarn-common @@ -130,12 +130,28 @@ test - com.fasterxml.jackson.core - jackson-databind + org.junit.jupiter + junit-jupiter-api + test - javax.ws.rs - javax.ws.rs-api + org.junit.jupiter + junit-jupiter-engine + test + + + org.junit.jupiter + junit-jupiter-params + test + + + org.junit.platform + junit-platform-launcher + test + + + com.fasterxml.jackson.core + jackson-databind diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timeline-pluginstorage/src/test/java/org/apache/hadoop/yarn/server/timeline/PluginStoreTestUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timeline-pluginstorage/src/test/java/org/apache/hadoop/yarn/server/timeline/PluginStoreTestUtils.java index 4609cb0b91db4..cb887fe264fab 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timeline-pluginstorage/src/test/java/org/apache/hadoop/yarn/server/timeline/PluginStoreTestUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timeline-pluginstorage/src/test/java/org/apache/hadoop/yarn/server/timeline/PluginStoreTestUtils.java @@ -49,8 +49,8 @@ import java.util.Map; import java.util.Set; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; /** * Utility methods related to the ATS v1.5 plugin storage tests. @@ -176,10 +176,8 @@ static void verifyTestEntities(TimelineDataManager tdm) UserGroupInformation.getLoginUser()); assertNotNull(entity1); assertNotNull(entity2); - assertEquals("Failed to read out entity 1", - (Long) 123l, entity1.getStartTime()); - assertEquals("Failed to read out entity 2", - (Long) 456l, entity2.getStartTime()); + assertEquals((Long) 123l, entity1.getStartTime(), "Failed to read out entity 1"); + assertEquals((Long) 456l, entity2.getStartTime(), "Failed to read out entity 2"); } /** diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timeline-pluginstorage/src/test/java/org/apache/hadoop/yarn/server/timeline/TestEntityGroupFSTimelineStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timeline-pluginstorage/src/test/java/org/apache/hadoop/yarn/server/timeline/TestEntityGroupFSTimelineStore.java index d421d35ce28f3..748cf92747ecc 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timeline-pluginstorage/src/test/java/org/apache/hadoop/yarn/server/timeline/TestEntityGroupFSTimelineStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timeline-pluginstorage/src/test/java/org/apache/hadoop/yarn/server/timeline/TestEntityGroupFSTimelineStore.java @@ -43,14 +43,12 @@ import org.apache.hadoop.yarn.server.timeline.EntityGroupFSTimelineStore.AppState; import org.apache.hadoop.yarn.server.timeline.TimelineReader.Field; import org.apache.hadoop.yarn.util.ConverterUtils; -import org.junit.After; -import org.junit.Assert; -import org.junit.AfterClass; -import org.junit.Before; -import org.junit.BeforeClass; -import org.junit.Rule; -import org.junit.Test; -import org.junit.rules.TestName; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestInfo; import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.core.JsonFactory; @@ -78,11 +76,12 @@ import java.util.function.Supplier; import static org.assertj.core.api.Assertions.assertThat; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertNotEquals; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertTrue; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.fail; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; @@ -118,12 +117,10 @@ public class TestEntityGroupFSTimelineStore extends TimelineStoreTestUtils { private EntityGroupFSTimelineStore store; private TimelineEntity entityNew; - @Rule - public TestName currTestName = new TestName(); private File rootDir; private File testJar; - @BeforeClass + @BeforeAll public static void setupClass() throws Exception { config.setBoolean(YarnConfiguration.TIMELINE_SERVICE_TTL_ENABLE, false); config.set( @@ -163,8 +160,8 @@ public static void setupClass() throws Exception { testActiveDirPath.toString()); } - @Before - public void setup() throws Exception { + @BeforeEach + public void setup(TestInfo testInfo) throws Exception { for (ApplicationId appId : sampleAppIds) { Path attemotDirPath = new Path(new Path(testActiveDirPath, appId.toString()), @@ -173,7 +170,7 @@ public void setup() throws Exception { } store = new EntityGroupFSTimelineStore(); - if (currTestName.getMethodName().contains("Plugin")) { + if (testInfo.getTestMethod().get().getName().contains("Plugin")) { rootDir = GenericTestUtils.getTestDir(getClass() .getSimpleName()); if (!rootDir.exists()) { @@ -201,7 +198,7 @@ public void setup() throws Exception { store.start(); } - @After + @AfterEach public void tearDown() throws Exception { store.stop(); for (ApplicationId appId : sampleAppIds) { @@ -213,7 +210,7 @@ public void tearDown() throws Exception { } } - @AfterClass + @AfterAll public static void tearDownClass() throws Exception { hdfsCluster.shutdown(); FileContext fileContext = FileContext.getLocalFSFileContext(); @@ -222,10 +219,10 @@ public static void tearDownClass() throws Exception { } @Test - public void testAppLogsScanLogs() throws Exception { + void testAppLogsScanLogs() throws Exception { EntityGroupFSTimelineStore.AppLogs appLogs = store.new AppLogs(mainTestAppId, mainTestAppDirPath, - AppState.COMPLETED); + AppState.COMPLETED); appLogs.scanForLogs(); List summaryLogs = appLogs.getSummaryLogs(); List detailLogs = appLogs.getDetailLogs(); @@ -245,13 +242,13 @@ store.new AppLogs(mainTestAppId, mainTestAppDirPath, } @Test - public void testAppLogsDomainLogLastlyScanned() throws Exception { + void testAppLogsDomainLogLastlyScanned() throws Exception { EntityGroupFSTimelineStore.AppLogs appLogs = - store.new AppLogs(mainTestAppId, mainTestAppDirPath, - AppState.COMPLETED); + store.new AppLogs(mainTestAppId, mainTestAppDirPath, + AppState.COMPLETED); Path attemptDirPath = new Path(new Path(testActiveDirPath, mainTestAppId.toString()), - getAttemptDirName(mainTestAppId)); + getAttemptDirName(mainTestAppId)); //Delete the domain log from AppDirPath so first scan won't find it fs.delete(new Path(attemptDirPath, TEST_DOMAIN_LOG_FILE_NAME), false); appLogs.scanForLogs(); @@ -261,7 +258,7 @@ store.new AppLogs(mainTestAppId, mainTestAppDirPath, //Generate the domain log FSDataOutputStream out = fs.create( - new Path(attemptDirPath, TEST_DOMAIN_LOG_FILE_NAME)); + new Path(attemptDirPath, TEST_DOMAIN_LOG_FILE_NAME)); out.close(); appLogs.scanForLogs(); @@ -270,10 +267,10 @@ store.new AppLogs(mainTestAppId, mainTestAppDirPath, } @Test - public void testMoveToDone() throws Exception { + void testMoveToDone() throws Exception { EntityGroupFSTimelineStore.AppLogs appLogs = store.new AppLogs(mainTestAppId, mainTestAppDirPath, - AppState.COMPLETED); + AppState.COMPLETED); Path pathBefore = appLogs.getAppDirPath(); appLogs.moveToDone(); Path pathAfter = appLogs.getAppDirPath(); @@ -284,13 +281,13 @@ store.new AppLogs(mainTestAppId, mainTestAppDirPath, } @Test - public void testParseSummaryLogs() throws Exception { + void testParseSummaryLogs() throws Exception { TimelineDataManager tdm = PluginStoreTestUtils.getTdmWithMemStore(config); MutableCounterLong scanned = store.metrics.getEntitiesReadToSummary(); long beforeScan = scanned.value(); EntityGroupFSTimelineStore.AppLogs appLogs = store.new AppLogs(mainTestAppId, mainTestAppDirPath, - AppState.COMPLETED); + AppState.COMPLETED); appLogs.scanForLogs(); appLogs.parseSummaryLogs(tdm); PluginStoreTestUtils.verifyTestEntities(tdm); @@ -298,31 +295,31 @@ store.new AppLogs(mainTestAppId, mainTestAppDirPath, } @Test - public void testWithAnonymousUser() throws Exception { + void testWithAnonymousUser() throws Exception { try { TimelineDataManager tdm = PluginStoreTestUtils.getTdmWithMemStore(config); EntityGroupFSTimelineStore.AppLogs appLogs = - store.new AppLogs(mainTestAppId, mainTestAppDirPath, - AppState.COMPLETED); + store.new AppLogs(mainTestAppId, mainTestAppDirPath, + AppState.COMPLETED); FileStatus fileStatus = mock(FileStatus.class); when(fileStatus.getOwner()).thenReturn(null); appLogs.scanForLogs(); appLogs.parseSummaryLogs(tdm); PluginStoreTestUtils.verifyTestEntities(tdm); } catch (IllegalArgumentException ie) { - Assert.fail("No exception needs to be thrown as anonymous user is configured"); + fail("No exception needs to be thrown as anonymous user is configured"); } } @Test - public void testCleanLogs() throws Exception { + void testCleanLogs() throws Exception { // Create test dirs and files // Irrelevant file, should not be reclaimed String appDirName = mainTestAppId.toString(); String attemptDirName = ApplicationAttemptId.appAttemptIdStrPrefix + appDirName + "_1"; Path irrelevantFilePath = new Path( - testDoneDirPath, "irrelevant.log"); + testDoneDirPath, "irrelevant.log"); FSDataOutputStream stream = fs.create(irrelevantFilePath); stream.close(); // Irrelevant directory, should not be reclaimed @@ -395,7 +392,7 @@ public void testCleanLogs() throws Exception { } @Test - public void testCleanBuckets() throws Exception { + void testCleanBuckets() throws Exception { // ClusterTimeStampDir with App Log Dirs Path clusterTimeStampDir1 = new Path(testDoneDirPath, Long.toString(sampleAppIds.get(0).getClusterTimestamp())); @@ -441,17 +438,17 @@ public void testCleanBuckets() throws Exception { } @Test - public void testNullCheckGetEntityTimelines() throws Exception { + void testNullCheckGetEntityTimelines() throws Exception { try { store.getEntityTimelines("YARN_APPLICATION", null, null, null, null, null); } catch (NullPointerException e) { - Assert.fail("NPE when getEntityTimelines called with Null EntityIds"); + fail("NPE when getEntityTimelines called with Null EntityIds"); } } @Test - public void testPluginRead() throws Exception { + void testPluginRead() throws Exception { // Verify precondition assertEquals(EntityGroupPlugInForTest.class.getName(), store.getConfig().get( @@ -459,8 +456,8 @@ public void testPluginRead() throws Exception { List currPlugins = store.getPlugins(); for (TimelineEntityGroupPlugin plugin : currPlugins) { ClassLoader pluginClassLoader = plugin.getClass().getClassLoader(); - assertTrue("Should set up ApplicationClassLoader", - pluginClassLoader instanceof ApplicationClassLoader); + assertTrue(pluginClassLoader instanceof ApplicationClassLoader, + "Should set up ApplicationClassLoader"); URL[] paths = ((URLClassLoader) pluginClassLoader).getURLs(); boolean foundJAR = false; for (URL path : paths) { @@ -468,13 +465,13 @@ public void testPluginRead() throws Exception { foundJAR = true; } } - assertTrue("Not found path " + testJar.getAbsolutePath() - + " for plugin " + plugin.getClass().getName(), foundJAR); + assertTrue(foundJAR, "Not found path " + testJar.getAbsolutePath() + + " for plugin " + plugin.getClass().getName()); } // Load data and cache item, prepare timeline store by making a cache item EntityGroupFSTimelineStore.AppLogs appLogs = store.new AppLogs(mainTestAppId, mainTestAppDirPath, - AppState.COMPLETED); + AppState.COMPLETED); EntityCacheItem cacheItem = new EntityCacheItem( EntityGroupPlugInForTest.getStandardTimelineGroupId(mainTestAppId), config); @@ -514,11 +511,11 @@ store.new AppLogs(mainTestAppId, mainTestAppDirPath, } @Test - public void testSummaryRead() throws Exception { + void testSummaryRead() throws Exception { // Load data EntityGroupFSTimelineStore.AppLogs appLogs = store.new AppLogs(mainTestAppId, mainTestAppDirPath, - AppState.COMPLETED); + AppState.COMPLETED); MutableCounterLong summaryLogEntityRead = store.metrics.getGetEntityToSummaryOps(); long numEntityReadBefore = summaryLogEntityRead.value(); @@ -543,7 +540,7 @@ store.new AppLogs(mainTestAppId, mainTestAppDirPath, } @Test - public void testGetEntityPluginRead() throws Exception { + void testGetEntityPluginRead() throws Exception { EntityGroupFSTimelineStore store = null; ApplicationId appId = ApplicationId.fromString("application_1501509265053_0001"); @@ -571,7 +568,7 @@ public void testGetEntityPluginRead() throws Exception { } @Test - public void testScanActiveLogsWithInvalidFile() throws Exception { + void testScanActiveLogsWithInvalidFile() throws Exception { Path invalidFile = new Path(testActiveDirPath, "invalidfile"); try { if (!fs.exists(invalidFile)) { @@ -579,7 +576,7 @@ public void testScanActiveLogsWithInvalidFile() throws Exception { } store.scanActiveLogs(); } catch (StackOverflowError error) { - Assert.fail("EntityLogScanner crashed with StackOverflowError"); + fail("EntityLogScanner crashed with StackOverflowError"); } finally { if (fs.exists(invalidFile)) { fs.delete(invalidFile, false); @@ -588,7 +585,7 @@ public void testScanActiveLogsWithInvalidFile() throws Exception { } @Test - public void testScanActiveLogsAndMoveToDonePluginRead() throws Exception { + void testScanActiveLogsAndMoveToDonePluginRead() throws Exception { EntityGroupFSTimelineStore store = null; ApplicationId appId = ApplicationId.fromString("application_1501509265053_0002"); @@ -637,7 +634,7 @@ public static int getEntitiesCount() { } @Test - public void testIfAnyDuplicateEntities() throws Exception { + void testIfAnyDuplicateEntities() throws Exception { // Create an application with some entities ApplicationId appId = ApplicationId.fromString("application_1501509265053_0002"); @@ -683,15 +680,14 @@ public Boolean get() { return TestTimelineStore.getEntitiesCount() == 2; } }, 100, 10000); - assertEquals("Wrong Initial Entities Count", - 2, TestTimelineStore.getEntitiesCount()); + assertEquals(2, TestTimelineStore.getEntitiesCount(), "Wrong Initial Entities Count"); // Append the Summary log file with few more entities TimelineEntities entities = PluginStoreTestUtils.generateTestEntities(); FSDataOutputStream outStream = fs.append( new Path(attemptDirPath, TEST_SUMMARY_LOG_FILE_NAME)); JsonGenerator jsonGenerator - = new JsonFactory().createGenerator((OutputStream)outStream); + = new JsonFactory().createGenerator((OutputStream) outStream); jsonGenerator.setPrettyPrinter(new MinimalPrettyPrinter("\n")); ObjectMapper objMapper = new ObjectMapper(); objMapper.setAnnotationIntrospector( @@ -710,8 +706,7 @@ public Boolean get() { return TestTimelineStore.getEntitiesCount() == 4; } }, 100, 10000); - assertEquals("Duplicate Entities present", - 4, TestTimelineStore.getEntitiesCount()); + assertEquals(4, TestTimelineStore.getEntitiesCount(), "Duplicate Entities present"); } finally { if (newStore != null) { @@ -722,7 +717,7 @@ public Boolean get() { } @Test - public void testStateStoreAndRecovery() throws Exception { + void testStateStoreAndRecovery() throws Exception { // Prepare the AppLogs Data EntityGroupFSTimelineStore.AppLogs appLogs = store.new AppLogs(mainTestAppId, mainTestAppDirPath, AppState.COMPLETED); @@ -736,7 +731,7 @@ public void testStateStoreAndRecovery() throws Exception { try (DataOutputStream dataOutputStream = fs.create(checkpointFile)) { store.storeLogFiles(logsList, dataOutputStream); } catch (IOException e) { - Assert.fail("Failed to store the log files"); + fail("Failed to store the log files"); } // Recover the Log files and validate the contents @@ -748,14 +743,14 @@ public void testStateStoreAndRecovery() throws Exception { String logFileName = logInfo.getAttemptDirName() + Path.SEPARATOR + logInfo.getFilename(); Pair pair = logFiles.get(logFileName); - assertNotNull("Failed to recover " + logFileName, pair); - assertTrue("LastProcessedTime is not same", - logInfo.getLastProcessedTime() == pair.getLeft()); - assertTrue("Offset is not same", - logInfo.getOffset() == pair.getRight()); + assertNotNull(pair, "Failed to recover " + logFileName); + assertTrue(logInfo.getLastProcessedTime() == pair.getLeft(), + "LastProcessedTime is not same"); + assertTrue(logInfo.getOffset() == pair.getRight(), + "Offset is not same"); } } catch (IOException e) { - Assert.fail("Failed to recover the log files"); + fail("Failed to recover the log files"); } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timeline-pluginstorage/src/test/java/org/apache/hadoop/yarn/server/timeline/TestLevelDBCacheTimelineStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timeline-pluginstorage/src/test/java/org/apache/hadoop/yarn/server/timeline/TestLevelDBCacheTimelineStore.java index 43b04a5a37097..7763956330e5a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timeline-pluginstorage/src/test/java/org/apache/hadoop/yarn/server/timeline/TestLevelDBCacheTimelineStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timeline-pluginstorage/src/test/java/org/apache/hadoop/yarn/server/timeline/TestLevelDBCacheTimelineStore.java @@ -20,16 +20,17 @@ import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.util.ReflectionUtils; -import org.junit.After; -import org.junit.Before; -import org.junit.Test; -import static org.junit.Assert.assertNotNull; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; import java.io.IOException; +import static org.junit.jupiter.api.Assertions.assertNotNull; + public class TestLevelDBCacheTimelineStore extends TimelineStoreTestUtils { - @Before + @BeforeEach public void setup() throws Exception { store = new LevelDBCacheTimelineStore("app1"); store.init(new YarnConfiguration()); @@ -39,7 +40,7 @@ public void setup() throws Exception { loadTestDomainData(); } - @After + @AfterEach public void tearDown() throws Exception { store.stop(); } @@ -49,13 +50,13 @@ public TimelineStore getTimelineStore() { } @Test - public void testDefaultConstructor() { + void testDefaultConstructor() { TimelineStore store = null; try { store = ReflectionUtils.newInstance(LevelDBCacheTimelineStore.class, new YarnConfiguration()); } finally { - assertNotNull("LevelDBCacheTimelineStore failed to instantiate", store); + assertNotNull(store, "LevelDBCacheTimelineStore failed to instantiate"); } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timeline-pluginstorage/src/test/java/org/apache/hadoop/yarn/server/timeline/TestLogInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timeline-pluginstorage/src/test/java/org/apache/hadoop/yarn/server/timeline/TestLogInfo.java index 9182b26c6abea..22495eaf08069 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timeline-pluginstorage/src/test/java/org/apache/hadoop/yarn/server/timeline/TestLogInfo.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timeline-pluginstorage/src/test/java/org/apache/hadoop/yarn/server/timeline/TestLogInfo.java @@ -34,18 +34,18 @@ import org.apache.hadoop.yarn.api.records.timeline.TimelineEntities; import org.apache.hadoop.yarn.api.records.timeline.TimelineEntity; import org.apache.hadoop.yarn.conf.YarnConfiguration; -import org.junit.After; -import org.junit.Before; -import org.junit.Test; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; import java.io.IOException; import java.io.OutputStream; import java.nio.charset.Charset; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertTrue; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; public class TestLogInfo { @@ -75,7 +75,7 @@ public class TestLogInfo { private static final short FILE_LOG_DIR_PERMISSIONS = 0770; - @Before + @BeforeEach public void setup() throws Exception { config.set(MiniDFSCluster.HDFS_MINIDFS_BASEDIR, TEST_ROOT_DIR.toString()); HdfsConfiguration hdfsConfig = new HdfsConfiguration(); @@ -101,7 +101,7 @@ public void setup() throws Exception { writeBrokenFile(new Path(testAppDirPath, TEST_BROKEN_FILE_NAME)); } - @After + @AfterEach public void tearDown() throws Exception { jsonGenerator.close(); outStream.close(); @@ -110,7 +110,7 @@ public void tearDown() throws Exception { } @Test - public void testMatchesGroupId() throws Exception { + void testMatchesGroupId() throws Exception { String testGroupId = "app1_group1"; // Match EntityLogInfo testLogInfo = new EntityLogInfo(TEST_ATTEMPT_DIR_NAME, @@ -145,7 +145,7 @@ public void testMatchesGroupId() throws Exception { } @Test - public void testParseEntity() throws Exception { + void testParseEntity() throws Exception { // Load test data TimelineDataManager tdm = PluginStoreTestUtils.getTdmWithMemStore(config); EntityLogInfo testLogInfo = new EntityLogInfo(TEST_ATTEMPT_DIR_NAME, @@ -159,7 +159,7 @@ public void testParseEntity() throws Exception { } @Test - public void testParseBrokenEntity() throws Exception { + void testParseBrokenEntity() throws Exception { // Load test data TimelineDataManager tdm = PluginStoreTestUtils.getTdmWithMemStore(config); EntityLogInfo testLogInfo = new EntityLogInfo(TEST_ATTEMPT_DIR_NAME, @@ -177,7 +177,7 @@ public void testParseBrokenEntity() throws Exception { } @Test - public void testParseDomain() throws Exception { + void testParseDomain() throws Exception { // Load test data TimelineDataManager tdm = PluginStoreTestUtils.getTdmWithMemStore(config); DomainLogInfo domainLogInfo = new DomainLogInfo(TEST_ATTEMPT_DIR_NAME, diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timeline-pluginstorage/src/test/java/org/apache/hadoop/yarn/server/timeline/TestOverrideTimelineStoreYarnClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timeline-pluginstorage/src/test/java/org/apache/hadoop/yarn/server/timeline/TestOverrideTimelineStoreYarnClient.java index c190266bfbdab..5b9500a401743 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timeline-pluginstorage/src/test/java/org/apache/hadoop/yarn/server/timeline/TestOverrideTimelineStoreYarnClient.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timeline-pluginstorage/src/test/java/org/apache/hadoop/yarn/server/timeline/TestOverrideTimelineStoreYarnClient.java @@ -22,20 +22,21 @@ import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.client.api.YarnClient; import org.apache.hadoop.yarn.conf.YarnConfiguration; -import org.junit.Assert; -import org.junit.Test; +import org.junit.jupiter.api.Test; import java.io.IOException; +import static org.junit.jupiter.api.Assertions.assertEquals; + public class TestOverrideTimelineStoreYarnClient { @Test - public void testLifecycleAndOverride() throws Throwable { + void testLifecycleAndOverride() throws Throwable { YarnConfiguration conf = new YarnConfiguration(); - try(NoRMStore store = new NoRMStore()) { + try (NoRMStore store = new NoRMStore()) { store.init(conf); store.start(); - Assert.assertEquals(EntityGroupFSTimelineStore.AppState.ACTIVE, + assertEquals(EntityGroupFSTimelineStore.AppState.ACTIVE, store.getAppState(ApplicationId.newInstance(1, 1))); store.stop(); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/pom.xml index adc4c3bcbb53e..a323eaff8558c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/pom.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/pom.xml @@ -438,10 +438,6 @@ commons-lang3 test - - javax.ws.rs - javax.ws.rs-api - diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/pom.xml index 39706647b7a23..02fc357226552 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/pom.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/pom.xml @@ -145,11 +145,6 @@ junit test - - - javax.ws.rs - javax.ws.rs-api - diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/pom.xml index 0876cb3e03c10..65af3afadda62 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/pom.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/pom.xml @@ -117,11 +117,6 @@ 1.1.1 - - javax.ws.rs - javax.ws.rs-api - - org.apache.hadoop diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/pom.xml index 1e0e9ae57ad8d..9681dfa4c1df5 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/pom.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/pom.xml @@ -69,11 +69,6 @@ mockito-core test - - junit - junit - test - org.assertj assertj-core @@ -109,6 +104,21 @@ grizzly-http-servlet test + + org.junit.jupiter + junit-jupiter-api + test + + + org.junit.jupiter + junit-jupiter-engine + test + + + org.junit.platform + junit-platform-launcher + test + @@ -124,7 +134,6 @@ org.bouncycastle bcpkix-jdk15on - diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/test/java/org/apache/hadoop/yarn/server/webproxy/TestAppReportFetcher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/test/java/org/apache/hadoop/yarn/server/webproxy/TestAppReportFetcher.java index 4bf0db6b7d7af..cc7542f3e596b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/test/java/org/apache/hadoop/yarn/server/webproxy/TestAppReportFetcher.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/test/java/org/apache/hadoop/yarn/server/webproxy/TestAppReportFetcher.java @@ -20,6 +20,10 @@ import java.io.IOException; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Test; +import org.mockito.Mockito; + import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.yarn.api.ApplicationClientProtocol; import org.apache.hadoop.yarn.api.ApplicationHistoryProtocol; @@ -29,10 +33,9 @@ import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.exceptions.ApplicationNotFoundException; import org.apache.hadoop.yarn.exceptions.YarnException; -import org.junit.After; -import org.junit.Assert; -import org.junit.Test; -import org.mockito.Mockito; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.fail; public class TestAppReportFetcher { @@ -42,7 +45,7 @@ public class TestAppReportFetcher { private static AppReportFetcher fetcher; private final String appNotFoundExceptionMsg = "APP NOT FOUND"; - @After + @AfterEach public void cleanUp() { historyManager = null; appManager = null; @@ -63,29 +66,29 @@ public void testHelper(boolean isAHSEnabled) } @Test - public void testFetchReportAHSEnabled() throws YarnException, IOException { + void testFetchReportAHSEnabled() throws YarnException, IOException { testHelper(true); Mockito.verify(historyManager, Mockito.times(1)) - .getApplicationReport(Mockito.any(GetApplicationReportRequest.class)); + .getApplicationReport(Mockito.any(GetApplicationReportRequest.class)); Mockito.verify(appManager, Mockito.times(1)) - .getApplicationReport(Mockito.any(GetApplicationReportRequest.class)); + .getApplicationReport(Mockito.any(GetApplicationReportRequest.class)); } @Test - public void testFetchReportAHSDisabled() throws YarnException, IOException { + void testFetchReportAHSDisabled() throws YarnException, IOException { try { testHelper(false); } catch (ApplicationNotFoundException e) { - Assert.assertTrue(e.getMessage() == appNotFoundExceptionMsg); + assertEquals(appNotFoundExceptionMsg, e.getMessage()); /* RM will not know of the app and Application History Service is disabled * So we will not try to get the report from AHS and RM will throw * ApplicationNotFoundException */ } Mockito.verify(appManager, Mockito.times(1)) - .getApplicationReport(Mockito.any(GetApplicationReportRequest.class)); + .getApplicationReport(Mockito.any(GetApplicationReportRequest.class)); if (historyManager != null) { - Assert.fail("HistoryManager should be null as AHS is disabled"); + fail("HistoryManager should be null as AHS is disabled"); } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/test/java/org/apache/hadoop/yarn/server/webproxy/TestProxyCA.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/test/java/org/apache/hadoop/yarn/server/webproxy/TestProxyCA.java index af97396a19de2..0ef5e690811e2 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/test/java/org/apache/hadoop/yarn/server/webproxy/TestProxyCA.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/test/java/org/apache/hadoop/yarn/server/webproxy/TestProxyCA.java @@ -18,20 +18,6 @@ package org.apache.hadoop.yarn.server.webproxy; -import org.apache.hadoop.security.ssl.KeyStoreTestUtil; -import org.apache.hadoop.yarn.api.records.ApplicationId; -import org.junit.Assert; -import org.junit.Test; -import org.mockito.Mockito; -import org.mockito.invocation.InvocationOnMock; -import org.mockito.stubbing.Answer; - -import javax.net.ssl.HostnameVerifier; -import javax.net.ssl.SSLPeerUnverifiedException; -import javax.net.ssl.SSLSession; -import javax.net.ssl.X509KeyManager; -import javax.net.ssl.X509TrustManager; -import javax.security.auth.x500.X500Principal; import java.security.InvalidKeyException; import java.security.Key; import java.security.KeyPair; @@ -52,109 +38,132 @@ import java.util.HashSet; import java.util.Random; import java.util.Set; +import javax.net.ssl.HostnameVerifier; +import javax.net.ssl.SSLPeerUnverifiedException; +import javax.net.ssl.SSLSession; +import javax.net.ssl.X509KeyManager; +import javax.net.ssl.X509TrustManager; +import javax.security.auth.x500.X500Principal; + +import org.junit.jupiter.api.Test; +import org.mockito.Mockito; +import org.mockito.invocation.InvocationOnMock; +import org.mockito.stubbing.Answer; + +import org.apache.hadoop.security.ssl.KeyStoreTestUtil; +import org.apache.hadoop.yarn.api.records.ApplicationId; + +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.fail; public class TestProxyCA { @Test - public void testInit() throws Exception { + void testInit() throws Exception { ProxyCA proxyCA = new ProxyCA(); - Assert.assertNull(proxyCA.getCaCert()); - Assert.assertNull(proxyCA.getCaKeyPair()); - Assert.assertNull(proxyCA.getX509KeyManager()); - Assert.assertNull(proxyCA.getHostnameVerifier()); + assertNull(proxyCA.getCaCert()); + assertNull(proxyCA.getCaKeyPair()); + assertNull(proxyCA.getX509KeyManager()); + assertNull(proxyCA.getHostnameVerifier()); proxyCA.init(); - Assert.assertNotNull(proxyCA.getCaCert()); - Assert.assertNotNull(proxyCA.getCaKeyPair()); - Assert.assertNotNull(proxyCA.getX509KeyManager()); - Assert.assertNotNull(proxyCA.getHostnameVerifier()); + assertNotNull(proxyCA.getCaCert()); + assertNotNull(proxyCA.getCaKeyPair()); + assertNotNull(proxyCA.getX509KeyManager()); + assertNotNull(proxyCA.getHostnameVerifier()); } @Test - public void testInit2Null() throws Exception { + void testInit2Null() throws Exception { ProxyCA proxyCA = new ProxyCA(); - Assert.assertNull(proxyCA.getCaCert()); - Assert.assertNull(proxyCA.getCaKeyPair()); - Assert.assertNull(proxyCA.getX509KeyManager()); - Assert.assertNull(proxyCA.getHostnameVerifier()); + assertNull(proxyCA.getCaCert()); + assertNull(proxyCA.getCaKeyPair()); + assertNull(proxyCA.getX509KeyManager()); + assertNull(proxyCA.getHostnameVerifier()); // null certificate and private key proxyCA.init(null, null); - Assert.assertNotNull(proxyCA.getCaCert()); - Assert.assertNotNull(proxyCA.getCaKeyPair()); - Assert.assertNotNull(proxyCA.getX509KeyManager()); - Assert.assertNotNull(proxyCA.getHostnameVerifier()); + assertNotNull(proxyCA.getCaCert()); + assertNotNull(proxyCA.getCaKeyPair()); + assertNotNull(proxyCA.getX509KeyManager()); + assertNotNull(proxyCA.getHostnameVerifier()); } @Test - public void testInit2Mismatch() throws Exception { + void testInit2Mismatch() throws Exception { ProxyCA proxyCA = new ProxyCA(); - Assert.assertNull(proxyCA.getCaCert()); - Assert.assertNull(proxyCA.getCaKeyPair()); - Assert.assertNull(proxyCA.getX509KeyManager()); - Assert.assertNull(proxyCA.getHostnameVerifier()); + assertNull(proxyCA.getCaCert()); + assertNull(proxyCA.getCaKeyPair()); + assertNull(proxyCA.getX509KeyManager()); + assertNull(proxyCA.getHostnameVerifier()); // certificate and private key don't match CertKeyPair pair1 = createCertAndKeyPair(); CertKeyPair pair2 = createCertAndKeyPair(); - Assert.assertNotEquals(pair1.getCert(), pair2.getCert()); - Assert.assertNotEquals(pair1.getKeyPair().getPrivate(), + assertNotEquals(pair1.getCert(), pair2.getCert()); + assertNotEquals(pair1.getKeyPair().getPrivate(), pair2.getKeyPair().getPrivate()); - Assert.assertNotEquals(pair1.getKeyPair().getPublic(), + assertNotEquals(pair1.getKeyPair().getPublic(), pair2.getKeyPair().getPublic()); proxyCA.init(pair1.getCert(), pair2.getKeyPair().getPrivate()); - Assert.assertNotNull(proxyCA.getCaCert()); - Assert.assertNotNull(proxyCA.getCaKeyPair()); - Assert.assertNotNull(proxyCA.getX509KeyManager()); - Assert.assertNotNull(proxyCA.getHostnameVerifier()); - Assert.assertNotEquals(proxyCA.getCaCert(), pair1.getCert()); - Assert.assertNotEquals(proxyCA.getCaKeyPair().getPrivate(), + assertNotNull(proxyCA.getCaCert()); + assertNotNull(proxyCA.getCaKeyPair()); + assertNotNull(proxyCA.getX509KeyManager()); + assertNotNull(proxyCA.getHostnameVerifier()); + assertNotEquals(proxyCA.getCaCert(), pair1.getCert()); + assertNotEquals(proxyCA.getCaKeyPair().getPrivate(), pair2.getKeyPair().getPrivate()); - Assert.assertNotEquals(proxyCA.getCaKeyPair().getPublic(), + assertNotEquals(proxyCA.getCaKeyPair().getPublic(), pair2.getKeyPair().getPublic()); } @Test - public void testInit2Invalid() throws Exception { + void testInit2Invalid() throws Exception { ProxyCA proxyCA = new ProxyCA(); - Assert.assertNull(proxyCA.getCaCert()); - Assert.assertNull(proxyCA.getCaKeyPair()); - Assert.assertNull(proxyCA.getX509KeyManager()); - Assert.assertNull(proxyCA.getHostnameVerifier()); + assertNull(proxyCA.getCaCert()); + assertNull(proxyCA.getCaKeyPair()); + assertNull(proxyCA.getX509KeyManager()); + assertNull(proxyCA.getHostnameVerifier()); // Invalid key - fail the verification X509Certificate certificate = Mockito.mock(X509Certificate.class); PrivateKey privateKey = Mockito.mock(PrivateKey.class); try { proxyCA.init(certificate, privateKey); - Assert.fail("Expected InvalidKeyException"); + fail("Expected InvalidKeyException"); } catch (InvalidKeyException e) { // expected } } @Test - public void testInit2() throws Exception { + void testInit2() throws Exception { ProxyCA proxyCA = new ProxyCA(); - Assert.assertNull(proxyCA.getCaCert()); - Assert.assertNull(proxyCA.getCaKeyPair()); - Assert.assertNull(proxyCA.getX509KeyManager()); - Assert.assertNull(proxyCA.getHostnameVerifier()); + assertNull(proxyCA.getCaCert()); + assertNull(proxyCA.getCaKeyPair()); + assertNull(proxyCA.getX509KeyManager()); + assertNull(proxyCA.getHostnameVerifier()); // certificate and private key do match CertKeyPair pair = createCertAndKeyPair(); proxyCA.init(pair.getCert(), pair.getKeyPair().getPrivate()); - Assert.assertEquals(pair.getCert(), proxyCA.getCaCert()); - Assert.assertEquals(pair.getKeyPair().getPrivate(), + assertEquals(pair.getCert(), proxyCA.getCaCert()); + assertEquals(pair.getKeyPair().getPrivate(), proxyCA.getCaKeyPair().getPrivate()); - Assert.assertEquals(pair.getKeyPair().getPublic(), + assertEquals(pair.getKeyPair().getPublic(), proxyCA.getCaKeyPair().getPublic()); - Assert.assertNotNull(proxyCA.getX509KeyManager()); - Assert.assertNotNull(proxyCA.getHostnameVerifier()); + assertNotNull(proxyCA.getX509KeyManager()); + assertNotNull(proxyCA.getHostnameVerifier()); } @Test - public void testCreateChildKeyStore() throws Exception { + void testCreateChildKeyStore() throws Exception { ProxyCA proxyCA = new ProxyCA(); proxyCA.init(); ApplicationId appId = @@ -163,29 +172,29 @@ public void testCreateChildKeyStore() throws Exception { "password"); KeyStore keyStore = KeyStoreTestUtil.bytesToKeyStore(keystoreBytes, "password"); - Assert.assertEquals(1, keyStore.size()); + assertEquals(1, keyStore.size()); Certificate[] certChain = keyStore.getCertificateChain("server"); - Assert.assertEquals(2, certChain.length); + assertEquals(2, certChain.length); X509Certificate caCert = (X509Certificate) certChain[1]; X509Certificate cert = (X509Certificate) certChain[0]; // check child cert - Assert.assertEquals(caCert.getSubjectX500Principal().toString(), + assertEquals(caCert.getSubjectX500Principal().toString(), cert.getIssuerDN().toString()); - Assert.assertEquals(new X500Principal("CN=" + appId), + assertEquals(new X500Principal("CN=" + appId), cert.getSubjectX500Principal()); - Assert.assertFalse("Found multiple fields in X500 Principal, when there " + - "should have only been one: " + cert.getSubjectX500Principal(), - cert.getSubjectX500Principal().toString().contains(",")); - Assert.assertEquals("SHA512withRSA", cert.getSigAlgName()); - Assert.assertEquals(cert.getNotBefore(), cert.getNotAfter()); - Assert.assertTrue("Expected certificate to be expired but was not: " - + cert.getNotAfter(), cert.getNotAfter().before(new Date())); - Assert.assertEquals(new X500Principal("CN=" + appId).toString(), + assertFalse(cert.getSubjectX500Principal().toString().contains(","), + "Found multiple fields in X500 Principal, when there " + + "should have only been one: " + cert.getSubjectX500Principal()); + assertEquals("SHA512withRSA", cert.getSigAlgName()); + assertEquals(cert.getNotBefore(), cert.getNotAfter()); + assertTrue(cert.getNotAfter().before(new Date()), + "Expected certificate to be expired but was not: " + cert.getNotAfter()); + assertEquals(new X500Principal("CN=" + appId).toString(), cert.getSubjectDN().toString()); Key privateKey = keyStore.getKey("server", "password".toCharArray()); - Assert.assertEquals("RSA", privateKey.getAlgorithm()); - Assert.assertEquals(-1, cert.getBasicConstraints()); + assertEquals("RSA", privateKey.getAlgorithm()); + assertEquals(-1, cert.getBasicConstraints()); // verify signature on child cert PublicKey caPublicKey = caCert.getPublicKey(); @@ -193,7 +202,7 @@ public void testCreateChildKeyStore() throws Exception { // check CA cert checkCACert(caCert); - Assert.assertEquals(proxyCA.getCaCert(), caCert); + assertEquals(proxyCA.getCaCert(), caCert); // verify signature on CA cert caCert.verify(caPublicKey); @@ -202,24 +211,24 @@ public void testCreateChildKeyStore() throws Exception { PrivateKey caPrivateKey = proxyCA.getX509KeyManager().getPrivateKey(null); checkPrivatePublicKeys(caPrivateKey, caPublicKey); - Assert.assertEquals(proxyCA.getCaKeyPair().getPublic(), caPublicKey); - Assert.assertEquals(proxyCA.getCaKeyPair().getPrivate(), caPrivateKey); + assertEquals(proxyCA.getCaKeyPair().getPublic(), caPublicKey); + assertEquals(proxyCA.getCaKeyPair().getPrivate(), caPrivateKey); } @Test - public void testGetChildTrustStore() throws Exception { + void testGetChildTrustStore() throws Exception { ProxyCA proxyCA = new ProxyCA(); proxyCA.init(); byte[] truststoreBytes = proxyCA.getChildTrustStore("password"); KeyStore truststore = KeyStoreTestUtil.bytesToKeyStore(truststoreBytes, "password"); - Assert.assertEquals(1, truststore.size()); + assertEquals(1, truststore.size()); X509Certificate caCert = (X509Certificate) truststore.getCertificate("client"); // check CA cert checkCACert(caCert); - Assert.assertEquals(proxyCA.getCaCert(), caCert); + assertEquals(proxyCA.getCaCert(), caCert); // verify signature on CA cert PublicKey caPublicKey = caCert.getPublicKey(); @@ -229,12 +238,12 @@ public void testGetChildTrustStore() throws Exception { PrivateKey caPrivateKey = proxyCA.getX509KeyManager().getPrivateKey(null); checkPrivatePublicKeys(caPrivateKey, caPublicKey); - Assert.assertEquals(proxyCA.getCaKeyPair().getPublic(), caPublicKey); - Assert.assertEquals(proxyCA.getCaKeyPair().getPrivate(), caPrivateKey); + assertEquals(proxyCA.getCaKeyPair().getPublic(), caPublicKey); + assertEquals(proxyCA.getCaKeyPair().getPrivate(), caPrivateKey); } @Test - public void testGenerateKeyStorePassword() throws Exception { + void testGenerateKeyStorePassword() throws Exception { // We can't possibly test every possible string, but we can at least verify // a few things about a few of the generated strings as a sanity check ProxyCA proxyCA = new ProxyCA(); @@ -243,23 +252,23 @@ public void testGenerateKeyStorePassword() throws Exception { for (int i = 0; i < 5; i++) { String password = proxyCA.generateKeyStorePassword(); - Assert.assertEquals(16, password.length()); + assertEquals(16, password.length()); for (char c : password.toCharArray()) { - Assert.assertFalse("Found character '" + c + "' in password '" - + password + "' which is outside of the expected range", c < ' '); - Assert.assertFalse("Found character '" + c + "' in password '" - + password + "' which is outside of the expected range", c > 'z'); + assertFalse(c < ' ', "Found character '" + c + "' in password '" + + password + "' which is outside of the expected range"); + assertFalse(c > 'z', "Found character '" + c + "' in password '" + + password + "' which is outside of the expected range"); } - Assert.assertFalse("Password " + password - + " was generated twice, which is _extremely_ unlikely" - + " and shouldn't practically happen: " + passwords, - passwords.contains(password)); + assertFalse(passwords.contains(password), + "Password " + password + + " was generated twice, which is _extremely_ unlikely" + + " and shouldn't practically happen: " + passwords); passwords.add(password); } } @Test - public void testCreateTrustManagerDefaultTrustManager() throws Exception { + void testCreateTrustManagerDefaultTrustManager() throws Exception { ProxyCA proxyCA = new ProxyCA(); proxyCA.init(); X509TrustManager defaultTrustManager = Mockito.mock(X509TrustManager.class); @@ -272,13 +281,13 @@ public void testCreateTrustManagerDefaultTrustManager() throws Exception { "CN=foo", KeyStoreTestUtil.generateKeyPair("RSA"), 30, "SHA1withRSA")}); - Assert.assertArrayEquals(defaultTrustManager.getAcceptedIssuers(), + assertArrayEquals(defaultTrustManager.getAcceptedIssuers(), trustManager.getAcceptedIssuers()); trustManager.checkClientTrusted(null, null); } @Test - public void testCreateTrustManagerYarnCert() throws Exception { + void testCreateTrustManagerYarnCert() throws Exception { ProxyCA proxyCA = new ProxyCA(); proxyCA.init(); X509TrustManager defaultTrustManager = Mockito.mock(X509TrustManager.class); @@ -297,7 +306,7 @@ public void testCreateTrustManagerYarnCert() throws Exception { } @Test - public void testCreateTrustManagerWrongApp() throws Exception { + void testCreateTrustManagerWrongApp() throws Exception { ProxyCA proxyCA = new ProxyCA(); proxyCA.init(); X509TrustManager defaultTrustManager = Mockito.mock(X509TrustManager.class); @@ -314,15 +323,15 @@ public void testCreateTrustManagerWrongApp() throws Exception { .getCertificateChain("server")); try { trustManager.checkServerTrusted(certChain, "RSA"); - Assert.fail("Should have thrown a CertificateException, but did not"); + fail("Should have thrown a CertificateException, but did not"); } catch (CertificateException ce) { - Assert.assertEquals("Expected to find Subject X500 Principal with CN=" + + assertEquals("Expected to find Subject X500 Principal with CN=" + appId + " but found CN=" + appId2, ce.getMessage()); } } @Test - public void testCreateTrustManagerWrongRM() throws Exception { + void testCreateTrustManagerWrongRM() throws Exception { ProxyCA proxyCA = new ProxyCA(); proxyCA.init(); X509TrustManager defaultTrustManager = Mockito.mock(X509TrustManager.class); @@ -345,7 +354,7 @@ public void testCreateTrustManagerWrongRM() throws Exception { } @Test - public void testCreateTrustManagerRealCert() throws Exception { + void testCreateTrustManagerRealCert() throws Exception { ProxyCA proxyCA = new ProxyCA(); proxyCA.init(); X509TrustManager defaultTrustManager = Mockito.mock(X509TrustManager.class); @@ -357,8 +366,8 @@ public void testCreateTrustManagerRealCert() throws Exception { // "real" cert X509Certificate[] certChain = new X509Certificate[]{ - KeyStoreTestUtil.generateCertificate("CN=foo.com", - KeyStoreTestUtil.generateKeyPair("RSA"), 30, "SHA1withRSA")}; + KeyStoreTestUtil.generateCertificate("CN=foo.com", + KeyStoreTestUtil.generateKeyPair("RSA"), 30, "SHA1withRSA")}; Mockito.verify(defaultTrustManager, Mockito.times(0)) .checkServerTrusted(certChain, "RSA"); trustManager.checkServerTrusted(certChain, "RSA"); @@ -379,7 +388,7 @@ public void testCreateTrustManagerRealCert() throws Exception { } @Test - public void testCreateTrustManagerExceptions() throws Exception { + void testCreateTrustManagerExceptions() throws Exception { ProxyCA proxyCA = new ProxyCA(); proxyCA.init(); X509TrustManager defaultTrustManager = Mockito.mock(X509TrustManager.class); @@ -409,37 +418,37 @@ public void testCreateTrustManagerExceptions() throws Exception { } @Test - public void testCreateKeyManager() throws Exception { + void testCreateKeyManager() throws Exception { ProxyCA proxyCA = new ProxyCA(); proxyCA.init(); X509KeyManager keyManager = proxyCA.getX509KeyManager(); - Assert.assertArrayEquals(new String[]{"client"}, + assertArrayEquals(new String[]{"client"}, keyManager.getClientAliases(null, null)); - Assert.assertEquals("client", + assertEquals("client", keyManager.chooseClientAlias(null, null, null)); - Assert.assertNull(keyManager.getServerAliases(null, null)); - Assert.assertNull(keyManager.chooseServerAlias(null, null, null)); + assertNull(keyManager.getServerAliases(null, null)); + assertNull(keyManager.chooseServerAlias(null, null, null)); byte[] truststoreBytes = proxyCA.getChildTrustStore("password"); KeyStore truststore = KeyStoreTestUtil.bytesToKeyStore(truststoreBytes, "password"); - Assert.assertEquals(1, truststore.size()); + assertEquals(1, truststore.size()); X509Certificate caCert = (X509Certificate) truststore.getCertificate("client"); - Assert.assertArrayEquals(new X509Certificate[]{caCert}, + assertArrayEquals(new X509Certificate[]{caCert}, keyManager.getCertificateChain(null)); - Assert.assertEquals(proxyCA.getCaCert(), caCert); + assertEquals(proxyCA.getCaCert(), caCert); PrivateKey caPrivateKey = keyManager.getPrivateKey(null); PublicKey caPublicKey = caCert.getPublicKey(); checkPrivatePublicKeys(caPrivateKey, caPublicKey); - Assert.assertEquals(proxyCA.getCaKeyPair().getPublic(), caPublicKey); - Assert.assertEquals(proxyCA.getCaKeyPair().getPrivate(), caPrivateKey); + assertEquals(proxyCA.getCaKeyPair().getPublic(), caPublicKey); + assertEquals(proxyCA.getCaKeyPair().getPrivate(), caPrivateKey); } @Test - public void testCreateHostnameVerifier() throws Exception { + void testCreateHostnameVerifier() throws Exception { ProxyCA proxyCA = new ProxyCA(); proxyCA.init(); HostnameVerifier verifier = proxyCA.getHostnameVerifier(); @@ -450,11 +459,11 @@ public void testCreateHostnameVerifier() throws Exception { proxyCA.createChildKeyStore( ApplicationId.newInstance(System.currentTimeMillis(), 1), "password"), "password").getCertificateChain("server")); - Assert.assertTrue(verifier.verify("foo", sslSession)); + assertTrue(verifier.verify("foo", sslSession)); } @Test - public void testCreateHostnameVerifierSSLPeerUnverifiedException() + void testCreateHostnameVerifierSSLPeerUnverifiedException() throws Exception { ProxyCA proxyCA = new ProxyCA(); proxyCA.init(); @@ -463,11 +472,11 @@ public void testCreateHostnameVerifierSSLPeerUnverifiedException() SSLSession sslSession = Mockito.mock(SSLSession.class); Mockito.when(sslSession.getPeerCertificates()).thenThrow( new SSLPeerUnverifiedException("")); - Assert.assertFalse(verifier.verify("foo", sslSession)); + assertFalse(verifier.verify("foo", sslSession)); } @Test - public void testCreateHostnameVerifierWrongRM() throws Exception { + void testCreateHostnameVerifierWrongRM() throws Exception { ProxyCA proxyCA = new ProxyCA(); proxyCA.init(); HostnameVerifier verifier = proxyCA.getHostnameVerifier(); @@ -480,11 +489,11 @@ public void testCreateHostnameVerifierWrongRM() throws Exception { proxyCA2.createChildKeyStore( ApplicationId.newInstance(System.currentTimeMillis(), 1), "password"), "password").getCertificateChain("server")); - Assert.assertFalse(verifier.verify("foo", sslSession)); + assertFalse(verifier.verify("foo", sslSession)); } @Test - public void testCreateHostnameVerifierExceptions() throws Exception { + void testCreateHostnameVerifierExceptions() throws Exception { ProxyCA proxyCA = new ProxyCA(); proxyCA.init(); HostnameVerifier verifier = proxyCA.getHostnameVerifier(); @@ -510,12 +519,12 @@ public Certificate[] answer(InvocationOnMock invocation) return certChain; } }); - Assert.assertFalse(verifier.verify("foo", sslSession)); + assertFalse(verifier.verify("foo", sslSession)); } } @Test - public void testCreateHostnameVerifierRealCert() throws Exception { + void testCreateHostnameVerifierRealCert() throws Exception { ProxyCA proxyCA = new ProxyCA(); proxyCA.init(); HostnameVerifier verifier = proxyCA.getHostnameVerifier(); @@ -534,11 +543,11 @@ public Certificate[] answer(InvocationOnMock invocation) return certChain; } }); - Assert.assertTrue(verifier.verify("foo.com", sslSession)); + assertTrue(verifier.verify("foo.com", sslSession)); } @Test - public void testCreateHostnameVerifierRealCertBad() throws Exception { + void testCreateHostnameVerifierRealCertBad() throws Exception { ProxyCA proxyCA = new ProxyCA(); proxyCA.init(); HostnameVerifier verifier = proxyCA.getHostnameVerifier(); @@ -557,27 +566,27 @@ public Certificate[] answer(InvocationOnMock invocation) return certChain; } }); - Assert.assertFalse(verifier.verify("bar.com", sslSession)); + assertFalse(verifier.verify("bar.com", sslSession)); } private void checkCACert(X509Certificate caCert) { - Assert.assertEquals(caCert.getSubjectX500Principal().toString(), + assertEquals(caCert.getSubjectX500Principal().toString(), caCert.getIssuerDN().toString()); - Assert.assertEquals(caCert.getSubjectX500Principal().toString(), + assertEquals(caCert.getSubjectX500Principal().toString(), caCert.getSubjectDN().toString()); - Assert.assertTrue("Expected CA certificate X500 Principal to start with" + - " 'OU=YARN-', but did not: " + caCert.getSubjectX500Principal(), - caCert.getSubjectX500Principal().toString().startsWith("OU=YARN-")); - Assert.assertFalse("Found multiple fields in X500 Principal, when there " + - "should have only been one: " + caCert.getSubjectX500Principal(), - caCert.getSubjectX500Principal().toString().contains(",")); - Assert.assertEquals("SHA512withRSA", caCert.getSigAlgName()); - Assert.assertEquals( + assertTrue(caCert.getSubjectX500Principal().toString().startsWith("OU=YARN-"), + "Expected CA certificate X500 Principal to start with" + + " 'OU=YARN-', but did not: " + caCert.getSubjectX500Principal()); + assertFalse(caCert.getSubjectX500Principal().toString().contains(","), + "Found multiple fields in X500 Principal, when there " + + "should have only been one: " + caCert.getSubjectX500Principal()); + assertEquals("SHA512withRSA", caCert.getSigAlgName()); + assertEquals( new GregorianCalendar(2037, Calendar.DECEMBER, 31).getTime(), caCert.getNotAfter()); - Assert.assertTrue("Expected certificate to have started but was not: " - + caCert.getNotBefore(), caCert.getNotBefore().before(new Date())); - Assert.assertEquals(0, caCert.getBasicConstraints()); + assertTrue(caCert.getNotBefore().before(new Date()), + "Expected certificate to have started but was not: " + caCert.getNotBefore()); + assertEquals(0, caCert.getBasicConstraints()); } private void checkPrivatePublicKeys(PrivateKey privateKey, @@ -592,7 +601,7 @@ private void checkPrivatePublicKeys(PrivateKey privateKey, signer = Signature.getInstance("SHA512withRSA"); signer.initVerify(publicKey); signer.update(data); - Assert.assertTrue(signer.verify(sig)); + assertTrue(signer.verify(sig)); } private X509Certificate[] castCertificateArrayToX509CertificateArray( diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/test/java/org/apache/hadoop/yarn/server/webproxy/TestProxyUriUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/test/java/org/apache/hadoop/yarn/server/webproxy/TestProxyUriUtils.java index 20e5cdb90074e..07c0dc1d5abc5 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/test/java/org/apache/hadoop/yarn/server/webproxy/TestProxyUriUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/test/java/org/apache/hadoop/yarn/server/webproxy/TestProxyUriUtils.java @@ -18,59 +18,64 @@ package org.apache.hadoop.yarn.server.webproxy; -import static org.junit.Assert.*; - import java.net.URI; import java.net.URISyntaxException; import java.util.List; +import org.junit.jupiter.api.Test; + import org.apache.hadoop.util.Lists; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.server.utils.BuilderUtils; import org.apache.hadoop.yarn.util.TrackingUriPlugin; -import org.junit.Test; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertThrows; public class TestProxyUriUtils { @Test - public void testGetPathApplicationId() { - assertEquals("/proxy/application_100_0001", + void testGetPathApplicationId() { + assertEquals("/proxy/application_100_0001", ProxyUriUtils.getPath(BuilderUtils.newApplicationId(100l, 1))); - assertEquals("/proxy/application_6384623_0005", + assertEquals("/proxy/application_6384623_0005", ProxyUriUtils.getPath(BuilderUtils.newApplicationId(6384623l, 5))); } - @Test(expected = IllegalArgumentException.class) - public void testGetPathApplicationIdBad() { - ProxyUriUtils.getPath(null); + @Test + void testGetPathApplicationIdBad() { + assertThrows(IllegalArgumentException.class, () -> { + ProxyUriUtils.getPath(null); + }); } - + @Test - public void testGetPathApplicationIdString() { - assertEquals("/proxy/application_6384623_0005", + void testGetPathApplicationIdString() { + assertEquals("/proxy/application_6384623_0005", ProxyUriUtils.getPath(BuilderUtils.newApplicationId(6384623l, 5), null)); assertEquals("/proxy/application_6384623_0005/static/app", ProxyUriUtils.getPath(BuilderUtils.newApplicationId(6384623l, 5), "/static/app")); - assertEquals("/proxy/application_6384623_0005/", + assertEquals("/proxy/application_6384623_0005/", ProxyUriUtils.getPath(BuilderUtils.newApplicationId(6384623l, 5), "/")); - assertEquals("/proxy/application_6384623_0005/some/path", + assertEquals("/proxy/application_6384623_0005/some/path", ProxyUriUtils.getPath(BuilderUtils.newApplicationId(6384623l, 5), "some/path")); } - - @Test - public void testGetPathAndQuery() { + + @Test + void testGetPathAndQuery() { assertEquals("/proxy/application_6384623_0005/static/app?foo=bar", - ProxyUriUtils.getPathAndQuery(BuilderUtils.newApplicationId(6384623l, 5), "/static/app", - "?foo=bar", false)); - + ProxyUriUtils.getPathAndQuery(BuilderUtils.newApplicationId(6384623l, 5), "/static/app", + "?foo=bar", false)); + assertEquals("/proxy/application_6384623_0005/static/app?foo=bar&bad=good&proxyapproved=true", - ProxyUriUtils.getPathAndQuery(BuilderUtils.newApplicationId(6384623l, 5), "/static/app", + ProxyUriUtils.getPathAndQuery(BuilderUtils.newApplicationId(6384623l, 5), "/static/app", "foo=bar&bad=good", true)); } @Test - public void testGetProxyUri() throws Exception { + void testGetProxyUri() throws Exception { URI originalUri = new URI("http://host.com/static/foo?bar=bar"); URI proxyUri = new URI("http://proxy.net:8080/"); ApplicationId id = BuilderUtils.newApplicationId(6384623l, 5); @@ -79,9 +84,9 @@ public void testGetProxyUri() throws Exception { assertEquals(expected, result); } - + @Test - public void testGetProxyUriNull() throws Exception { + void testGetProxyUriNull() throws Exception { URI originalUri = null; URI proxyUri = new URI("http://proxy.net:8080/"); ApplicationId id = BuilderUtils.newApplicationId(6384623l, 5); @@ -91,7 +96,7 @@ public void testGetProxyUriNull() throws Exception { } @Test - public void testGetProxyUriFromPluginsReturnsNullIfNoPlugins() + void testGetProxyUriFromPluginsReturnsNullIfNoPlugins() throws URISyntaxException { ApplicationId id = BuilderUtils.newApplicationId(6384623l, 5); List list = @@ -100,7 +105,7 @@ public void testGetProxyUriFromPluginsReturnsNullIfNoPlugins() } @Test - public void testGetProxyUriFromPluginsReturnsValidUriWhenAble() + void testGetProxyUriFromPluginsReturnsValidUriWhenAble() throws URISyntaxException { ApplicationId id = BuilderUtils.newApplicationId(6384623l, 5); List list = @@ -119,6 +124,6 @@ public URI getTrackingUri(ApplicationId id) throws URISyntaxException { }); URI result = ProxyUriUtils.getUriFromTrackingPlugins(id, list); assertNotNull(result); - + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/test/java/org/apache/hadoop/yarn/server/webproxy/TestWebAppProxyServer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/test/java/org/apache/hadoop/yarn/server/webproxy/TestWebAppProxyServer.java index 87907a64f2875..8ddcd64a13343 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/test/java/org/apache/hadoop/yarn/server/webproxy/TestWebAppProxyServer.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/test/java/org/apache/hadoop/yarn/server/webproxy/TestWebAppProxyServer.java @@ -18,17 +18,17 @@ package org.apache.hadoop.yarn.server.webproxy; -import static org.junit.Assert.assertEquals; +import java.net.InetSocketAddress; + +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; import org.apache.hadoop.service.Service; import org.apache.hadoop.service.Service.STATE; import org.apache.hadoop.yarn.conf.YarnConfiguration; -import org.junit.After; -import org.junit.Assert; -import org.junit.Before; -import org.junit.Test; -import java.net.InetSocketAddress; +import static org.junit.jupiter.api.Assertions.assertEquals; public class TestWebAppProxyServer { private WebAppProxyServer webAppProxy = null; @@ -36,20 +36,20 @@ public class TestWebAppProxyServer { private final String proxyAddress = "localhost:" + port; private YarnConfiguration conf = null; - @Before + @BeforeEach public void setUp() throws Exception { conf = new YarnConfiguration(); conf.set(YarnConfiguration.PROXY_ADDRESS, proxyAddress); webAppProxy = new WebAppProxyServer(); } - @After + @AfterEach public void tearDown() throws Exception { webAppProxy.stop(); } @Test - public void testStart() { + void testStart() { webAppProxy.init(conf); assertEquals(STATE.INITED, webAppProxy.getServiceState()); webAppProxy.start(); @@ -62,7 +62,7 @@ public void testStart() { } @Test - public void testStartWithBindHost() { + void testStartWithBindHost() { String bindHost = "0.0.0.0"; conf.set(YarnConfiguration.PROXY_BIND_HOST, bindHost); webAppProxy.init(conf); @@ -80,12 +80,12 @@ public void testStartWithBindHost() { @Test - public void testBindAddress() { + void testBindAddress() { conf = new YarnConfiguration(); InetSocketAddress defaultBindAddress = WebAppProxyServer.getBindAddress(conf); - Assert.assertEquals("Web Proxy default bind address port is incorrect", - YarnConfiguration.DEFAULT_PROXY_PORT, - defaultBindAddress.getPort()); + assertEquals(YarnConfiguration.DEFAULT_PROXY_PORT, + defaultBindAddress.getPort(), + "Web Proxy default bind address port is incorrect"); } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/test/java/org/apache/hadoop/yarn/server/webproxy/TestWebAppProxyServlet.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/test/java/org/apache/hadoop/yarn/server/webproxy/TestWebAppProxyServlet.java index 6c8993f6e80b7..c59d5b952a8c5 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/test/java/org/apache/hadoop/yarn/server/webproxy/TestWebAppProxyServlet.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/test/java/org/apache/hadoop/yarn/server/webproxy/TestWebAppProxyServlet.java @@ -18,14 +18,6 @@ package org.apache.hadoop.yarn.server.webproxy; -import static org.assertj.core.api.Assertions.assertThat; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertTrue; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; - import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.InputStream; @@ -35,14 +27,13 @@ import java.net.ConnectException; import java.net.HttpCookie; import java.net.HttpURLConnection; +import java.net.SocketTimeoutException; import java.net.URI; import java.net.URL; -import java.net.SocketTimeoutException; import java.util.Collections; import java.util.Enumeration; import java.util.List; import java.util.Map; - import javax.servlet.ServletConfig; import javax.servlet.ServletContext; import javax.servlet.ServletException; @@ -50,6 +41,19 @@ import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; +import org.eclipse.jetty.server.Server; +import org.eclipse.jetty.server.ServerConnector; +import org.eclipse.jetty.servlet.ServletContextHandler; +import org.eclipse.jetty.servlet.ServletHolder; +import org.eclipse.jetty.util.thread.QueuedThreadPool; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.Timeout; +import org.mockito.Mockito; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.CommonConfigurationKeys; import org.apache.hadoop.http.HttpServer2; @@ -65,17 +69,15 @@ import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.webapp.MimeType; import org.apache.hadoop.yarn.webapp.util.WebAppUtils; -import org.eclipse.jetty.server.ServerConnector; -import org.eclipse.jetty.servlet.ServletContextHandler; -import org.eclipse.jetty.util.thread.QueuedThreadPool; -import org.junit.AfterClass; -import org.junit.BeforeClass; -import org.junit.Test; -import org.eclipse.jetty.server.Server; -import org.eclipse.jetty.servlet.ServletHolder; -import org.mockito.Mockito; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; /** * Test the WebAppProxyServlet and WebAppProxy. For back end use simple web @@ -96,7 +98,7 @@ public class TestWebAppProxyServlet { /** * Simple http server. Server should send answer with status 200 */ - @BeforeClass + @BeforeAll public static void start() throws Exception { server = new Server(0); ((QueuedThreadPool)server.getThreadPool()).setMaxThreads(20); @@ -175,8 +177,9 @@ protected void doPost(HttpServletRequest req, HttpServletResponse resp) } } - @Test(timeout=5000) - public void testWebAppProxyServlet() throws Exception { + @Test + @Timeout(5000) + void testWebAppProxyServlet() throws Exception { configuration.set(YarnConfiguration.PROXY_ADDRESS, "localhost:9090"); // overriding num of web server threads, see HttpServer.HTTP_MAXTHREADS configuration.setInt("hadoop.http.max.threads", 10); @@ -221,17 +224,17 @@ public void testWebAppProxyServlet() throws Exception { proxyConn = (HttpURLConnection) redirectUrl.openConnection(); proxyConn.setInstanceFollowRedirects(false); proxyConn.connect(); - assertEquals("The proxy returned an unexpected status code rather than" - + "redirecting the connection (302)", - HttpURLConnection.HTTP_MOVED_TEMP, proxyConn.getResponseCode()); + assertEquals(HttpURLConnection.HTTP_MOVED_TEMP, proxyConn.getResponseCode(), + "The proxy returned an unexpected status code rather than" + + "redirecting the connection (302)"); String expected = WebAppUtils.getResolvedRMWebAppURLWithScheme(configuration) - + "/cluster/failure/application_00_0"; + + "/cluster/failure/application_00_0"; String redirect = proxyConn.getHeaderField(ProxyUtils.LOCATION); - assertEquals("The proxy did not redirect the connection to the failure " - + "page of the RM", expected, redirect); + assertEquals(expected, redirect, "The proxy did not redirect the connection to the failure " + + "page of the RM"); // cannot found application 1: null appReportFetcher.answer = 1; @@ -274,7 +277,7 @@ public void testWebAppProxyServlet() throws Exception { // original tracking url appReportFetcher.answer = 5; URL clientUrl = new URL("http://localhost:" + proxyPort - + "/proxy/application_00_0/test/tez?x=y&h=p"); + + "/proxy/application_00_0/test/tez?x=y&h=p"); proxyConn = (HttpURLConnection) clientUrl.openConnection(); proxyConn.connect(); LOG.info("" + proxyConn.getURL()); @@ -286,47 +289,51 @@ public void testWebAppProxyServlet() throws Exception { } } - @Test(expected = SocketTimeoutException.class) - public void testWebAppProxyConnectionTimeout() - throws IOException, ServletException{ - HttpServletRequest request = mock(HttpServletRequest.class); - when(request.getMethod()).thenReturn("GET"); - when(request.getRemoteUser()).thenReturn("dr.who"); - when(request.getPathInfo()).thenReturn("/application_00_0"); - when(request.getHeaderNames()).thenReturn(Collections.emptyEnumeration()); + @Test + void testWebAppProxyConnectionTimeout() + throws IOException, ServletException { + assertThrows(SocketTimeoutException.class, () -> { + HttpServletRequest request = mock(HttpServletRequest.class); + when(request.getMethod()).thenReturn("GET"); + when(request.getRemoteUser()).thenReturn("dr.who"); + when(request.getPathInfo()).thenReturn("/application_00_0"); + when(request.getHeaderNames()).thenReturn(Collections.emptyEnumeration()); - HttpServletResponse response = mock(HttpServletResponse.class); - when(response.getOutputStream()).thenReturn(null); + HttpServletResponse response = mock(HttpServletResponse.class); + when(response.getOutputStream()).thenReturn(null); - WebAppProxyServlet servlet = new WebAppProxyServlet(); - YarnConfiguration conf = new YarnConfiguration(); - conf.setBoolean(YarnConfiguration.RM_PROXY_TIMEOUT_ENABLED, - true); - conf.setInt(YarnConfiguration.RM_PROXY_CONNECTION_TIMEOUT, - 1000); + WebAppProxyServlet servlet = new WebAppProxyServlet(); + YarnConfiguration conf = new YarnConfiguration(); + conf.setBoolean(YarnConfiguration.RM_PROXY_TIMEOUT_ENABLED, + true); + conf.setInt(YarnConfiguration.RM_PROXY_CONNECTION_TIMEOUT, + 1000); + + servlet.setConf(conf); - servlet.setConf(conf); + ServletConfig config = mock(ServletConfig.class); + ServletContext context = mock(ServletContext.class); + when(config.getServletContext()).thenReturn(context); - ServletConfig config = mock(ServletConfig.class); - ServletContext context = mock(ServletContext.class); - when(config.getServletContext()).thenReturn(context); + AppReportFetcherForTest appReportFetcher = + new AppReportFetcherForTest(new YarnConfiguration()); - AppReportFetcherForTest appReportFetcher = - new AppReportFetcherForTest(new YarnConfiguration()); + when(config.getServletContext() + .getAttribute(WebAppProxy.FETCHER_ATTRIBUTE)) + .thenReturn(appReportFetcher); - when(config.getServletContext() - .getAttribute(WebAppProxy.FETCHER_ATTRIBUTE)) - .thenReturn(appReportFetcher); + appReportFetcher.answer = 7; - appReportFetcher.answer = 7; + servlet.init(config); + servlet.doGet(request, response); - servlet.init(config); - servlet.doGet(request, response); + }); } - @Test(timeout=5000) - public void testAppReportForEmptyTrackingUrl() throws Exception { + @Test + @Timeout(5000) + void testAppReportForEmptyTrackingUrl() throws Exception { configuration.set(YarnConfiguration.PROXY_ADDRESS, "localhost:9090"); // overriding num of web server threads, see HttpServer.HTTP_MAXTHREADS configuration.setInt("hadoop.http.max.threads", 10); @@ -338,53 +345,51 @@ public void testAppReportForEmptyTrackingUrl() throws Exception { AppReportFetcherForTest appReportFetcher = proxy.proxy.appReportFetcher; try { - //set AHS_ENBALED = false to simulate getting the app report from RM - configuration.setBoolean(YarnConfiguration.APPLICATION_HISTORY_ENABLED, - false); - ApplicationId app = ApplicationId.newInstance(0, 0); - appReportFetcher.answer = 6; - URL url = new URL("http://localhost:" + proxyPort + - "/proxy/" + app.toString()); - HttpURLConnection proxyConn = (HttpURLConnection) url.openConnection(); - proxyConn.connect(); - try { - proxyConn.getResponseCode(); - } catch (ConnectException e) { - // Connection Exception is expected as we have set - // appReportFetcher.answer = 6, which does not set anything for - // original tracking url field in the app report. - } - String appAddressInRm = - WebAppUtils.getResolvedRMWebAppURLWithScheme(configuration) + - "/cluster" + "/app/" + app.toString(); - assertTrue("Webapp proxy servlet should have redirected to RM", - proxyConn.getURL().toString().equals(appAddressInRm)); - - //set AHS_ENBALED = true to simulate getting the app report from AHS - configuration.setBoolean(YarnConfiguration.APPLICATION_HISTORY_ENABLED, - true); - proxyConn = (HttpURLConnection) url.openConnection(); - proxyConn.connect(); - try { - proxyConn.getResponseCode(); - } catch (ConnectException e) { - // Connection Exception is expected as we have set - // appReportFetcher.answer = 6, which does not set anything for - // original tracking url field in the app report. - } - String appAddressInAhs = WebAppUtils.getHttpSchemePrefix(configuration) + - WebAppUtils.getAHSWebAppURLWithoutScheme(configuration) + - "/applicationhistory" + "/app/" + app.toString(); - assertTrue("Webapp proxy servlet should have redirected to AHS", - proxyConn.getURL().toString().equals(appAddressInAhs)); - } - finally { + //set AHS_ENBALED = false to simulate getting the app report from RM + configuration.setBoolean(YarnConfiguration.APPLICATION_HISTORY_ENABLED, + false); + ApplicationId app = ApplicationId.newInstance(0, 0); + appReportFetcher.answer = 6; + URL url = new URL("http://localhost:" + proxyPort + + "/proxy/" + app.toString()); + HttpURLConnection proxyConn = (HttpURLConnection) url.openConnection(); + proxyConn.connect(); + try { + proxyConn.getResponseCode(); + } catch (ConnectException e) { + // Connection Exception is expected as we have set + // appReportFetcher.answer = 6, which does not set anything for + // original tracking url field in the app report. + } + String appAddressInRm = + WebAppUtils.getResolvedRMWebAppURLWithScheme(configuration) + + "/cluster" + "/app/" + app.toString(); + assertEquals(proxyConn.getURL().toString(), appAddressInRm); + + //set AHS_ENBALED = true to simulate getting the app report from AHS + configuration.setBoolean(YarnConfiguration.APPLICATION_HISTORY_ENABLED, + true); + proxyConn = (HttpURLConnection) url.openConnection(); + proxyConn.connect(); + try { + proxyConn.getResponseCode(); + } catch (ConnectException e) { + // Connection Exception is expected as we have set + // appReportFetcher.answer = 6, which does not set anything for + // original tracking url field in the app report. + } + String appAddressInAhs = + WebAppUtils.getHttpSchemePrefix(configuration) + WebAppUtils.getAHSWebAppURLWithoutScheme( + configuration) + "/applicationhistory" + "/app/" + app.toString(); + assertEquals(proxyConn.getURL().toString(), appAddressInAhs); + } finally { proxy.close(); } } - @Test(timeout=5000) - public void testWebAppProxyPassThroughHeaders() throws Exception { + @Test + @Timeout(5000) + void testWebAppProxyPassThroughHeaders() throws Exception { Configuration configuration = new Configuration(); configuration.set(YarnConfiguration.PROXY_ADDRESS, "localhost:9091"); configuration.setInt("hadoop.http.max.threads", 10); @@ -424,8 +429,9 @@ public void testWebAppProxyPassThroughHeaders() throws Exception { /** * Test main method of WebAppProxyServer */ - @Test(timeout=5000) - public void testWebAppProxyServerMainMethod() throws Exception { + @Test + @Timeout(5000) + void testWebAppProxyServerMainMethod() throws Exception { WebAppProxyServer mainServer = null; Configuration conf = new YarnConfiguration(); conf.set(YarnConfiguration.PROXY_ADDRESS, "localhost:9099"); @@ -458,8 +464,9 @@ public void testWebAppProxyServerMainMethod() throws Exception { } } - @Test(timeout=5000) - public void testCheckHttpsStrictAndNotProvided() throws Exception { + @Test + @Timeout(5000) + void testCheckHttpsStrictAndNotProvided() throws Exception { HttpServletResponse resp = mock(HttpServletResponse.class); StringWriter sw = new StringWriter(); when(resp.getWriter()).thenReturn(new PrintWriter(sw)); @@ -498,8 +505,9 @@ public void testCheckHttpsStrictAndNotProvided() throws Exception { assertTrue(WebAppProxyServlet.checkHttpsStrictAndNotProvided( resp, httpLink, conf)); String s = sw.toString(); - assertTrue("Was expecting an HTML page explaining that an HTTPS tracking" + - " url must be used but found " + s, s.contains("HTTPS must be used")); + assertTrue(s.contains("HTTPS must be used"), + "Was expecting an HTML page explaining that an HTTPS tracking" + + " url must be used but found " + s); Mockito.verify(resp, Mockito.times(1)).setContentType(MimeType.HTML); } @@ -529,7 +537,7 @@ private boolean isResponseCookiePresent(HttpURLConnection proxyConn, return false; } - @AfterClass + @AfterAll public static void stop() throws Exception { try { server.stop(); @@ -584,8 +592,7 @@ protected void serviceStart() throws Exception { ProxyUriUtils.PROXY_PATH_SPEC, WebAppProxyServlet.class); appReportFetcher = new AppReportFetcherForTest(conf); - proxyServer.setAttribute(FETCHER_ATTRIBUTE, - appReportFetcher ); + proxyServer.setAttribute(FETCHER_ATTRIBUTE, appReportFetcher); proxyServer.setAttribute(IS_SECURITY_ENABLED_ATTRIBUTE, Boolean.TRUE); String proxy = WebAppUtils.getProxyHostAndPort(conf); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/test/java/org/apache/hadoop/yarn/server/webproxy/amfilter/TestAmFilter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/test/java/org/apache/hadoop/yarn/server/webproxy/amfilter/TestAmFilter.java index 857e24fdb5d09..07302e6527675 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/test/java/org/apache/hadoop/yarn/server/webproxy/amfilter/TestAmFilter.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/test/java/org/apache/hadoop/yarn/server/webproxy/amfilter/TestAmFilter.java @@ -22,45 +22,46 @@ import java.io.PrintWriter; import java.io.StringWriter; import java.net.HttpURLConnection; -import java.util.Set; -import java.util.HashSet; -import java.util.Enumeration; import java.util.Collections; -import java.util.Map; +import java.util.Enumeration; import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; import java.util.concurrent.atomic.AtomicBoolean; - -import javax.servlet.FilterConfig; -import javax.servlet.FilterChain; +import java.util.function.Supplier; import javax.servlet.Filter; +import javax.servlet.FilterChain; +import javax.servlet.FilterConfig; import javax.servlet.ServletContext; -import javax.servlet.ServletResponse; -import javax.servlet.ServletRequest; import javax.servlet.ServletException; +import javax.servlet.ServletRequest; +import javax.servlet.ServletResponse; import javax.servlet.http.Cookie; import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; -import static org.assertj.core.api.Assertions.assertThat; -import static org.junit.Assert.assertTrue; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.fail; - -import java.util.function.Supplier; -import org.apache.hadoop.http.TestHttpServer; -import org.apache.hadoop.test.GenericTestUtils; -import org.apache.hadoop.yarn.server.webproxy.ProxyUtils; -import org.apache.hadoop.yarn.server.webproxy.WebAppProxyServlet; import org.eclipse.jetty.server.Server; import org.eclipse.jetty.server.ServerConnector; import org.eclipse.jetty.servlet.ServletContextHandler; import org.eclipse.jetty.servlet.ServletHolder; import org.eclipse.jetty.util.thread.QueuedThreadPool; import org.glassfish.grizzly.servlet.HttpServletResponseImpl; -import org.junit.Test; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.Timeout; import org.mockito.Mockito; +import org.apache.hadoop.http.TestHttpServer; +import org.apache.hadoop.test.GenericTestUtils; +import org.apache.hadoop.yarn.server.webproxy.ProxyUtils; +import org.apache.hadoop.yarn.server.webproxy.WebAppProxyServlet; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.fail; + /** * Test AmIpFilter. Requests to a no declared hosts should has way through * proxy. Another requests can be filtered with (without) user name. @@ -114,9 +115,10 @@ public ServletContext getServletContext() { } } - @Test(timeout = 5000) + @Test + @Timeout(5000) @SuppressWarnings("deprecation") - public void filterNullCookies() throws Exception { + void filterNullCookies() throws Exception { HttpServletRequest request = Mockito.mock(HttpServletRequest.class); Mockito.when(request.getCookies()).thenReturn(null); @@ -145,7 +147,7 @@ public void doFilter(ServletRequest servletRequest, } @Test - public void testFindRedirectUrl() throws Exception { + void testFindRedirectUrl() throws Exception { final String rm1 = "rm1"; final String rm2 = "rm2"; // generate a valid URL @@ -159,7 +161,7 @@ public void testFindRedirectUrl() throws Exception { spy.proxyUriBases = new HashMap<>(); spy.proxyUriBases.put(rm1, rm1Url); spy.proxyUriBases.put(rm2, rm2Url); - spy.rmUrls = new String[] { rm1, rm2 }; + spy.rmUrls = new String[]{rm1, rm2}; assertThat(spy.findRedirectUrl()).isEqualTo(rm1Url); } @@ -179,8 +181,9 @@ private String startHttpServer() throws Exception { return server.getURI().toString() + servletPath; } - @Test(timeout = 2000) - public void testProxyUpdate() throws Exception { + @Test + @Timeout(2000) + void testProxyUpdate() throws Exception { Map params = new HashMap<>(); params.put(AmIpFilter.PROXY_HOSTS, proxyHost); params.put(AmIpFilter.PROXY_URI_BASES, proxyUri); @@ -220,9 +223,10 @@ public Boolean get() { /** * Test AmIpFilter */ - @Test(timeout = 10000) + @Test + @Timeout(10000) @SuppressWarnings("deprecation") - public void testFilter() throws Exception { + void testFilter() throws Exception { Map params = new HashMap(); params.put(AmIpFilter.PROXY_HOST, proxyHost); params.put(AmIpFilter.PROXY_URI_BASE, proxyUri); @@ -286,7 +290,7 @@ public void doFilter(ServletRequest servletRequest, assertTrue(doFilterRequest.contains("HttpServletRequest")); // cookie added - Cookie[] cookies = new Cookie[] { + Cookie[] cookies = new Cookie[]{ new Cookie(WebAppProxyServlet.PROXY_USER_COOKIE_NAME, "user") }; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/test/java/org/apache/hadoop/yarn/server/webproxy/amfilter/TestAmFilterInitializer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/test/java/org/apache/hadoop/yarn/server/webproxy/amfilter/TestAmFilterInitializer.java index 97625ac0882b0..0aec45fce8a57 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/test/java/org/apache/hadoop/yarn/server/webproxy/amfilter/TestAmFilterInitializer.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/test/java/org/apache/hadoop/yarn/server/webproxy/amfilter/TestAmFilterInitializer.java @@ -22,9 +22,10 @@ import java.util.Collections; import java.util.List; import java.util.Map; -import org.junit.Before; -import org.junit.Test; -import static org.junit.Assert.*; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.http.FilterContainer; import org.apache.hadoop.http.HttpConfig; @@ -32,12 +33,15 @@ import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.webapp.util.WebAppUtils; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; + /** * Test class for {@Link AmFilterInitializer}. */ public class TestAmFilterInitializer { - @Before + @BeforeEach public void setUp() throws Exception { NetUtils.addStaticResolution("host1", "172.0.0.1"); NetUtils.addStaticResolution("host2", "172.0.0.1"); @@ -48,7 +52,7 @@ public void setUp() throws Exception { } @Test - public void testInitFilter() { + void testInitFilter() { // Check PROXY_ADDRESS MockFilterContainer con = new MockFilterContainer(); Configuration conf = new Configuration(false); @@ -60,7 +64,7 @@ public void testInitFilter() { assertEquals("host1", con.givenParameters.get(AmIpFilter.PROXY_HOSTS)); assertEquals("http://host1:1000/foo", con.givenParameters.get(AmIpFilter.PROXY_URI_BASES)); - assertEquals(null, con.givenParameters.get(AmFilterInitializer.RM_HA_URLS)); + assertNull(con.givenParameters.get(AmFilterInitializer.RM_HA_URLS)); // Check a single RM_WEBAPP_ADDRESS con = new MockFilterContainer(); @@ -73,7 +77,7 @@ public void testInitFilter() { assertEquals("host2", con.givenParameters.get(AmIpFilter.PROXY_HOSTS)); assertEquals("http://host2:2000/foo", con.givenParameters.get(AmIpFilter.PROXY_URI_BASES)); - assertEquals(null, con.givenParameters.get(AmFilterInitializer.RM_HA_URLS)); + assertNull(con.givenParameters.get(AmFilterInitializer.RM_HA_URLS)); // Check multiple RM_WEBAPP_ADDRESSes (RM HA) con = new MockFilterContainer(); @@ -134,7 +138,7 @@ public void testInitFilter() { } @Test - public void testGetProxyHostsAndPortsForAmFilter() { + void testGetProxyHostsAndPortsForAmFilter() { // Check no configs given Configuration conf = new Configuration(false); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/test/java/org/apache/hadoop/yarn/server/webproxy/amfilter/TestSecureAmFilter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/test/java/org/apache/hadoop/yarn/server/webproxy/amfilter/TestSecureAmFilter.java index 5bbfc8fafa0ce..f6cc054361d94 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/test/java/org/apache/hadoop/yarn/server/webproxy/amfilter/TestSecureAmFilter.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/test/java/org/apache/hadoop/yarn/server/webproxy/amfilter/TestSecureAmFilter.java @@ -21,28 +21,30 @@ import java.io.File; import java.net.URI; import java.net.URL; -import java.util.Set; -import java.util.HashSet; import java.util.HashMap; +import java.util.HashSet; +import java.util.Set; -import static org.assertj.core.api.Assertions.assertThat; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.mockito.Mockito; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.CommonConfigurationKeysPublic; import org.apache.hadoop.http.HttpServer2; import org.apache.hadoop.minikdc.MiniKdc; import org.apache.hadoop.net.NetUtils; +import org.apache.hadoop.security.AuthenticationFilterInitializer; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.security.authentication.KerberosTestUtils; -import org.apache.hadoop.security.AuthenticationFilterInitializer; import org.apache.hadoop.security.authorize.AccessControlList; import org.apache.hadoop.yarn.conf.YarnConfiguration; -import org.junit.AfterClass; -import org.junit.BeforeClass; -import org.junit.Test; -import org.mockito.Mockito; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.fail; /** * Test AmIpFilter. Requests to a no declared hosts should has way through @@ -63,7 +65,7 @@ public class TestSecureAmFilter { private static boolean miniKDCStarted = false; private static MiniKdc testMiniKDC; - @BeforeClass + @BeforeAll public static void setUp() { rmconf.setBoolean(YarnConfiguration.YARN_ACL_ENABLE, true); rmconf.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION, @@ -88,11 +90,11 @@ public static void setUp() { testMiniKDC = new MiniKdc(MiniKdc.createConf(), TEST_ROOT_DIR); setupKDC(); } catch (Exception e) { - assertTrue("Couldn't create MiniKDC", false); + fail("Couldn't create MiniKDC"); } } - @AfterClass + @AfterAll public static void tearDown() { if (testMiniKDC != null) { testMiniKDC.stop(); @@ -125,7 +127,7 @@ protected Set getProxyAddresses() { } @Test - public void testFindRedirectUrl() throws Exception { + void testFindRedirectUrl() throws Exception { final String rm1 = "rm1"; final String rm2 = "rm2"; // generate a valid URL @@ -139,7 +141,7 @@ public void testFindRedirectUrl() throws Exception { spy.proxyUriBases = new HashMap<>(); spy.proxyUriBases.put(rm1, rm1Url); spy.proxyUriBases.put(rm2, rm2Url); - spy.rmUrls = new String[] {rm1, rm2}; + spy.rmUrls = new String[]{rm1, rm2}; assertTrue(spy.isValidUrl(rm1Url)); assertFalse(spy.isValidUrl(rm2Url)); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/Federation.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/Federation.md index c7836e75475b1..31a071837c9e2 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/Federation.md +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/Federation.md @@ -258,6 +258,16 @@ Optional: |`yarn.federation.cache-ttl.secs` | `60` | The Router caches informations, and this is the time to leave before the cache is invalidated. | |`yarn.router.webapp.interceptor-class.pipeline` | `org.apache.hadoop.yarn.server.router.webapp.FederationInterceptorREST` | A comma-separated list of interceptor classes to be run at the router when interfacing with the client via REST interface. The last step of this pipeline must be the Federation Interceptor REST. | +Security: + +Kerberos supported in federation. + +| Property | Example | Description | +|:---- |:---- | +| `yarn.router.keytab.file` | | The keytab file used by router to login as its service principal. The principal name is configured with 'yarn.router.kerberos.principal'.| +| `yarn.router.kerberos.principal` | | The Router service principal. This is typically set to router/_HOST@REALM.TLD. Each Router will substitute _HOST with its own fully qualified hostname at startup. The _HOST placeholder allows using the same configuration setting on all Routers in setup. | +| `yarn.router.kerberos.principal.hostname` | | Optional. The hostname for the Router containing this configuration file. Will be different for each machine. Defaults to current hostname. | + ###ON NMs: These are extra configurations that should appear in the **conf/yarn-site.xml** at each NodeManager.