diff --git a/LICENSE-binary b/LICENSE-binary index 824dbee29f6a9..a9faf26db5a78 100644 --- a/LICENSE-binary +++ b/LICENSE-binary @@ -250,7 +250,6 @@ commons-codec:commons-codec:1.11 commons-collections:commons-collections:3.2.2 commons-daemon:commons-daemon:1.0.13 commons-io:commons-io:2.8.0 -commons-lang:commons-lang:2.6 commons-logging:commons-logging:1.1.3 commons-net:commons-net:3.6 de.ruedigermoeller:fst:2.50 @@ -290,7 +289,7 @@ org.apache.commons:commons-compress:1.21 org.apache.commons:commons-configuration2:2.1.1 org.apache.commons:commons-csv:1.0 org.apache.commons:commons-digester:1.8.1 -org.apache.commons:commons-lang3:3.7 +org.apache.commons:commons-lang3:3.12.0 org.apache.commons:commons-math3:3.1.1 org.apache.commons:commons-text:1.4 org.apache.commons:commons-validator:1.6 diff --git a/dev-support/Jenkinsfile b/dev-support/Jenkinsfile index 7896810d017d5..0ec32e385d275 100644 --- a/dev-support/Jenkinsfile +++ b/dev-support/Jenkinsfile @@ -15,13 +15,10 @@ // specific language governing permissions and limitations // under the License. -def getGithubAndJiraCreds() { +def getGithubCreds() { return [usernamePassword(credentialsId: 'apache-hadoop-at-github.com', passwordVariable: 'GITHUB_TOKEN', - usernameVariable: 'GITHUB_USER'), - usernamePassword(credentialsId: 'hadoopqa-at-asf-jira', - passwordVariable: 'JIRA_PASSWORD', - usernameVariable: 'JIRA_USER')] + usernameVariable: 'GITHUB_USER')] } // Publish JUnit results only if there are XML files under surefire-reports @@ -130,7 +127,7 @@ pipeline { } steps { - withCredentials(getGithubAndJiraCreds()) { + withCredentials(getGithubCreds()) { sh '''#!/usr/bin/env bash chmod u+x "${SOURCEDIR}/dev-support/jenkins.sh" @@ -176,7 +173,7 @@ pipeline { } steps { - withCredentials(getGithubAndJiraCreds()) { + withCredentials(getGithubCreds()) { sh '''#!/usr/bin/env bash chmod u+x "${SOURCEDIR}/dev-support/jenkins.sh" @@ -222,7 +219,7 @@ pipeline { } steps { - withCredentials(getGithubAndJiraCreds()) { + withCredentials(getGithubCreds()) { sh '''#!/usr/bin/env bash chmod u+x "${SOURCEDIR}/dev-support/jenkins.sh" @@ -265,7 +262,7 @@ pipeline { } steps { - withCredentials(getGithubAndJiraCreds()) { + withCredentials(getGithubCreds()) { sh '''#!/usr/bin/env bash chmod u+x "${SOURCEDIR}/dev-support/jenkins.sh" diff --git a/dev-support/jenkins.sh b/dev-support/jenkins.sh index 7ead90b5da24d..1bb080d19cabc 100644 --- a/dev-support/jenkins.sh +++ b/dev-support/jenkins.sh @@ -149,10 +149,6 @@ function run_ci() { # enable writing back to Github YETUS_ARGS+=("--github-token=${GITHUB_TOKEN}") - # enable writing back to ASF JIRA - YETUS_ARGS+=("--jira-password=${JIRA_PASSWORD}") - YETUS_ARGS+=("--jira-user=${JIRA_USER}") - # auto-kill any surefire stragglers during unit test runs YETUS_ARGS+=("--reapermode=kill") @@ -173,7 +169,7 @@ function run_ci() { YETUS_ARGS+=("--build-url-artifacts=artifact/out") # plugins to enable - YETUS_ARGS+=("--plugins=all") + YETUS_ARGS+=("--plugins=all,-jira") # don't let these tests cause -1s because we aren't really paying that # much attention to them diff --git a/hadoop-build-tools/src/main/resources/checkstyle/checkstyle.xml b/hadoop-build-tools/src/main/resources/checkstyle/checkstyle.xml index 73012dd47a864..ca8d137dd5e49 100644 --- a/hadoop-build-tools/src/main/resources/checkstyle/checkstyle.xml +++ b/hadoop-build-tools/src/main/resources/checkstyle/checkstyle.xml @@ -122,9 +122,8 @@ - - - + + diff --git a/hadoop-client-modules/hadoop-client-api/pom.xml b/hadoop-client-modules/hadoop-client-api/pom.xml index d0d62f5277b9b..9b70f8c8b01e0 100644 --- a/hadoop-client-modules/hadoop-client-api/pom.xml +++ b/hadoop-client-modules/hadoop-client-api/pom.xml @@ -179,6 +179,8 @@ com/sun/security/**/* com/sun/jndi/**/* com/sun/management/**/* + com/ibm/security/* + com/ibm/security/**/* diff --git a/hadoop-client-modules/hadoop-client-minicluster/pom.xml b/hadoop-client-modules/hadoop-client-minicluster/pom.xml index c9ce6f297d22a..066b518652441 100644 --- a/hadoop-client-modules/hadoop-client-minicluster/pom.xml +++ b/hadoop-client-modules/hadoop-client-minicluster/pom.xml @@ -919,6 +919,8 @@ com/sun/security/**/* com/sun/jndi/**/* com/sun/management/**/* + com/ibm/security/* + com/ibm/security/**/* diff --git a/hadoop-client-modules/hadoop-client-runtime/pom.xml b/hadoop-client-modules/hadoop-client-runtime/pom.xml index b6a71e50c41ab..6dae4f61969de 100644 --- a/hadoop-client-modules/hadoop-client-runtime/pom.xml +++ b/hadoop-client-modules/hadoop-client-runtime/pom.xml @@ -288,6 +288,8 @@ com/sun/security/**/* com/sun/jndi/**/* com/sun/management/**/* + com/ibm/security/* + com/ibm/security/**/* diff --git a/hadoop-cloud-storage-project/hadoop-cos/pom.xml b/hadoop-cloud-storage-project/hadoop-cos/pom.xml index 0e801c4e439ac..fa47e354c7998 100644 --- a/hadoop-cloud-storage-project/hadoop-cos/pom.xml +++ b/hadoop-cloud-storage-project/hadoop-cos/pom.xml @@ -96,38 +96,6 @@ - - org.apache.maven.plugins - maven-enforcer-plugin - - - de.skuzzle.enforcer - restrict-imports-enforcer-rule - ${restrict-imports.enforcer.version} - - - - - banned-illegal-imports - process-sources - - enforce - - - - - true - Use hadoop-common provided VisibleForTesting rather than the one provided by Guava - - org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting - com.google.common.annotations.VisibleForTesting - - - - - - - diff --git a/hadoop-cloud-storage-project/hadoop-huaweicloud/pom.xml b/hadoop-cloud-storage-project/hadoop-huaweicloud/pom.xml index 7a9ee61836e1c..43360c11cd9d2 100755 --- a/hadoop-cloud-storage-project/hadoop-huaweicloud/pom.xml +++ b/hadoop-cloud-storage-project/hadoop-huaweicloud/pom.xml @@ -92,38 +92,6 @@ - - org.apache.maven.plugins - maven-enforcer-plugin - - - de.skuzzle.enforcer - restrict-imports-enforcer-rule - ${restrict-imports.enforcer.version} - - - - - banned-illegal-imports - process-sources - - enforce - - - - - true - Use hadoop-common provided VisibleForTesting rather than the one provided by Guava - - org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting - com.google.common.annotations.VisibleForTesting - - - - - - - diff --git a/hadoop-common-project/hadoop-auth/pom.xml b/hadoop-common-project/hadoop-auth/pom.xml index d21d6273fffd4..f7f09025b6983 100644 --- a/hadoop-common-project/hadoop-auth/pom.xml +++ b/hadoop-common-project/hadoop-auth/pom.xml @@ -271,8 +271,6 @@ true Use hadoop-common provided implementations rather than the one provided by Guava - org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting - com.google.common.annotations.VisibleForTesting org.apache.hadoop.thirdparty.com.google.common.base.Preconditions com.google.common.base.Preconditions diff --git a/hadoop-common-project/hadoop-common/pom.xml b/hadoop-common-project/hadoop-common/pom.xml index 6db6c2f5483a2..c0235b4db9e9c 100644 --- a/hadoop-common-project/hadoop-common/pom.xml +++ b/hadoop-common-project/hadoop-common/pom.xml @@ -674,8 +674,6 @@ true Use hadoop-common provided implementations rather than the one provided by Guava - org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting - com.google.common.annotations.VisibleForTesting org.apache.hadoop.thirdparty.com.google.common.base.Preconditions com.google.common.base.Preconditions diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/PathIOException.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/PathIOException.java index deb3880ee4195..f32f2a93544bf 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/PathIOException.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/PathIOException.java @@ -64,7 +64,13 @@ public PathIOException(String path, String error) { this.path = path; } - protected PathIOException(String path, String error, Throwable cause) { + /** + * Use a subclass of PathIOException if possible. + * @param path for the exception + * @param error custom string to use an the error text + * @param cause cause of exception. + */ + public PathIOException(String path, String error, Throwable cause) { super(error, cause); this.path = path; } diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java index 230cad5d3aa46..7a391c4994cdc 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java @@ -369,11 +369,20 @@ public static int getCallRetryCount() { } /** Returns the remote side ip address when invoked inside an RPC - * Returns null incase of an error. + * Returns null in case of an error. */ public static InetAddress getRemoteIp() { Call call = CurCall.get(); - return (call != null ) ? call.getHostInetAddress() : null; + return (call != null) ? call.getHostInetAddress() : null; + } + + /** + * Returns the remote side port when invoked inside an RPC + * Returns 0 in case of an error. + */ + public static int getRemotePort() { + Call call = CurCall.get(); + return (call != null) ? call.getRemotePort() : 0; } /** @@ -409,7 +418,7 @@ public static byte[] getClientId() { Call call = CurCall.get(); return call != null ? call.clientId : RpcConstants.DUMMY_CLIENT_ID; } - + /** Returns remote address as a string when invoked inside an RPC. * Returns null in case of an error. */ @@ -447,7 +456,7 @@ public static int getPriorityLevel() { return call != null? call.getPriorityLevel() : 0; } - private String bindAddress; + private String bindAddress; private int port; // port we listen on private int handlerCount; // number of handler threads private int readThreads; // number of read threads @@ -455,7 +464,7 @@ public static int getPriorityLevel() { private Class rpcRequestClass; // class used for deserializing the rpc request final protected RpcMetrics rpcMetrics; final protected RpcDetailedMetrics rpcDetailedMetrics; - + private Configuration conf; private String portRangeConfig = null; private SecretManager secretManager; @@ -652,17 +661,17 @@ public static void bind(ServerSocket socket, InetSocketAddress address, } } - @org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting + @VisibleForTesting int getPriorityLevel(Schedulable e) { return callQueue.getPriorityLevel(e); } - @org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting + @VisibleForTesting int getPriorityLevel(UserGroupInformation ugi) { return callQueue.getPriorityLevel(ugi); } - @org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting + @VisibleForTesting void setPriorityLevel(UserGroupInformation ugi, int priority) { callQueue.setPriorityLevel(ugi, priority); } @@ -973,6 +982,9 @@ public UserGroupInformation getRemoteUser() { public InetAddress getHostInetAddress() { return null; } + public int getRemotePort() { + return 0; + } public String getHostAddress() { InetAddress addr = getHostInetAddress(); return (addr != null) ? addr.getHostAddress() : null; @@ -1130,6 +1142,11 @@ public InetAddress getHostInetAddress() { return connection.getHostInetAddress(); } + @Override + public int getRemotePort() { + return connection.getRemotePort(); + } + @Override public Void run() throws Exception { if (!connection.channel.isOpen()) { @@ -2011,6 +2028,10 @@ public int getIngressPort() { return ingressPort; } + public int getRemotePort() { + return remotePort; + } + public InetAddress getHostInetAddress() { return addr; } diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/Groups.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/Groups.java index bc2ea458b33f3..70c633cdf8a23 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/Groups.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/Groups.java @@ -493,4 +493,9 @@ public static synchronized Groups getUserToGroupsMappingService( GROUPS = new Groups(conf); return GROUPS; } + + @VisibleForTesting + public static void reset() { + GROUPS = null; + } } diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/JsonSerialization.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/JsonSerialization.java index 81a356c7a835a..2b794f9c38245 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/JsonSerialization.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/JsonSerialization.java @@ -18,6 +18,7 @@ package org.apache.hadoop.util; +import javax.annotation.Nullable; import java.io.EOFException; import java.io.File; import java.io.FileNotFoundException; @@ -42,8 +43,13 @@ import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.fs.FSDataInputStream; +import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.FutureDataInputStreamBuilder; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.PathIOException; + +import static org.apache.hadoop.util.functional.FutureIO.awaitFuture; /** * Support for marshalling objects to and from JSON. @@ -228,30 +234,44 @@ public T fromInstance(T instance) throws IOException { /** * Load from a Hadoop filesystem. - * There's a check for data availability after the file is open, by - * raising an EOFException if stream.available == 0. - * This allows for a meaningful exception without the round trip overhead - * of a getFileStatus call before opening the file. It may be brittle - * against an FS stream which doesn't return a value here, but the - * standard filesystems all do. - * JSON parsing and mapping problems - * are converted to IOEs. * @param fs filesystem * @param path path * @return a loaded object - * @throws IOException IO or JSON parse problems + * @throws PathIOException JSON parse problem + * @throws IOException IO problems */ public T load(FileSystem fs, Path path) throws IOException { - try (FSDataInputStream dataInputStream = fs.open(path)) { - // throw an EOF exception if there is no data available. - if (dataInputStream.available() == 0) { - throw new EOFException("No data in " + path); - } + return load(fs, path, null); + } + + /** + * Load from a Hadoop filesystem. + * If a file status is supplied, it's passed in to the openFile() + * call so that FS implementations can optimize their opening. + * @param fs filesystem + * @param path path + * @param status status of the file to open. + * @return a loaded object + * @throws PathIOException JSON parse problem + * @throws EOFException file status references an empty file + * @throws IOException IO problems + */ + public T load(FileSystem fs, Path path, @Nullable FileStatus status) + throws IOException { + + if (status != null && status.getLen() == 0) { + throw new EOFException("No data in " + path); + } + FutureDataInputStreamBuilder builder = fs.openFile(path); + if (status != null) { + builder.withFileStatus(status); + } + try (FSDataInputStream dataInputStream = + awaitFuture(builder.build())) { return fromJsonStream(dataInputStream); } catch (JsonProcessingException e) { - throw new IOException( - String.format("Failed to read JSON file \"%s\": %s", path, e), - e); + throw new PathIOException(path.toString(), + "Failed to read JSON file " + e, e); } } diff --git a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml index f2c04427d2dff..27c86bbc9ac8b 100644 --- a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml +++ b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml @@ -2205,7 +2205,16 @@ - + + fs.s3a.downgrade.syncable.exceptions + true + + Warn but continue when applications use Syncable.hsync when writing + to S3A. + + + + fs.AbstractFileSystem.wasb.impl org.apache.hadoop.fs.azure.Wasb diff --git a/hadoop-common-project/hadoop-common/src/site/markdown/release/2.10.0/CHANGELOG.2.10.0.md b/hadoop-common-project/hadoop-common/src/site/markdown/release/2.10.0/CHANGELOG.2.10.0.md index 03af8e5716e36..d8dd2daca09cc 100644 --- a/hadoop-common-project/hadoop-common/src/site/markdown/release/2.10.0/CHANGELOG.2.10.0.md +++ b/hadoop-common-project/hadoop-common/src/site/markdown/release/2.10.0/CHANGELOG.2.10.0.md @@ -16,9 +16,9 @@ # See the License for the specific language governing permissions and # limitations under the License. --> -# Apache Hadoop Changelog +# "Apache Hadoop" Changelog -## Release 2.10.0 - Unreleased (as of 2018-09-02) +## Release 2.10.0 - 2019-10-22 ### INCOMPATIBLE CHANGES: @@ -27,6 +27,8 @@ | [HDFS-12883](https://issues.apache.org/jira/browse/HDFS-12883) | RBF: Document Router and State Store metrics | Major | documentation | Yiqun Lin | Yiqun Lin | | [HDFS-12895](https://issues.apache.org/jira/browse/HDFS-12895) | RBF: Add ACL support for mount table | Major | . | Yiqun Lin | Yiqun Lin | | [HDFS-13099](https://issues.apache.org/jira/browse/HDFS-13099) | RBF: Use the ZooKeeper as the default State Store | Minor | documentation | Yiqun Lin | Yiqun Lin | +| [HADOOP-16055](https://issues.apache.org/jira/browse/HADOOP-16055) | Upgrade AWS SDK to 1.11.271 in branch-2 | Blocker | fs/s3 | Akira Ajisaka | Akira Ajisaka | +| [HADOOP-16053](https://issues.apache.org/jira/browse/HADOOP-16053) | Backport HADOOP-14816 to branch-2 | Major | build | Akira Ajisaka | Akira Ajisaka | ### IMPORTANT ISSUES: @@ -42,6 +44,9 @@ |:---- |:---- | :--- |:---- |:---- |:---- | | [HDFS-13283](https://issues.apache.org/jira/browse/HDFS-13283) | Percentage based Reserved Space Calculation for DataNode | Major | datanode, hdfs | Lukas Majercak | Lukas Majercak | | [HDFS-13553](https://issues.apache.org/jira/browse/HDFS-13553) | RBF: Support global quota | Major | . | Íñigo Goiri | Yiqun Lin | +| [HADOOP-15950](https://issues.apache.org/jira/browse/HADOOP-15950) | Failover for LdapGroupsMapping | Major | common, security | Lukas Majercak | Lukas Majercak | +| [YARN-9761](https://issues.apache.org/jira/browse/YARN-9761) | Allow overriding application submissions based on server side configs | Major | . | Jonathan Hung | pralabhkumar | +| [YARN-9760](https://issues.apache.org/jira/browse/YARN-9760) | Support configuring application priorities on a workflow level | Major | . | Jonathan Hung | Varun Saxena | ### IMPROVEMENTS: @@ -53,10 +58,11 @@ | [HADOOP-14960](https://issues.apache.org/jira/browse/HADOOP-14960) | Add GC time percentage monitor/alerter | Major | . | Misha Dmitriev | Misha Dmitriev | | [HADOOP-15023](https://issues.apache.org/jira/browse/HADOOP-15023) | ValueQueue should also validate (lowWatermark \* numValues) \> 0 on construction | Minor | . | Xiao Chen | Xiao Chen | | [YARN-6851](https://issues.apache.org/jira/browse/YARN-6851) | Capacity Scheduler: document configs for controlling # containers allowed to be allocated per node heartbeat | Minor | . | Wei Yan | Wei Yan | -| [YARN-7495](https://issues.apache.org/jira/browse/YARN-7495) | Improve robustness of the AggregatedLogDeletionService | Major | log-aggregation | Jonathan Eagles | Jonathan Eagles | +| [YARN-7495](https://issues.apache.org/jira/browse/YARN-7495) | Improve robustness of the AggregatedLogDeletionService | Major | log-aggregation | Jonathan Turner Eagles | Jonathan Turner Eagles | | [HADOOP-15056](https://issues.apache.org/jira/browse/HADOOP-15056) | Fix TestUnbuffer#testUnbufferException failure | Minor | test | Jack Bearden | Jack Bearden | | [HADOOP-15012](https://issues.apache.org/jira/browse/HADOOP-15012) | Add readahead, dropbehind, and unbuffer to StreamCapabilities | Major | fs | John Zhuge | John Zhuge | | [HADOOP-15104](https://issues.apache.org/jira/browse/HADOOP-15104) | AliyunOSS: change the default value of max error retry | Major | fs/oss | wujinhu | wujinhu | +| [YARN-7274](https://issues.apache.org/jira/browse/YARN-7274) | Ability to disable elasticity at leaf queue level | Major | capacityscheduler | Scott Brokaw | Zian Chen | | [YARN-7642](https://issues.apache.org/jira/browse/YARN-7642) | Add test case to verify context update after container promotion or demotion with or without auto update | Minor | nodemanager | Weiwei Yang | Weiwei Yang | | [HADOOP-15111](https://issues.apache.org/jira/browse/HADOOP-15111) | AliyunOSS: backport HADOOP-14993 to branch-2 | Major | fs/oss | Genmao Yu | Genmao Yu | | [HDFS-12818](https://issues.apache.org/jira/browse/HDFS-12818) | Support multiple storages in DataNodeCluster / SimulatedFSDataset | Minor | datanode, test | Erik Krogen | Erik Krogen | @@ -88,14 +94,16 @@ | [HDFS-13492](https://issues.apache.org/jira/browse/HDFS-13492) | Limit httpfs binds to certain IP addresses in branch-2 | Major | httpfs | Wei-Chiu Chuang | Wei-Chiu Chuang | | [HDFS-12981](https://issues.apache.org/jira/browse/HDFS-12981) | renameSnapshot a Non-Existent snapshot to itself should throw error | Minor | hdfs | Sailesh Patel | Kitti Nanasi | | [HADOOP-15441](https://issues.apache.org/jira/browse/HADOOP-15441) | Log kms url and token service at debug level. | Minor | . | Wei-Chiu Chuang | Gabor Bota | -| [YARN-8249](https://issues.apache.org/jira/browse/YARN-8249) | Few REST api's in RMWebServices are missing static user check | Critical | webapp, yarn | Sunil Govindan | Sunil Govindan | +| [HDFS-13544](https://issues.apache.org/jira/browse/HDFS-13544) | Improve logging for JournalNode in federated cluster | Major | federation, hdfs | Hanisha Koneru | Hanisha Koneru | +| [YARN-8249](https://issues.apache.org/jira/browse/YARN-8249) | Few REST api's in RMWebServices are missing static user check | Critical | webapp, yarn | Sunil G | Sunil G | | [HADOOP-15486](https://issues.apache.org/jira/browse/HADOOP-15486) | Make NetworkTopology#netLock fair | Major | net | Nanda kumar | Nanda kumar | | [HADOOP-15449](https://issues.apache.org/jira/browse/HADOOP-15449) | Increase default timeout of ZK session to avoid frequent NameNode failover | Critical | common | Karthik Palanisamy | Karthik Palanisamy | | [HDFS-13602](https://issues.apache.org/jira/browse/HDFS-13602) | Add checkOperation(WRITE) checks in FSNamesystem | Major | ha, namenode | Erik Krogen | Chao Sun | | [HDFS-13644](https://issues.apache.org/jira/browse/HDFS-13644) | Backport HDFS-10376 to branch-2 | Major | . | Yiqun Lin | Zsolt Venczel | | [HDFS-13653](https://issues.apache.org/jira/browse/HDFS-13653) | Make dfs.client.failover.random.order a per nameservice configuration | Major | federation | Ekanth Sethuramalingam | Ekanth Sethuramalingam | +| [HDFS-13686](https://issues.apache.org/jira/browse/HDFS-13686) | Add overall metrics for FSNamesystemLock | Major | hdfs, namenode | Lukas Majercak | Lukas Majercak | | [HDFS-13714](https://issues.apache.org/jira/browse/HDFS-13714) | Fix TestNameNodePrunesMissingStorages test failures on Windows | Major | hdfs, namenode, test | Lukas Majercak | Lukas Majercak | -| [HDFS-13719](https://issues.apache.org/jira/browse/HDFS-13719) | Docs around dfs.image.transfer.timeout are misleading | Major | . | Kitti Nanasi | Kitti Nanasi | +| [HDFS-13719](https://issues.apache.org/jira/browse/HDFS-13719) | Docs around dfs.image.transfer.timeout are misleading | Major | documentation | Kitti Nanasi | Kitti Nanasi | | [HDFS-11060](https://issues.apache.org/jira/browse/HDFS-11060) | make DEFAULT\_MAX\_CORRUPT\_FILEBLOCKS\_RETURNED configurable | Minor | hdfs | Lantao Jin | Lantao Jin | | [YARN-8155](https://issues.apache.org/jira/browse/YARN-8155) | Improve ATSv2 client logging in RM and NM publisher | Major | . | Rohith Sharma K S | Abhishek Modi | | [HDFS-13814](https://issues.apache.org/jira/browse/HDFS-13814) | Remove super user privilege requirement for NameNode.getServiceStatus | Minor | namenode | Chao Sun | Chao Sun | @@ -105,17 +113,84 @@ | [HADOOP-15689](https://issues.apache.org/jira/browse/HADOOP-15689) | Add "\*.patch" into .gitignore file of branch-2 | Major | . | Rui Gao | Rui Gao | | [HDFS-13831](https://issues.apache.org/jira/browse/HDFS-13831) | Make block increment deletion number configurable | Major | . | Yiqun Lin | Ryan Wu | | [YARN-8051](https://issues.apache.org/jira/browse/YARN-8051) | TestRMEmbeddedElector#testCallbackSynchronization is flakey | Major | test | Robert Kanter | Robert Kanter | +| [HADOOP-15547](https://issues.apache.org/jira/browse/HADOOP-15547) | WASB: improve listStatus performance | Major | fs/azure | Thomas Marqardt | Thomas Marqardt | +| [HDFS-13857](https://issues.apache.org/jira/browse/HDFS-13857) | RBF: Choose to enable the default nameservice to read/write files | Major | federation, hdfs | yanghuafeng | yanghuafeng | +| [HDFS-13812](https://issues.apache.org/jira/browse/HDFS-13812) | Fix the inconsistent default refresh interval on Caching documentation | Trivial | documentation | David Mollitor | Hrishikesh Gadre | +| [HADOOP-15657](https://issues.apache.org/jira/browse/HADOOP-15657) | Registering MutableQuantiles via Metric annotation | Major | metrics | Sushil Ks | Sushil Ks | +| [HDFS-13902](https://issues.apache.org/jira/browse/HDFS-13902) | Add JMX, conf and stacks menus to the datanode page | Minor | datanode | fengchuang | fengchuang | +| [HADOOP-15726](https://issues.apache.org/jira/browse/HADOOP-15726) | Create utility to limit frequency of log statements | Major | common, util | Erik Krogen | Erik Krogen | +| [YARN-7974](https://issues.apache.org/jira/browse/YARN-7974) | Allow updating application tracking url after registration | Major | . | Jonathan Hung | Jonathan Hung | +| [YARN-8750](https://issues.apache.org/jira/browse/YARN-8750) | Refactor TestQueueMetrics | Minor | resourcemanager | Szilard Nemeth | Szilard Nemeth | +| [YARN-8896](https://issues.apache.org/jira/browse/YARN-8896) | Limit the maximum number of container assignments per heartbeat | Major | . | Weiwei Yang | Zhankun Tang | +| [HADOOP-15804](https://issues.apache.org/jira/browse/HADOOP-15804) | upgrade to commons-compress 1.18 | Major | . | PJ Fanning | Akira Ajisaka | +| [YARN-8915](https://issues.apache.org/jira/browse/YARN-8915) | Update the doc about the default value of "maximum-container-assignments" for capacity scheduler | Minor | . | Zhankun Tang | Zhankun Tang | +| [YARN-7225](https://issues.apache.org/jira/browse/YARN-7225) | Add queue and partition info to RM audit log | Major | resourcemanager | Jonathan Hung | Eric Payne | +| [HADOOP-15919](https://issues.apache.org/jira/browse/HADOOP-15919) | AliyunOSS: Enable Yarn to use OSS | Major | fs/oss | wujinhu | wujinhu | +| [HADOOP-15943](https://issues.apache.org/jira/browse/HADOOP-15943) | AliyunOSS: add missing owner & group attributes for oss FileStatus | Major | fs/oss | wujinhu | wujinhu | +| [YARN-9036](https://issues.apache.org/jira/browse/YARN-9036) | Escape newlines in health report in YARN UI | Major | . | Jonathan Hung | Keqiu Hu | +| [YARN-9085](https://issues.apache.org/jira/browse/YARN-9085) | Add Guaranteed and MaxCapacity to CSQueueMetrics | Major | . | Jonathan Hung | Jonathan Hung | +| [HDFS-14171](https://issues.apache.org/jira/browse/HDFS-14171) | Performance improvement in Tailing EditLog | Major | namenode | Kenneth Yang | Kenneth Yang | +| [HADOOP-15481](https://issues.apache.org/jira/browse/HADOOP-15481) | Emit FairCallQueue stats as metrics | Major | metrics, rpc-server | Erik Krogen | Christopher Gregorian | +| [HADOOP-15617](https://issues.apache.org/jira/browse/HADOOP-15617) | Node.js and npm package loading in the Dockerfile failing on branch-2 | Major | build | Allen Wittenauer | Akhil PB | +| [HADOOP-16089](https://issues.apache.org/jira/browse/HADOOP-16089) | AliyunOSS: update oss-sdk version to 3.4.1 | Major | fs/oss | wujinhu | wujinhu | +| [YARN-7171](https://issues.apache.org/jira/browse/YARN-7171) | RM UI should sort memory / cores numerically | Major | . | Eric Maynard | Ahmed Hussein | +| [YARN-9282](https://issues.apache.org/jira/browse/YARN-9282) | Typo in javadoc of class LinuxContainerExecutor: hadoop.security.authetication should be 'authentication' | Trivial | . | Szilard Nemeth | Charan Hebri | +| [HADOOP-16126](https://issues.apache.org/jira/browse/HADOOP-16126) | ipc.Client.stop() may sleep too long to wait for all connections | Major | ipc | Tsz-wo Sze | Tsz-wo Sze | +| [HDFS-14247](https://issues.apache.org/jira/browse/HDFS-14247) | Repeat adding node description into network topology | Minor | datanode | HuangTao | HuangTao | +| [YARN-9150](https://issues.apache.org/jira/browse/YARN-9150) | Making TimelineSchemaCreator support different backends for Timeline Schema Creation in ATSv2 | Major | ATSv2 | Sushil Ks | Sushil Ks | +| [HDFS-14366](https://issues.apache.org/jira/browse/HDFS-14366) | Improve HDFS append performance | Major | hdfs | Chao Sun | Chao Sun | +| [HDFS-14205](https://issues.apache.org/jira/browse/HDFS-14205) | Backport HDFS-6440 to branch-2 | Major | . | Chen Liang | Chao Sun | +| [HDFS-14391](https://issues.apache.org/jira/browse/HDFS-14391) | Backport HDFS-9659 to branch-2 | Minor | . | Chao Sun | Chao Sun | +| [HDFS-14415](https://issues.apache.org/jira/browse/HDFS-14415) | Backport HDFS-13799 to branch-2 | Trivial | . | Chao Sun | Chao Sun | +| [HDFS-14432](https://issues.apache.org/jira/browse/HDFS-14432) | dfs.datanode.shared.file.descriptor.paths duplicated in hdfs-default.xml | Minor | hdfs | puleya7 | puleya7 | +| [YARN-9529](https://issues.apache.org/jira/browse/YARN-9529) | Log correct cpu controller path on error while initializing CGroups. | Major | nodemanager | Jonathan Hung | Jonathan Hung | +| [HDFS-14502](https://issues.apache.org/jira/browse/HDFS-14502) | keepResults option in NNThroughputBenchmark should call saveNamespace() | Major | benchmarks, hdfs | Konstantin Shvachko | Konstantin Shvachko | +| [HADOOP-16323](https://issues.apache.org/jira/browse/HADOOP-16323) | https everywhere in Maven settings | Minor | build | Akira Ajisaka | Akira Ajisaka | +| [YARN-9563](https://issues.apache.org/jira/browse/YARN-9563) | Resource report REST API could return NaN or Inf | Minor | . | Ahmed Hussein | Ahmed Hussein | +| [HDFS-14513](https://issues.apache.org/jira/browse/HDFS-14513) | FSImage which is saving should be clean while NameNode shutdown | Major | namenode | Xiaoqiao He | Xiaoqiao He | +| [HADOOP-16369](https://issues.apache.org/jira/browse/HADOOP-16369) | Fix zstandard shortname misspelled as zts | Major | . | Jonathan Turner Eagles | Jonathan Turner Eagles | +| [HADOOP-16359](https://issues.apache.org/jira/browse/HADOOP-16359) | Bundle ZSTD native in branch-2 | Major | native | Chao Sun | Chao Sun | +| [HADOOP-15253](https://issues.apache.org/jira/browse/HADOOP-15253) | Should update maxQueueSize when refresh call queue | Minor | . | Tao Jie | Tao Jie | +| [HADOOP-16266](https://issues.apache.org/jira/browse/HADOOP-16266) | Add more fine-grained processing time metrics to the RPC layer | Minor | ipc | Christopher Gregorian | Erik Krogen | +| [HDFS-13694](https://issues.apache.org/jira/browse/HDFS-13694) | Making md5 computing being in parallel with image loading | Major | . | zhouyingchao | Lisheng Sun | +| [HDFS-14632](https://issues.apache.org/jira/browse/HDFS-14632) | Reduce useless #getNumLiveDataNodes call in SafeModeMonitor | Major | namenode | Xiaoqiao He | Xiaoqiao He | +| [HDFS-14547](https://issues.apache.org/jira/browse/HDFS-14547) | DirectoryWithQuotaFeature.quota costs additional memory even the storage type quota is not set. | Major | . | Jinglun | Jinglun | +| [HDFS-14697](https://issues.apache.org/jira/browse/HDFS-14697) | Backport HDFS-14513 to branch-2 | Minor | namenode | Xiaoqiao He | Xiaoqiao He | +| [YARN-8045](https://issues.apache.org/jira/browse/YARN-8045) | Reduce log output from container status calls | Major | . | Shane Kumpf | Craig Condit | +| [HDFS-14313](https://issues.apache.org/jira/browse/HDFS-14313) | Get hdfs used space from FsDatasetImpl#volumeMap#ReplicaInfo in memory instead of df/du | Major | datanode, performance | Lisheng Sun | Lisheng Sun | +| [HDFS-14696](https://issues.apache.org/jira/browse/HDFS-14696) | Backport HDFS-11273 to branch-2 (Move TransferFsImage#doGetUrl function to a Util class) | Major | . | Siyao Meng | Siyao Meng | +| [HDFS-14370](https://issues.apache.org/jira/browse/HDFS-14370) | Edit log tailing fast-path should allow for backoff | Major | namenode, qjm | Erik Krogen | Erik Krogen | +| [YARN-9442](https://issues.apache.org/jira/browse/YARN-9442) | container working directory has group read permissions | Minor | yarn | Jim Brennan | Jim Brennan | +| [HADOOP-16459](https://issues.apache.org/jira/browse/HADOOP-16459) | Backport [HADOOP-16266] "Add more fine-grained processing time metrics to the RPC layer" to branch-2 | Major | . | Erik Krogen | Erik Krogen | +| [HDFS-14707](https://issues.apache.org/jira/browse/HDFS-14707) | Add JAVA\_LIBRARY\_PATH to HTTPFS startup options in branch-2 | Major | httpfs | Masatake Iwasaki | Masatake Iwasaki | +| [HDFS-14723](https://issues.apache.org/jira/browse/HDFS-14723) | Add helper method FSNamesystem#setBlockManagerForTesting() in branch-2 | Blocker | namenode | Wei-Chiu Chuang | Wei-Chiu Chuang | +| [HDFS-14276](https://issues.apache.org/jira/browse/HDFS-14276) | [SBN read] Reduce tailing overhead | Major | ha, namenode | Wei-Chiu Chuang | Ayush Saxena | +| [HDFS-14617](https://issues.apache.org/jira/browse/HDFS-14617) | Improve fsimage load time by writing sub-sections to the fsimage index | Major | namenode | Stephen O'Donnell | Stephen O'Donnell | +| [YARN-9756](https://issues.apache.org/jira/browse/YARN-9756) | Create metric that sums total memory/vcores preempted per round | Major | capacity scheduler | Eric Payne | Manikandan R | +| [HDFS-14633](https://issues.apache.org/jira/browse/HDFS-14633) | The StorageType quota and consume in QuotaFeature is not handled for rename | Major | . | Jinglun | Jinglun | +| [HADOOP-16439](https://issues.apache.org/jira/browse/HADOOP-16439) | Upgrade bundled Tomcat in branch-2 | Major | httpfs, kms | Masatake Iwasaki | Masatake Iwasaki | +| [YARN-9810](https://issues.apache.org/jira/browse/YARN-9810) | Add queue capacity/maxcapacity percentage metrics | Major | . | Jonathan Hung | Shubham Gupta | +| [YARN-9763](https://issues.apache.org/jira/browse/YARN-9763) | Print application tags in application summary | Major | . | Jonathan Hung | Manoj Kumar | +| [YARN-9764](https://issues.apache.org/jira/browse/YARN-9764) | Print application submission context label in application summary | Major | . | Jonathan Hung | Manoj Kumar | +| [HADOOP-16530](https://issues.apache.org/jira/browse/HADOOP-16530) | Update xercesImpl in branch-2 | Major | . | Wei-Chiu Chuang | Wei-Chiu Chuang | +| [YARN-9824](https://issues.apache.org/jira/browse/YARN-9824) | Fall back to configured queue ordering policy class name | Major | . | Jonathan Hung | Jonathan Hung | +| [YARN-9825](https://issues.apache.org/jira/browse/YARN-9825) | Changes for initializing placement rules with ResourceScheduler in branch-2 | Major | . | Jonathan Hung | Jonathan Hung | +| [YARN-9762](https://issues.apache.org/jira/browse/YARN-9762) | Add submission context label to audit logs | Major | . | Jonathan Hung | Manoj Kumar | +| [HDFS-14667](https://issues.apache.org/jira/browse/HDFS-14667) | Backport [HDFS-14403] "Cost-based FairCallQueue" to branch-2 | Major | . | Erik Krogen | Erik Krogen | ### BUG FIXES: | JIRA | Summary | Priority | Component | Reporter | Contributor | |:---- |:---- | :--- |:---- |:---- |:---- | +| [MAPREDUCE-6896](https://issues.apache.org/jira/browse/MAPREDUCE-6896) | Document wrong spelling in usage of MapredTestDriver tools. | Major | documentation | LiXin Ge | LiXin Ge | | [HDFS-12052](https://issues.apache.org/jira/browse/HDFS-12052) | Set SWEBHDFS delegation token kind when ssl is enabled in HttpFS | Major | httpfs, webhdfs | Zoran Dimitrijevic | Zoran Dimitrijevic | | [HDFS-12318](https://issues.apache.org/jira/browse/HDFS-12318) | Fix IOException condition for openInfo in DFSInputStream | Major | . | legend | legend | | [HDFS-12614](https://issues.apache.org/jira/browse/HDFS-12614) | FSPermissionChecker#getINodeAttrs() throws NPE when INodeAttributesProvider configured | Major | . | Manoj Govindassamy | Manoj Govindassamy | +| [YARN-7396](https://issues.apache.org/jira/browse/YARN-7396) | NPE when accessing container logs due to null dirsHandler | Major | . | Jonathan Hung | Jonathan Hung | | [YARN-7370](https://issues.apache.org/jira/browse/YARN-7370) | Preemption properties should be refreshable | Major | capacity scheduler, scheduler preemption | Eric Payne | Gergely Novák | | [YARN-7428](https://issues.apache.org/jira/browse/YARN-7428) | Add containerId to Localizer failed logs | Minor | nodemanager | Prabhu Joseph | Prabhu Joseph | +| [YARN-7410](https://issues.apache.org/jira/browse/YARN-7410) | Cleanup FixedValueResource to avoid dependency to ResourceUtils | Major | resourcemanager | Sunil G | Wangda Tan | | [HDFS-12783](https://issues.apache.org/jira/browse/HDFS-12783) | [branch-2] "dfsrouter" should use hdfsScript | Major | . | Brahma Reddy Battula | Brahma Reddy Battula | | [HDFS-12788](https://issues.apache.org/jira/browse/HDFS-12788) | Reset the upload button when file upload fails | Critical | ui, webhdfs | Brahma Reddy Battula | Brahma Reddy Battula | | [YARN-7469](https://issues.apache.org/jira/browse/YARN-7469) | Capacity Scheduler Intra-queue preemption: User can starve if newest app is exactly at user limit | Major | capacity scheduler, yarn | Eric Payne | Eric Payne | @@ -128,36 +203,37 @@ | [YARN-7363](https://issues.apache.org/jira/browse/YARN-7363) | ContainerLocalizer doesn't have a valid log4j config when using LinuxContainerExecutor | Major | nodemanager | Yufei Gu | Yufei Gu | | [HDFS-12754](https://issues.apache.org/jira/browse/HDFS-12754) | Lease renewal can hit a deadlock | Major | . | Kuhu Shukla | Kuhu Shukla | | [HDFS-12832](https://issues.apache.org/jira/browse/HDFS-12832) | INode.getFullPathName may throw ArrayIndexOutOfBoundsException lead to NameNode exit | Critical | namenode | DENG FEI | Konstantin Shvachko | -| [HDFS-11754](https://issues.apache.org/jira/browse/HDFS-11754) | Make FsServerDefaults cache configurable. | Minor | . | Rushabh S Shah | Mikhail Erofeev | +| [HDFS-11754](https://issues.apache.org/jira/browse/HDFS-11754) | Make FsServerDefaults cache configurable. | Minor | . | Rushabh Shah | Mikhail Erofeev | | [HADOOP-15042](https://issues.apache.org/jira/browse/HADOOP-15042) | Azure PageBlobInputStream.skip() can return negative value when numberOfPagesRemaining is 0 | Minor | fs/azure | Rajesh Balamohan | Rajesh Balamohan | | [HDFS-12638](https://issues.apache.org/jira/browse/HDFS-12638) | Delete copy-on-truncate block along with the original block, when deleting a file being truncated | Blocker | hdfs | Jiandan Yang | Konstantin Shvachko | | [YARN-4813](https://issues.apache.org/jira/browse/YARN-4813) | TestRMWebServicesDelegationTokenAuthentication.testDoAs fails intermittently | Major | resourcemanager | Daniel Templeton | Gergo Repas | -| [MAPREDUCE-5124](https://issues.apache.org/jira/browse/MAPREDUCE-5124) | AM lacks flow control for task events | Major | mr-am | Jason Lowe | Peter Bacsko | -| [YARN-7455](https://issues.apache.org/jira/browse/YARN-7455) | quote\_and\_append\_arg can overflow buffer | Major | nodemanager | Jason Lowe | Jim Brennan | +| [MAPREDUCE-5124](https://issues.apache.org/jira/browse/MAPREDUCE-5124) | AM lacks flow control for task events | Major | mr-am | Jason Darrell Lowe | Peter Bacsko | +| [YARN-7455](https://issues.apache.org/jira/browse/YARN-7455) | quote\_and\_append\_arg can overflow buffer | Major | nodemanager | Jason Darrell Lowe | Jim Brennan | +| [YARN-7594](https://issues.apache.org/jira/browse/YARN-7594) | TestNMWebServices#testGetNMResourceInfo fails on trunk | Major | nodemanager, webapp | Gergely Novák | Gergely Novák | | [YARN-5594](https://issues.apache.org/jira/browse/YARN-5594) | Handle old RMDelegationToken format when recovering RM | Major | resourcemanager | Tatyana But | Robert Kanter | | [HADOOP-14985](https://issues.apache.org/jira/browse/HADOOP-14985) | Remove subversion related code from VersionInfoMojo.java | Minor | build | Akira Ajisaka | Ajay Kumar | | [HDFS-11751](https://issues.apache.org/jira/browse/HDFS-11751) | DFSZKFailoverController daemon exits with wrong status code | Major | auto-failover | Doris Gu | Bharat Viswanadham | | [HDFS-12889](https://issues.apache.org/jira/browse/HDFS-12889) | Router UI is missing robots.txt file | Major | . | Bharat Viswanadham | Bharat Viswanadham | | [YARN-7607](https://issues.apache.org/jira/browse/YARN-7607) | Remove the trailing duplicated timestamp in container diagnostics message | Minor | nodemanager | Weiwei Yang | Weiwei Yang | -| [HADOOP-15080](https://issues.apache.org/jira/browse/HADOOP-15080) | Aliyun OSS: update oss sdk from 2.8.1 to 2.8.3 to remove its dependency on Cat-x "json-lib" | Blocker | fs/oss | Chris Douglas | Sammi Chen | +| [HADOOP-15080](https://issues.apache.org/jira/browse/HADOOP-15080) | Aliyun OSS: update oss sdk from 2.8.1 to 2.8.3 to remove its dependency on Cat-x "json-lib" | Blocker | fs/oss | Christopher Douglas | Sammi Chen | | [YARN-7608](https://issues.apache.org/jira/browse/YARN-7608) | Incorrect sTarget column causing DataTable warning on RM application and scheduler web page | Major | resourcemanager, webapp | Weiwei Yang | Gergely Novák | | [HDFS-12833](https://issues.apache.org/jira/browse/HDFS-12833) | Distcp : Update the usage of delete option for dependency with update and overwrite option | Minor | distcp, hdfs | Harshakiran Reddy | usharani | | [YARN-7647](https://issues.apache.org/jira/browse/YARN-7647) | NM print inappropriate error log when node-labels is enabled | Minor | . | Yang Wang | Yang Wang | -| [HDFS-12907](https://issues.apache.org/jira/browse/HDFS-12907) | Allow read-only access to reserved raw for non-superusers | Major | namenode | Daryn Sharp | Rushabh S Shah | -| [HDFS-12881](https://issues.apache.org/jira/browse/HDFS-12881) | Output streams closed with IOUtils suppressing write errors | Major | . | Jason Lowe | Ajay Kumar | -| [YARN-7595](https://issues.apache.org/jira/browse/YARN-7595) | Container launching code suppresses close exceptions after writes | Major | nodemanager | Jason Lowe | Jim Brennan | -| [HADOOP-15085](https://issues.apache.org/jira/browse/HADOOP-15085) | Output streams closed with IOUtils suppressing write errors | Major | . | Jason Lowe | Jim Brennan | +| [HDFS-12907](https://issues.apache.org/jira/browse/HDFS-12907) | Allow read-only access to reserved raw for non-superusers | Major | namenode | Daryn Sharp | Rushabh Shah | +| [HDFS-12881](https://issues.apache.org/jira/browse/HDFS-12881) | Output streams closed with IOUtils suppressing write errors | Major | . | Jason Darrell Lowe | Ajay Kumar | +| [YARN-7595](https://issues.apache.org/jira/browse/YARN-7595) | Container launching code suppresses close exceptions after writes | Major | nodemanager | Jason Darrell Lowe | Jim Brennan | +| [HADOOP-15085](https://issues.apache.org/jira/browse/HADOOP-15085) | Output streams closed with IOUtils suppressing write errors | Major | . | Jason Darrell Lowe | Jim Brennan | | [HADOOP-15123](https://issues.apache.org/jira/browse/HADOOP-15123) | KDiag tries to load krb5.conf from KRB5CCNAME instead of KRB5\_CONFIG | Minor | security | Vipin Rathor | Vipin Rathor | | [YARN-7661](https://issues.apache.org/jira/browse/YARN-7661) | NodeManager metrics return wrong value after update node resource | Major | . | Yang Wang | Yang Wang | | [HDFS-12347](https://issues.apache.org/jira/browse/HDFS-12347) | TestBalancerRPCDelay#testBalancerRPCDelay fails very frequently | Critical | test | Xiao Chen | Bharat Viswanadham | | [YARN-7662](https://issues.apache.org/jira/browse/YARN-7662) | [Atsv2] Define new set of configurations for reader and collectors to bind. | Major | . | Rohith Sharma K S | Rohith Sharma K S | -| [YARN-7674](https://issues.apache.org/jira/browse/YARN-7674) | Update Timeline Reader web app address in UI2 | Major | . | Rohith Sharma K S | Sunil Govindan | +| [YARN-7674](https://issues.apache.org/jira/browse/YARN-7674) | Update Timeline Reader web app address in UI2 | Major | . | Rohith Sharma K S | Sunil G | | [YARN-7542](https://issues.apache.org/jira/browse/YARN-7542) | Fix issue that causes some Running Opportunistic Containers to be recovered as PAUSED | Major | . | Arun Suresh | Sampada Dehankar | | [HADOOP-15143](https://issues.apache.org/jira/browse/HADOOP-15143) | NPE due to Invalid KerberosTicket in UGI | Major | . | Jitendra Nath Pandey | Mukul Kumar Singh | -| [YARN-7692](https://issues.apache.org/jira/browse/YARN-7692) | Skip validating priority acls while recovering applications | Blocker | resourcemanager | Charan Hebri | Sunil Govindan | +| [YARN-7692](https://issues.apache.org/jira/browse/YARN-7692) | Skip validating priority acls while recovering applications | Blocker | resourcemanager | Charan Hebri | Sunil G | | [MAPREDUCE-7028](https://issues.apache.org/jira/browse/MAPREDUCE-7028) | Concurrent task progress updates causing NPE in Application Master | Blocker | mr-am | Gergo Repas | Gergo Repas | | [YARN-7619](https://issues.apache.org/jira/browse/YARN-7619) | Max AM Resource value in Capacity Scheduler UI has to be refreshed for every user | Major | capacity scheduler, yarn | Eric Payne | Eric Payne | -| [YARN-7699](https://issues.apache.org/jira/browse/YARN-7699) | queueUsagePercentage is coming as INF for getApp REST api call | Major | webapp | Sunil Govindan | Sunil Govindan | +| [YARN-7699](https://issues.apache.org/jira/browse/YARN-7699) | queueUsagePercentage is coming as INF for getApp REST api call | Major | webapp | Sunil G | Sunil G | | [HDFS-12985](https://issues.apache.org/jira/browse/HDFS-12985) | NameNode crashes during restart after an OpenForWrite file present in the Snapshot got deleted | Major | hdfs | Manoj Govindassamy | Manoj Govindassamy | | [YARN-4227](https://issues.apache.org/jira/browse/YARN-4227) | Ignore expired containers from removed nodes in FairScheduler | Critical | fairscheduler | Wilfred Spiegelenburg | Wilfred Spiegelenburg | | [YARN-7508](https://issues.apache.org/jira/browse/YARN-7508) | NPE in FiCaSchedulerApp when debug log enabled in async-scheduling mode | Major | capacityscheduler | Tao Yang | Tao Yang | @@ -185,27 +261,27 @@ | [YARN-7698](https://issues.apache.org/jira/browse/YARN-7698) | A misleading variable's name in ApplicationAttemptEventDispatcher | Minor | resourcemanager | Jinjiang Ling | Jinjiang Ling | | [HDFS-12528](https://issues.apache.org/jira/browse/HDFS-12528) | Add an option to not disable short-circuit reads on failures | Major | hdfs-client, performance | Andre Araujo | Xiao Chen | | [HDFS-13100](https://issues.apache.org/jira/browse/HDFS-13100) | Handle IllegalArgumentException when GETSERVERDEFAULTS is not implemented in webhdfs. | Critical | hdfs, webhdfs | Yongjun Zhang | Yongjun Zhang | -| [YARN-7849](https://issues.apache.org/jira/browse/YARN-7849) | TestMiniYarnClusterNodeUtilization#testUpdateNodeUtilization fails due to heartbeat sync error | Major | test | Jason Lowe | Botong Huang | +| [YARN-7849](https://issues.apache.org/jira/browse/YARN-7849) | TestMiniYarnClusterNodeUtilization#testUpdateNodeUtilization fails due to heartbeat sync error | Major | test | Jason Darrell Lowe | Botong Huang | | [YARN-7801](https://issues.apache.org/jira/browse/YARN-7801) | AmFilterInitializer should addFilter after fill all parameters | Critical | . | Sumana Sathish | Wangda Tan | -| [YARN-7890](https://issues.apache.org/jira/browse/YARN-7890) | NPE during container relaunch | Major | . | Billie Rinaldi | Jason Lowe | +| [YARN-7890](https://issues.apache.org/jira/browse/YARN-7890) | NPE during container relaunch | Major | . | Billie Rinaldi | Jason Darrell Lowe | | [HDFS-13115](https://issues.apache.org/jira/browse/HDFS-13115) | In getNumUnderConstructionBlocks(), ignore the inodeIds for which the inodes have been deleted | Major | . | Yongjun Zhang | Yongjun Zhang | | [HDFS-12935](https://issues.apache.org/jira/browse/HDFS-12935) | Get ambiguous result for DFSAdmin command in HA mode when only one namenode is up | Major | tools | Jianfei Jiang | Jianfei Jiang | | [HDFS-13120](https://issues.apache.org/jira/browse/HDFS-13120) | Snapshot diff could be corrupted after concat | Major | namenode, snapshots | Xiaoyu Yao | Xiaoyu Yao | -| [HDFS-10453](https://issues.apache.org/jira/browse/HDFS-10453) | ReplicationMonitor thread could stuck for long time due to the race between replication and delete of same file in a large cluster. | Major | namenode | He Xiaoqiao | He Xiaoqiao | +| [HDFS-10453](https://issues.apache.org/jira/browse/HDFS-10453) | ReplicationMonitor thread could stuck for long time due to the race between replication and delete of same file in a large cluster. | Major | namenode | Xiaoqiao He | Xiaoqiao He | | [HDFS-8693](https://issues.apache.org/jira/browse/HDFS-8693) | refreshNamenodes does not support adding a new standby to a running DN | Critical | datanode, ha | Jian Fang | Ajith S | | [MAPREDUCE-7052](https://issues.apache.org/jira/browse/MAPREDUCE-7052) | TestFixedLengthInputFormat#testFormatCompressedIn is flaky | Major | client, test | Peter Bacsko | Peter Bacsko | | [HDFS-13112](https://issues.apache.org/jira/browse/HDFS-13112) | Token expiration edits may cause log corruption or deadlock | Critical | namenode | Daryn Sharp | Daryn Sharp | -| [MAPREDUCE-7053](https://issues.apache.org/jira/browse/MAPREDUCE-7053) | Timed out tasks can fail to produce thread dump | Major | . | Jason Lowe | Jason Lowe | +| [MAPREDUCE-7053](https://issues.apache.org/jira/browse/MAPREDUCE-7053) | Timed out tasks can fail to produce thread dump | Major | . | Jason Darrell Lowe | Jason Darrell Lowe | | [HADOOP-15206](https://issues.apache.org/jira/browse/HADOOP-15206) | BZip2 drops and duplicates records when input split size is small | Major | . | Aki Tanaka | Aki Tanaka | | [YARN-7937](https://issues.apache.org/jira/browse/YARN-7937) | Fix http method name in Cluster Application Timeout Update API example request | Minor | docs, documentation | Charan Hebri | Charan Hebri | | [YARN-7947](https://issues.apache.org/jira/browse/YARN-7947) | Capacity Scheduler intra-queue preemption can NPE for non-schedulable apps | Major | capacity scheduler, scheduler preemption | Eric Payne | Eric Payne | | [YARN-7945](https://issues.apache.org/jira/browse/YARN-7945) | Java Doc error in UnmanagedAMPoolManager for branch-2 | Major | . | Rohith Sharma K S | Botong Huang | | [HADOOP-14903](https://issues.apache.org/jira/browse/HADOOP-14903) | Add json-smart explicitly to pom.xml | Major | common | Ray Chiang | Ray Chiang | | [HADOOP-15236](https://issues.apache.org/jira/browse/HADOOP-15236) | Fix typo in RequestHedgingProxyProvider and RequestHedgingRMFailoverProxyProvider | Trivial | documentation | Akira Ajisaka | Gabor Bota | -| [MAPREDUCE-7027](https://issues.apache.org/jira/browse/MAPREDUCE-7027) | HadoopArchiveLogs shouldn't delete the original logs if the HAR creation fails | Critical | mrv2 | Gergely Novák | Gergely Novák | +| [MAPREDUCE-7027](https://issues.apache.org/jira/browse/MAPREDUCE-7027) | HadoopArchiveLogs shouldn't delete the original logs if the HAR creation fails | Critical | harchive | Gergely Novák | Gergely Novák | | [HDFS-12781](https://issues.apache.org/jira/browse/HDFS-12781) | After Datanode down, In Namenode UI Datanode tab is throwing warning message. | Major | datanode | Harshakiran Reddy | Brahma Reddy Battula | | [HDFS-12070](https://issues.apache.org/jira/browse/HDFS-12070) | Failed block recovery leaves files open indefinitely and at risk for data loss | Major | . | Daryn Sharp | Kihwal Lee | -| [HADOOP-15251](https://issues.apache.org/jira/browse/HADOOP-15251) | Backport HADOOP-13514 (surefire upgrade) to branch-2 | Major | test | Chris Douglas | Chris Douglas | +| [HADOOP-15251](https://issues.apache.org/jira/browse/HADOOP-15251) | Backport HADOOP-13514 (surefire upgrade) to branch-2 | Major | test | Christopher Douglas | Christopher Douglas | | [HDFS-13194](https://issues.apache.org/jira/browse/HDFS-13194) | CachePool permissions incorrectly checked | Major | . | Yiqun Lin | Jianfei Jiang | | [HADOOP-15276](https://issues.apache.org/jira/browse/HADOOP-15276) | branch-2 site not building after ADL troubleshooting doc added | Major | documentation | Steve Loughran | Steve Loughran | | [YARN-7835](https://issues.apache.org/jira/browse/YARN-7835) | [Atsv2] Race condition in NM while publishing events if second attempt is launched on the same node | Critical | . | Rohith Sharma K S | Rohith Sharma K S | @@ -227,11 +303,11 @@ | [YARN-8039](https://issues.apache.org/jira/browse/YARN-8039) | Clean up log dir configuration in TestLinuxContainerExecutorWithMocks.testStartLocalizer | Minor | . | Miklos Szegedi | Miklos Szegedi | | [HDFS-13296](https://issues.apache.org/jira/browse/HDFS-13296) | GenericTestUtils generates paths with drive letter in Windows and fail webhdfs related test cases | Major | . | Xiao Liang | Xiao Liang | | [HDFS-13268](https://issues.apache.org/jira/browse/HDFS-13268) | TestWebHdfsFileContextMainOperations fails on Windows | Major | . | Íñigo Goiri | Xiao Liang | -| [YARN-8054](https://issues.apache.org/jira/browse/YARN-8054) | Improve robustness of the LocalDirsHandlerService MonitoringTimerTask thread | Major | . | Jonathan Eagles | Jonathan Eagles | +| [YARN-8054](https://issues.apache.org/jira/browse/YARN-8054) | Improve robustness of the LocalDirsHandlerService MonitoringTimerTask thread | Major | . | Jonathan Turner Eagles | Jonathan Turner Eagles | | [YARN-7873](https://issues.apache.org/jira/browse/YARN-7873) | Revert YARN-6078 | Blocker | . | Billie Rinaldi | Billie Rinaldi | | [HDFS-13195](https://issues.apache.org/jira/browse/HDFS-13195) | DataNode conf page cannot display the current value after reconfig | Minor | datanode | maobaolong | maobaolong | | [YARN-8063](https://issues.apache.org/jira/browse/YARN-8063) | DistributedShellTimelinePlugin wrongly check for entityId instead of entityType | Major | . | Rohith Sharma K S | Rohith Sharma K S | -| [YARN-8068](https://issues.apache.org/jira/browse/YARN-8068) | Application Priority field causes NPE in app timeline publish when Hadoop 2.7 based clients to 2.8+ | Blocker | yarn | Sunil Govindan | Sunil Govindan | +| [YARN-8068](https://issues.apache.org/jira/browse/YARN-8068) | Application Priority field causes NPE in app timeline publish when Hadoop 2.7 based clients to 2.8+ | Blocker | yarn | Sunil G | Sunil G | | [HADOOP-12862](https://issues.apache.org/jira/browse/HADOOP-12862) | LDAP Group Mapping over SSL can not specify trust store | Major | . | Wei-Chiu Chuang | Wei-Chiu Chuang | | [HADOOP-15317](https://issues.apache.org/jira/browse/HADOOP-15317) | Improve NetworkTopology chooseRandom's loop | Major | . | Xiao Chen | Xiao Chen | | [HADOOP-15355](https://issues.apache.org/jira/browse/HADOOP-15355) | TestCommonConfigurationFields is broken by HADOOP-15312 | Major | test | Konstantin Shvachko | LiXin Ge | @@ -244,8 +320,8 @@ | [HDFS-13427](https://issues.apache.org/jira/browse/HDFS-13427) | Fix the section titles of transparent encryption document | Minor | documentation | Akira Ajisaka | Akira Ajisaka | | [YARN-7527](https://issues.apache.org/jira/browse/YARN-7527) | Over-allocate node resource in async-scheduling mode of CapacityScheduler | Major | capacityscheduler | Tao Yang | Tao Yang | | [HDFS-7101](https://issues.apache.org/jira/browse/HDFS-7101) | Potential null dereference in DFSck#doWork() | Minor | . | Ted Yu | skrho | -| [YARN-8120](https://issues.apache.org/jira/browse/YARN-8120) | JVM can crash with SIGSEGV when exiting due to custom leveldb logger | Major | nodemanager, resourcemanager | Jason Lowe | Jason Lowe | -| [YARN-8147](https://issues.apache.org/jira/browse/YARN-8147) | TestClientRMService#testGetApplications sporadically fails | Major | test | Jason Lowe | Jason Lowe | +| [YARN-8120](https://issues.apache.org/jira/browse/YARN-8120) | JVM can crash with SIGSEGV when exiting due to custom leveldb logger | Major | nodemanager, resourcemanager | Jason Darrell Lowe | Jason Darrell Lowe | +| [YARN-8147](https://issues.apache.org/jira/browse/YARN-8147) | TestClientRMService#testGetApplications sporadically fails | Major | test | Jason Darrell Lowe | Jason Darrell Lowe | | [HADOOP-14970](https://issues.apache.org/jira/browse/HADOOP-14970) | MiniHadoopClusterManager doesn't respect lack of format option | Minor | . | Erik Krogen | Erik Krogen | | [YARN-8156](https://issues.apache.org/jira/browse/YARN-8156) | Increase the default value of yarn.timeline-service.app-collector.linger-period.ms | Major | . | Rohith Sharma K S | Charan Hebri | | [YARN-8165](https://issues.apache.org/jira/browse/YARN-8165) | Incorrect queue name logging in AbstractContainerAllocator | Trivial | capacityscheduler | Weiwei Yang | Weiwei Yang | @@ -259,48 +335,50 @@ | [HADOOP-15390](https://issues.apache.org/jira/browse/HADOOP-15390) | Yarn RM logs flooded by DelegationTokenRenewer trying to renew KMS tokens | Critical | . | Xiao Chen | Xiao Chen | | [HDFS-13336](https://issues.apache.org/jira/browse/HDFS-13336) | Test cases of TestWriteToReplica failed in windows | Major | . | Xiao Liang | Xiao Liang | | [YARN-7598](https://issues.apache.org/jira/browse/YARN-7598) | Document how to use classpath isolation for aux-services in YARN | Major | . | Xuan Gong | Xuan Gong | -| [HADOOP-15385](https://issues.apache.org/jira/browse/HADOOP-15385) | Many tests are failing in hadoop-distcp project in branch-2 | Critical | tools/distcp | Rushabh S Shah | Jason Lowe | +| [YARN-8183](https://issues.apache.org/jira/browse/YARN-8183) | Fix ConcurrentModificationException inside RMAppAttemptMetrics#convertAtomicLongMaptoLongMap | Critical | yarn | Sumana Sathish | Suma Shivaprasad | +| [HADOOP-15385](https://issues.apache.org/jira/browse/HADOOP-15385) | Many tests are failing in hadoop-distcp project in branch-2 | Critical | tools/distcp | Rushabh Shah | Jason Darrell Lowe | | [MAPREDUCE-7042](https://issues.apache.org/jira/browse/MAPREDUCE-7042) | Killed MR job data does not move to mapreduce.jobhistory.done-dir when ATS v2 is enabled | Major | . | Yesha Vora | Xuan Gong | | [YARN-8205](https://issues.apache.org/jira/browse/YARN-8205) | Application State is not updated to ATS if AM launching is delayed. | Critical | . | Sumana Sathish | Rohith Sharma K S | | [MAPREDUCE-7072](https://issues.apache.org/jira/browse/MAPREDUCE-7072) | mapred job -history prints duplicate counter in human output | Major | client | Wilfred Spiegelenburg | Wilfred Spiegelenburg | -| [YARN-8221](https://issues.apache.org/jira/browse/YARN-8221) | RMWebServices also need to honor yarn.resourcemanager.display.per-user-apps | Major | webapp | Sunil Govindan | Sunil Govindan | +| [YARN-8221](https://issues.apache.org/jira/browse/YARN-8221) | RMWebServices also need to honor yarn.resourcemanager.display.per-user-apps | Major | webapp | Sunil G | Sunil G | | [HDFS-13509](https://issues.apache.org/jira/browse/HDFS-13509) | Bug fix for breakHardlinks() of ReplicaInfo/LocalReplica, and fix TestFileAppend failures on Windows | Major | . | Xiao Liang | Xiao Liang | -| [MAPREDUCE-7073](https://issues.apache.org/jira/browse/MAPREDUCE-7073) | Optimize TokenCache#obtainTokensForNamenodesInternal | Major | . | Bibin A Chundatt | Bibin A Chundatt | +| [MAPREDUCE-7073](https://issues.apache.org/jira/browse/MAPREDUCE-7073) | Optimize TokenCache#obtainTokensForNamenodesInternal | Major | . | Bibin Chundatt | Bibin Chundatt | +| [YARN-8025](https://issues.apache.org/jira/browse/YARN-8025) | UsersManangers#getComputedResourceLimitForActiveUsers throws NPE due to preComputedActiveUserLimit is empty | Major | yarn | Jiandan Yang | Tao Yang | | [YARN-8232](https://issues.apache.org/jira/browse/YARN-8232) | RMContainer lost queue name when RM HA happens | Major | resourcemanager | Hu Ziqian | Hu Ziqian | | [HDFS-13537](https://issues.apache.org/jira/browse/HDFS-13537) | TestHdfsHelper does not generate jceks path properly for relative path in Windows | Major | . | Xiao Liang | Xiao Liang | -| [HADOOP-15446](https://issues.apache.org/jira/browse/HADOOP-15446) | WASB: PageBlobInputStream.skip breaks HBASE replication | Major | fs/azure | Thomas Marquardt | Thomas Marquardt | +| [HADOOP-15446](https://issues.apache.org/jira/browse/HADOOP-15446) | WASB: PageBlobInputStream.skip breaks HBASE replication | Major | fs/azure | Thomas Marqardt | Thomas Marqardt | | [YARN-8244](https://issues.apache.org/jira/browse/YARN-8244) | TestContainerSchedulerQueuing.testStartMultipleContainers failed | Major | . | Miklos Szegedi | Jim Brennan | | [HDFS-13586](https://issues.apache.org/jira/browse/HDFS-13586) | Fsync fails on directories on Windows | Critical | datanode, hdfs | Lukas Majercak | Lukas Majercak | | [HDFS-13590](https://issues.apache.org/jira/browse/HDFS-13590) | Backport HDFS-12378 to branch-2 | Major | datanode, hdfs, test | Lukas Majercak | Lukas Majercak | -| [HADOOP-15478](https://issues.apache.org/jira/browse/HADOOP-15478) | WASB: hflush() and hsync() regression | Major | fs/azure | Thomas Marquardt | Thomas Marquardt | +| [HADOOP-15478](https://issues.apache.org/jira/browse/HADOOP-15478) | WASB: hflush() and hsync() regression | Major | fs/azure | Thomas Marqardt | Thomas Marqardt | | [HADOOP-15450](https://issues.apache.org/jira/browse/HADOOP-15450) | Avoid fsync storm triggered by DiskChecker and handle disk full situation | Blocker | . | Kihwal Lee | Arpit Agarwal | | [HDFS-13601](https://issues.apache.org/jira/browse/HDFS-13601) | Optimize ByteString conversions in PBHelper | Major | . | Andrew Wang | Andrew Wang | | [HDFS-13588](https://issues.apache.org/jira/browse/HDFS-13588) | Fix TestFsDatasetImpl test failures on Windows | Major | . | Xiao Liang | Xiao Liang | | [YARN-8310](https://issues.apache.org/jira/browse/YARN-8310) | Handle old NMTokenIdentifier, AMRMTokenIdentifier, and ContainerTokenIdentifier formats | Major | . | Robert Kanter | Robert Kanter | | [YARN-8344](https://issues.apache.org/jira/browse/YARN-8344) | Missing nm.stop() in TestNodeManagerResync to fix testKillContainersOnResync | Major | . | Giovanni Matteo Fumarola | Giovanni Matteo Fumarola | | [YARN-8327](https://issues.apache.org/jira/browse/YARN-8327) | Fix TestAggregatedLogFormat#testReadAcontainerLogs1 on Windows | Major | log-aggregation | Giovanni Matteo Fumarola | Giovanni Matteo Fumarola | -| [YARN-8346](https://issues.apache.org/jira/browse/YARN-8346) | Upgrading to 3.1 kills running containers with error "Opportunistic container queue is full" | Blocker | . | Rohith Sharma K S | Jason Lowe | +| [YARN-8346](https://issues.apache.org/jira/browse/YARN-8346) | Upgrading to 3.1 kills running containers with error "Opportunistic container queue is full" | Blocker | . | Rohith Sharma K S | Jason Darrell Lowe | | [HDFS-13611](https://issues.apache.org/jira/browse/HDFS-13611) | Unsafe use of Text as a ConcurrentHashMap key in PBHelperClient | Major | . | Andrew Wang | Andrew Wang | | [HDFS-13618](https://issues.apache.org/jira/browse/HDFS-13618) | Fix TestDataNodeFaultInjector test failures on Windows | Major | test | Xiao Liang | Xiao Liang | | [HADOOP-15473](https://issues.apache.org/jira/browse/HADOOP-15473) | Configure serialFilter in KeyProvider to avoid UnrecoverableKeyException caused by JDK-8189997 | Critical | kms | Gabor Bota | Gabor Bota | | [HDFS-13626](https://issues.apache.org/jira/browse/HDFS-13626) | Fix incorrect username when deny the setOwner operation | Minor | namenode | luhuachao | Zsolt Venczel | | [MAPREDUCE-7103](https://issues.apache.org/jira/browse/MAPREDUCE-7103) | Fix TestHistoryViewerPrinter on windows due to a mismatch line separator | Minor | . | Giovanni Matteo Fumarola | Giovanni Matteo Fumarola | -| [YARN-8359](https://issues.apache.org/jira/browse/YARN-8359) | Exclude containermanager.linux test classes on Windows | Major | . | Giovanni Matteo Fumarola | Jason Lowe | +| [YARN-8359](https://issues.apache.org/jira/browse/YARN-8359) | Exclude containermanager.linux test classes on Windows | Major | . | Giovanni Matteo Fumarola | Jason Darrell Lowe | | [HDFS-13664](https://issues.apache.org/jira/browse/HDFS-13664) | Refactor ConfiguredFailoverProxyProvider to make inheritance easier | Minor | hdfs-client | Chao Sun | Chao Sun | | [YARN-8405](https://issues.apache.org/jira/browse/YARN-8405) | RM zk-state-store.parent-path ACLs has been changed since HADOOP-14773 | Major | . | Rohith Sharma K S | Íñigo Goiri | | [MAPREDUCE-7108](https://issues.apache.org/jira/browse/MAPREDUCE-7108) | TestFileOutputCommitter fails on Windows | Minor | test | Zuoming Zhang | Zuoming Zhang | -| [MAPREDUCE-7101](https://issues.apache.org/jira/browse/MAPREDUCE-7101) | Add config parameter to allow JHS to alway scan user dir irrespective of modTime | Critical | . | Wangda Tan | Thomas Marquardt | +| [MAPREDUCE-7101](https://issues.apache.org/jira/browse/MAPREDUCE-7101) | Add config parameter to allow JHS to alway scan user dir irrespective of modTime | Critical | . | Wangda Tan | Thomas Marqardt | | [YARN-8404](https://issues.apache.org/jira/browse/YARN-8404) | Timeline event publish need to be async to avoid Dispatcher thread leak in case ATS is down | Blocker | . | Rohith Sharma K S | Rohith Sharma K S | | [HDFS-13675](https://issues.apache.org/jira/browse/HDFS-13675) | Speed up TestDFSAdminWithHA | Major | hdfs, namenode | Lukas Majercak | Lukas Majercak | | [HDFS-13673](https://issues.apache.org/jira/browse/HDFS-13673) | TestNameNodeMetrics fails on Windows | Minor | test | Zuoming Zhang | Zuoming Zhang | | [HDFS-13676](https://issues.apache.org/jira/browse/HDFS-13676) | TestEditLogRace fails on Windows | Minor | test | Zuoming Zhang | Zuoming Zhang | | [HADOOP-15523](https://issues.apache.org/jira/browse/HADOOP-15523) | Shell command timeout given is in seconds whereas it is taken as millisec while scheduling | Major | . | Bilwa S T | Bilwa S T | | [YARN-8444](https://issues.apache.org/jira/browse/YARN-8444) | NodeResourceMonitor crashes on bad swapFree value | Major | . | Jim Brennan | Jim Brennan | -| [YARN-8457](https://issues.apache.org/jira/browse/YARN-8457) | Compilation is broken with -Pyarn-ui | Major | webapp | Sunil Govindan | Sunil Govindan | -| [YARN-8401](https://issues.apache.org/jira/browse/YARN-8401) | [UI2] new ui is not accessible with out internet connection | Blocker | . | Bibin A Chundatt | Bibin A Chundatt | +| [YARN-8457](https://issues.apache.org/jira/browse/YARN-8457) | Compilation is broken with -Pyarn-ui | Major | webapp | Sunil G | Sunil G | +| [YARN-8401](https://issues.apache.org/jira/browse/YARN-8401) | [UI2] new ui is not accessible with out internet connection | Blocker | . | Bibin Chundatt | Bibin Chundatt | | [YARN-8451](https://issues.apache.org/jira/browse/YARN-8451) | Multiple NM heartbeat thread created when a slow NM resync with RM | Major | nodemanager | Botong Huang | Botong Huang | | [HADOOP-15548](https://issues.apache.org/jira/browse/HADOOP-15548) | Randomize local dirs | Minor | . | Jim Brennan | Jim Brennan | -| [YARN-8473](https://issues.apache.org/jira/browse/YARN-8473) | Containers being launched as app tears down can leave containers in NEW state | Major | nodemanager | Jason Lowe | Jason Lowe | +| [YARN-8473](https://issues.apache.org/jira/browse/YARN-8473) | Containers being launched as app tears down can leave containers in NEW state | Major | nodemanager | Jason Darrell Lowe | Jason Darrell Lowe | | [HDFS-13729](https://issues.apache.org/jira/browse/HDFS-13729) | Fix broken links to RBF documentation | Minor | documentation | jwhitter | Gabor Bota | | [YARN-8515](https://issues.apache.org/jira/browse/YARN-8515) | container-executor can crash with SIGPIPE after nodemanager restart | Major | . | Jim Brennan | Jim Brennan | | [YARN-8421](https://issues.apache.org/jira/browse/YARN-8421) | when moving app, activeUsers is increased, even though app does not have outstanding request | Major | . | kyungwan nam | | @@ -317,6 +395,131 @@ | [YARN-8679](https://issues.apache.org/jira/browse/YARN-8679) | [ATSv2] If HBase cluster is down for long time, high chances that NM ContainerManager dispatcher get blocked | Major | . | Rohith Sharma K S | Wangda Tan | | [HADOOP-14314](https://issues.apache.org/jira/browse/HADOOP-14314) | The OpenSolaris taxonomy link is dead in InterfaceClassification.md | Major | documentation | Daniel Templeton | Rui Gao | | [YARN-8649](https://issues.apache.org/jira/browse/YARN-8649) | NPE in localizer hearbeat processing if a container is killed while localizing | Major | . | lujie | lujie | +| [HADOOP-10219](https://issues.apache.org/jira/browse/HADOOP-10219) | ipc.Client.setupIOstreams() needs to check for ClientCache.stopClient requested shutdowns | Major | ipc | Steve Loughran | Kihwal Lee | +| [MAPREDUCE-7131](https://issues.apache.org/jira/browse/MAPREDUCE-7131) | Job History Server has race condition where it moves files from intermediate to finished but thinks file is in intermediate | Major | . | Anthony Hsu | Anthony Hsu | +| [HDFS-13836](https://issues.apache.org/jira/browse/HDFS-13836) | RBF: Handle mount table znode with null value | Major | federation, hdfs | yanghuafeng | yanghuafeng | +| [HDFS-12716](https://issues.apache.org/jira/browse/HDFS-12716) | 'dfs.datanode.failed.volumes.tolerated' to support minimum number of volumes to be available | Major | datanode | usharani | Ranith Sardar | +| [YARN-8709](https://issues.apache.org/jira/browse/YARN-8709) | CS preemption monitor always fails since one under-served queue was deleted | Major | capacityscheduler, scheduler preemption | Tao Yang | Tao Yang | +| [HDFS-13051](https://issues.apache.org/jira/browse/HDFS-13051) | Fix dead lock during async editlog rolling if edit queue is full | Major | namenode | zhangwei | Daryn Sharp | +| [HDFS-13914](https://issues.apache.org/jira/browse/HDFS-13914) | Fix DN UI logs link broken when https is enabled after HDFS-13902 | Minor | datanode | Jianfei Jiang | Jianfei Jiang | +| [MAPREDUCE-7133](https://issues.apache.org/jira/browse/MAPREDUCE-7133) | History Server task attempts REST API returns invalid data | Major | jobhistoryserver | Oleksandr Shevchenko | Oleksandr Shevchenko | +| [YARN-8720](https://issues.apache.org/jira/browse/YARN-8720) | CapacityScheduler does not enforce max resource allocation check at queue level | Major | capacity scheduler, capacityscheduler, resourcemanager | Tarun Parimi | Tarun Parimi | +| [HDFS-13844](https://issues.apache.org/jira/browse/HDFS-13844) | Fix the fmt\_bytes function in the dfs-dust.js | Minor | hdfs, ui | yanghuafeng | yanghuafeng | +| [HADOOP-15755](https://issues.apache.org/jira/browse/HADOOP-15755) | StringUtils#createStartupShutdownMessage throws NPE when args is null | Major | . | Lokesh Jain | Dinesh Chitlangia | +| [MAPREDUCE-3801](https://issues.apache.org/jira/browse/MAPREDUCE-3801) | org.apache.hadoop.mapreduce.v2.app.TestRuntimeEstimators.testExponentialEstimator fails intermittently | Major | mrv2 | Robert Joseph Evans | Jason Darrell Lowe | +| [MAPREDUCE-7137](https://issues.apache.org/jira/browse/MAPREDUCE-7137) | MRAppBenchmark.benchmark1() fails with NullPointerException | Minor | test | Oleksandr Shevchenko | Oleksandr Shevchenko | +| [MAPREDUCE-7138](https://issues.apache.org/jira/browse/MAPREDUCE-7138) | ThrottledContainerAllocator in MRAppBenchmark should implement RMHeartbeatHandler | Minor | test | Oleksandr Shevchenko | Oleksandr Shevchenko | +| [HDFS-13908](https://issues.apache.org/jira/browse/HDFS-13908) | TestDataNodeMultipleRegistrations is flaky | Major | . | Íñigo Goiri | Ayush Saxena | +| [HADOOP-15772](https://issues.apache.org/jira/browse/HADOOP-15772) | Remove the 'Path ... should be specified as a URI' warnings on startup | Major | conf | Arpit Agarwal | Ayush Saxena | +| [YARN-8804](https://issues.apache.org/jira/browse/YARN-8804) | resourceLimits may be wrongly calculated when leaf-queue is blocked in cluster with 3+ level queues | Critical | capacityscheduler | Tao Yang | Tao Yang | +| [YARN-8774](https://issues.apache.org/jira/browse/YARN-8774) | Memory leak when CapacityScheduler allocates from reserved container with non-default label | Critical | capacityscheduler | Tao Yang | Tao Yang | +| [YARN-8840](https://issues.apache.org/jira/browse/YARN-8840) | Add missing cleanupSSLConfig() call for TestTimelineClient test | Minor | test, timelineclient | Aki Tanaka | Aki Tanaka | +| [HADOOP-15817](https://issues.apache.org/jira/browse/HADOOP-15817) | Reuse Object Mapper in KMSJSONReader | Major | kms | Jonathan Turner Eagles | Jonathan Turner Eagles | +| [HADOOP-15820](https://issues.apache.org/jira/browse/HADOOP-15820) | ZStandardDecompressor native code sets an integer field as a long | Blocker | . | Jason Darrell Lowe | Jason Darrell Lowe | +| [HDFS-13964](https://issues.apache.org/jira/browse/HDFS-13964) | RBF: TestRouterWebHDFSContractAppend fails with No Active Namenode under nameservice | Major | . | Ayush Saxena | Ayush Saxena | +| [HDFS-13768](https://issues.apache.org/jira/browse/HDFS-13768) | Adding replicas to volume map makes DataNode start slowly | Major | . | Yiqun Lin | Surendra Singh Lilhore | +| [HADOOP-15818](https://issues.apache.org/jira/browse/HADOOP-15818) | Fix deprecated maven-surefire-plugin configuration in hadoop-kms module | Minor | kms | Akira Ajisaka | Vidura Bhathiya Mudalige | +| [HDFS-13802](https://issues.apache.org/jira/browse/HDFS-13802) | RBF: Remove FSCK from Router Web UI | Major | . | Fei Hui | Fei Hui | +| [HADOOP-15859](https://issues.apache.org/jira/browse/HADOOP-15859) | ZStandardDecompressor.c mistakes a class for an instance | Blocker | . | Ben Lau | Jason Darrell Lowe | +| [HADOOP-15850](https://issues.apache.org/jira/browse/HADOOP-15850) | CopyCommitter#concatFileChunks should check that the blocks per chunk is not 0 | Critical | tools/distcp | Ted Yu | Ted Yu | +| [YARN-7502](https://issues.apache.org/jira/browse/YARN-7502) | Nodemanager restart docs should describe nodemanager supervised property | Major | documentation | Jason Darrell Lowe | Suma Shivaprasad | +| [HADOOP-15866](https://issues.apache.org/jira/browse/HADOOP-15866) | Renamed HADOOP\_SECURITY\_GROUP\_SHELL\_COMMAND\_TIMEOUT keys break compatibility | Blocker | . | Wei-Chiu Chuang | Wei-Chiu Chuang | +| [YARN-8826](https://issues.apache.org/jira/browse/YARN-8826) | Fix lingering timeline collector after serviceStop in TimelineCollectorManager | Trivial | ATSv2 | Prabha Manepalli | Prabha Manepalli | +| [HADOOP-15822](https://issues.apache.org/jira/browse/HADOOP-15822) | zstd compressor can fail with a small output buffer | Major | . | Jason Darrell Lowe | Jason Darrell Lowe | +| [HDFS-13959](https://issues.apache.org/jira/browse/HDFS-13959) | TestUpgradeDomainBlockPlacementPolicy is flaky | Major | . | Ayush Saxena | Ayush Saxena | +| [HADOOP-15899](https://issues.apache.org/jira/browse/HADOOP-15899) | Update AWS Java SDK versions in NOTICE.txt | Major | . | Akira Ajisaka | Akira Ajisaka | +| [HADOOP-15900](https://issues.apache.org/jira/browse/HADOOP-15900) | Update JSch versions in LICENSE.txt | Major | . | Akira Ajisaka | Akira Ajisaka | +| [HDFS-14043](https://issues.apache.org/jira/browse/HDFS-14043) | Tolerate corrupted seen\_txid file | Major | hdfs, namenode | Lukas Majercak | Lukas Majercak | +| [YARN-8858](https://issues.apache.org/jira/browse/YARN-8858) | CapacityScheduler should respect maximum node resource when per-queue maximum-allocation is being used. | Major | . | Sumana Sathish | Wangda Tan | +| [HDFS-14048](https://issues.apache.org/jira/browse/HDFS-14048) | DFSOutputStream close() throws exception on subsequent call after DataNode restart | Major | hdfs-client | Erik Krogen | Erik Krogen | +| [MAPREDUCE-7156](https://issues.apache.org/jira/browse/MAPREDUCE-7156) | NullPointerException when reaching max shuffle connections | Major | mrv2 | Peter Bacsko | Peter Bacsko | +| [YARN-8233](https://issues.apache.org/jira/browse/YARN-8233) | NPE in CapacityScheduler#tryCommit when handling allocate/reserve proposal whose allocatedOrReservedContainer is null | Critical | capacityscheduler | Tao Yang | Tao Yang | +| [HADOOP-15923](https://issues.apache.org/jira/browse/HADOOP-15923) | create-release script should set max-cache-ttl as well as default-cache-ttl for gpg-agent | Blocker | build | Akira Ajisaka | Akira Ajisaka | +| [HADOOP-15930](https://issues.apache.org/jira/browse/HADOOP-15930) | Exclude MD5 checksum files from release artifact | Critical | build | Akira Ajisaka | Akira Ajisaka | +| [HADOOP-15925](https://issues.apache.org/jira/browse/HADOOP-15925) | The config and log of gpg-agent are removed in create-release script | Major | build | Akira Ajisaka | Dinesh Chitlangia | +| [HDFS-14056](https://issues.apache.org/jira/browse/HDFS-14056) | Fix error messages in HDFS-12716 | Minor | hdfs | Adam Antal | Ayush Saxena | +| [YARN-7794](https://issues.apache.org/jira/browse/YARN-7794) | SLSRunner is not loading timeline service jars causing failure | Blocker | scheduler-load-simulator | Sunil G | Yufei Gu | +| [HADOOP-16008](https://issues.apache.org/jira/browse/HADOOP-16008) | Fix typo in CommandsManual.md | Minor | . | Akira Ajisaka | Shweta | +| [HADOOP-15973](https://issues.apache.org/jira/browse/HADOOP-15973) | Configuration: Included properties are not cached if resource is a stream | Critical | . | Eric Payne | Eric Payne | +| [YARN-9175](https://issues.apache.org/jira/browse/YARN-9175) | Null resources check in ResourceInfo for branch-3.0 | Major | . | Jonathan Hung | Jonathan Hung | +| [HADOOP-15992](https://issues.apache.org/jira/browse/HADOOP-15992) | JSON License is included in the transitive dependency of aliyun-sdk-oss 3.0.0 | Blocker | . | Akira Ajisaka | Akira Ajisaka | +| [HADOOP-16030](https://issues.apache.org/jira/browse/HADOOP-16030) | AliyunOSS: bring fixes back from HADOOP-15671 | Blocker | fs/oss | wujinhu | wujinhu | +| [YARN-9162](https://issues.apache.org/jira/browse/YARN-9162) | Fix TestRMAdminCLI#testHelp | Major | resourcemanager, test | Ayush Saxena | Ayush Saxena | +| [YARN-8833](https://issues.apache.org/jira/browse/YARN-8833) | Avoid potential integer overflow when computing fair shares | Major | fairscheduler | liyakun | liyakun | +| [HADOOP-16016](https://issues.apache.org/jira/browse/HADOOP-16016) | TestSSLFactory#testServerWeakCiphers sporadically fails in precommit builds | Major | security, test | Jason Darrell Lowe | Akira Ajisaka | +| [HADOOP-16013](https://issues.apache.org/jira/browse/HADOOP-16013) | DecayRpcScheduler decay thread should run as a daemon | Major | ipc | Erik Krogen | Erik Krogen | +| [YARN-8747](https://issues.apache.org/jira/browse/YARN-8747) | [UI2] YARN UI2 page loading failed due to js error under some time zone configuration | Critical | webapp | collinma | collinma | +| [YARN-9210](https://issues.apache.org/jira/browse/YARN-9210) | RM nodes web page can not display node info | Blocker | yarn | Jiandan Yang | Jiandan Yang | +| [YARN-7088](https://issues.apache.org/jira/browse/YARN-7088) | Add application launch time to Resource Manager REST API | Major | . | Abdullah Yousufi | Kanwaljeet Sachdev | +| [YARN-9222](https://issues.apache.org/jira/browse/YARN-9222) | Print launchTime in ApplicationSummary | Major | . | Jonathan Hung | Jonathan Hung | +| [YARN-6616](https://issues.apache.org/jira/browse/YARN-6616) | YARN AHS shows submitTime for jobs same as startTime | Minor | . | Prabhu Joseph | Prabhu Joseph | +| [MAPREDUCE-7177](https://issues.apache.org/jira/browse/MAPREDUCE-7177) | Disable speculative execution in TestDFSIO | Major | . | Kihwal Lee | Zhaohui Xin | +| [YARN-9206](https://issues.apache.org/jira/browse/YARN-9206) | RMServerUtils does not count SHUTDOWN as an accepted state | Major | . | Kuhu Shukla | Kuhu Shukla | +| [YARN-9283](https://issues.apache.org/jira/browse/YARN-9283) | Javadoc of LinuxContainerExecutor#addSchedPriorityCommand has a wrong property name as reference | Minor | documentation | Szilard Nemeth | Adam Antal | +| [HADOOP-15813](https://issues.apache.org/jira/browse/HADOOP-15813) | Enable more reliable SSL connection reuse | Major | common | Daryn Sharp | Daryn Sharp | +| [HDFS-14314](https://issues.apache.org/jira/browse/HDFS-14314) | fullBlockReportLeaseId should be reset after registering to NN | Critical | datanode | star | star | +| [YARN-5714](https://issues.apache.org/jira/browse/YARN-5714) | ContainerExecutor does not order environment map | Trivial | nodemanager | Remi Catherinot | Remi Catherinot | +| [HADOOP-16192](https://issues.apache.org/jira/browse/HADOOP-16192) | CallQueue backoff bug fixes: doesn't perform backoff when add() is used, and doesn't update backoff when refreshed | Major | ipc | Erik Krogen | Erik Krogen | +| [HDFS-14399](https://issues.apache.org/jira/browse/HDFS-14399) | Backport HDFS-10536 to branch-2 | Critical | . | Chao Sun | Chao Sun | +| [HDFS-14407](https://issues.apache.org/jira/browse/HDFS-14407) | Fix misuse of SLF4j logging API in DatasetVolumeChecker#checkAllVolumes | Minor | . | Wanqiang Ji | Wanqiang Ji | +| [HDFS-14414](https://issues.apache.org/jira/browse/HDFS-14414) | Clean up findbugs warning in branch-2 | Major | . | Wei-Chiu Chuang | Dinesh Chitlangia | +| [HADOOP-14544](https://issues.apache.org/jira/browse/HADOOP-14544) | DistCp documentation for command line options is misaligned. | Minor | documentation | Chris Nauroth | Masatake Iwasaki | +| [HDFS-10477](https://issues.apache.org/jira/browse/HDFS-10477) | Stop decommission a rack of DataNodes caused NameNode fail over to standby | Major | namenode | yunjiong zhao | yunjiong zhao | +| [HDFS-13677](https://issues.apache.org/jira/browse/HDFS-13677) | Dynamic refresh Disk configuration results in overwriting VolumeMap | Blocker | . | xuzq | xuzq | +| [YARN-9285](https://issues.apache.org/jira/browse/YARN-9285) | RM UI progress column is of wrong type | Minor | yarn | Ahmed Hussein | Ahmed Hussein | +| [HDFS-14500](https://issues.apache.org/jira/browse/HDFS-14500) | NameNode StartupProgress continues to report edit log segments after the LOADING\_EDITS phase is finished | Major | namenode | Erik Krogen | Erik Krogen | +| [HADOOP-16331](https://issues.apache.org/jira/browse/HADOOP-16331) | Fix ASF License check in pom.xml | Major | . | Wanqiang Ji | Akira Ajisaka | +| [HDFS-14514](https://issues.apache.org/jira/browse/HDFS-14514) | Actual read size of open file in encryption zone still larger than listing size even after enabling HDFS-11402 in Hadoop 2 | Major | encryption, hdfs, snapshots | Siyao Meng | Siyao Meng | +| [HDFS-14512](https://issues.apache.org/jira/browse/HDFS-14512) | ONE\_SSD policy will be violated while write data with DistributedFileSystem.create(....favoredNodes) | Major | . | Shen Yinjie | Ayush Saxena | +| [HDFS-14521](https://issues.apache.org/jira/browse/HDFS-14521) | Suppress setReplication logging. | Major | . | Kihwal Lee | Kihwal Lee | +| [YARN-8625](https://issues.apache.org/jira/browse/YARN-8625) | Aggregate Resource Allocation for each job is not present in ATS | Major | ATSv2 | Prabhu Joseph | Prabhu Joseph | +| [HADOOP-16345](https://issues.apache.org/jira/browse/HADOOP-16345) | Potential NPE when instantiating FairCallQueue metrics | Major | ipc | Erik Krogen | Erik Krogen | +| [HDFS-14494](https://issues.apache.org/jira/browse/HDFS-14494) | Move Server logging of StatedId inside receiveRequestState() | Major | . | Konstantin Shvachko | Shweta | +| [HDFS-14535](https://issues.apache.org/jira/browse/HDFS-14535) | The default 8KB buffer in requestFileDescriptors#BufferedOutputStream is causing lots of heap allocation in HBase when using short-circut read | Major | hdfs-client | Zheng Hu | Zheng Hu | +| [HDFS-13730](https://issues.apache.org/jira/browse/HDFS-13730) | BlockReaderRemote.sendReadResult throws NPE | Major | hdfs-client | Wei-Chiu Chuang | Yuanbo Liu | +| [HDFS-13770](https://issues.apache.org/jira/browse/HDFS-13770) | dfsadmin -report does not always decrease "missing blocks (with replication factor 1)" metrics when file is deleted | Major | hdfs | Kitti Nanasi | Kitti Nanasi | +| [HDFS-14101](https://issues.apache.org/jira/browse/HDFS-14101) | Random failure of testListCorruptFilesCorruptedBlock | Major | test | Kihwal Lee | Zsolt Venczel | +| [HDFS-14465](https://issues.apache.org/jira/browse/HDFS-14465) | When the Block expected replications is larger than the number of DataNodes, entering maintenance will never exit. | Major | . | Yicong Cai | Yicong Cai | +| [HDFS-14541](https://issues.apache.org/jira/browse/HDFS-14541) | When evictableMmapped or evictable size is zero, do not throw NoSuchElementException | Major | hdfs-client, performance | Zheng Hu | Lisheng Sun | +| [HDFS-14629](https://issues.apache.org/jira/browse/HDFS-14629) | Property value Hard Coded in DNConf.java | Trivial | . | hemanthboyina | hemanthboyina | +| [HDFS-12703](https://issues.apache.org/jira/browse/HDFS-12703) | Exceptions are fatal to decommissioning monitor | Critical | namenode | Daryn Sharp | Xiaoqiao He | +| [HDFS-12748](https://issues.apache.org/jira/browse/HDFS-12748) | NameNode memory leak when accessing webhdfs GETHOMEDIRECTORY | Major | hdfs | Jiandan Yang | Weiwei Yang | +| [HADOOP-16386](https://issues.apache.org/jira/browse/HADOOP-16386) | FindBugs warning in branch-2: GlobalStorageStatistics defines non-transient non-serializable instance field map | Major | fs | Wei-Chiu Chuang | Masatake Iwasaki | +| [MAPREDUCE-6521](https://issues.apache.org/jira/browse/MAPREDUCE-6521) | MiniMRYarnCluster should not create /tmp/hadoop-yarn/staging on local filesystem in unit test | Major | test | Masatake Iwasaki | Masatake Iwasaki | +| [MAPREDUCE-7076](https://issues.apache.org/jira/browse/MAPREDUCE-7076) | TestNNBench#testNNBenchCreateReadAndDelete failing in our internal build | Minor | test | Rushabh Shah | kevin su | +| [YARN-9668](https://issues.apache.org/jira/browse/YARN-9668) | UGI conf doesn't read user overridden configurations on RM and NM startup | Major | . | Jonathan Hung | Jonathan Hung | +| [HADOOP-9844](https://issues.apache.org/jira/browse/HADOOP-9844) | NPE when trying to create an error message response of SASL RPC | Major | ipc | Steve Loughran | Steve Loughran | +| [HADOOP-16245](https://issues.apache.org/jira/browse/HADOOP-16245) | Enabling SSL within LdapGroupsMapping can break system SSL configs | Major | common, security | Erik Krogen | Erik Krogen | +| [HDFS-14660](https://issues.apache.org/jira/browse/HDFS-14660) | [SBN Read] ObserverNameNode should throw StandbyException for requests not from ObserverProxyProvider | Major | . | Chao Sun | Chao Sun | +| [HDFS-14429](https://issues.apache.org/jira/browse/HDFS-14429) | Block remain in COMMITTED but not COMPLETE caused by Decommission | Major | . | Yicong Cai | Yicong Cai | +| [HDFS-14672](https://issues.apache.org/jira/browse/HDFS-14672) | Backport HDFS-12703 to branch-2 | Major | namenode | Xiaoqiao He | Xiaoqiao He | +| [HADOOP-16435](https://issues.apache.org/jira/browse/HADOOP-16435) | RpcMetrics should not be retained forever | Critical | rpc-server | Zoltan Haindrich | Zoltan Haindrich | +| [HDFS-14464](https://issues.apache.org/jira/browse/HDFS-14464) | Remove unnecessary log message from DFSInputStream | Trivial | . | Kihwal Lee | Chao Sun | +| [HDFS-14569](https://issues.apache.org/jira/browse/HDFS-14569) | Result of crypto -listZones is not formatted properly | Major | . | hemanthboyina | hemanthboyina | +| [YARN-9596](https://issues.apache.org/jira/browse/YARN-9596) | QueueMetrics has incorrect metrics when labelled partitions are involved | Major | capacity scheduler | Muhammad Samir Khan | Muhammad Samir Khan | +| [HADOOP-15237](https://issues.apache.org/jira/browse/HADOOP-15237) | In KMS docs there should be one space between KMS\_LOG and NOTE | Minor | kms | Snigdhanjali Mishra | Snigdhanjali Mishra | +| [HDFS-14462](https://issues.apache.org/jira/browse/HDFS-14462) | WebHDFS throws "Error writing request body to server" instead of DSQuotaExceededException | Major | webhdfs | Erik Krogen | Simbarashe Dzinamarira | +| [HDFS-14631](https://issues.apache.org/jira/browse/HDFS-14631) | The DirectoryScanner doesn't fix the wrongly placed replica. | Major | . | Jinglun | Jinglun | +| [HDFS-14724](https://issues.apache.org/jira/browse/HDFS-14724) | Fix JDK7 compatibility in branch-2 | Blocker | . | Wei-Chiu Chuang | Chen Liang | +| [HDFS-14423](https://issues.apache.org/jira/browse/HDFS-14423) | Percent (%) and plus (+) characters no longer work in WebHDFS | Major | webhdfs | Jing Wang | Masatake Iwasaki | +| [HDFS-13101](https://issues.apache.org/jira/browse/HDFS-13101) | Yet another fsimage corruption related to snapshot | Major | snapshots | Yongjun Zhang | Shashikant Banerjee | +| [HDFS-14311](https://issues.apache.org/jira/browse/HDFS-14311) | Multi-threading conflict at layoutVersion when loading block pool storage | Major | rolling upgrades | Yicong Cai | Yicong Cai | +| [HADOOP-16494](https://issues.apache.org/jira/browse/HADOOP-16494) | Add SHA-256 or SHA-512 checksum to release artifacts to comply with the release distribution policy | Blocker | build | Akira Ajisaka | Akira Ajisaka | +| [HDFS-13977](https://issues.apache.org/jira/browse/HDFS-13977) | NameNode can kill itself if it tries to send too many txns to a QJM simultaneously | Major | namenode, qjm | Erik Krogen | Erik Krogen | +| [YARN-9438](https://issues.apache.org/jira/browse/YARN-9438) | launchTime not written to state store for running applications | Major | . | Jonathan Hung | Jonathan Hung | +| [YARN-7585](https://issues.apache.org/jira/browse/YARN-7585) | NodeManager should go unhealthy when state store throws DBException | Major | nodemanager | Wilfred Spiegelenburg | Wilfred Spiegelenburg | +| [HDFS-14726](https://issues.apache.org/jira/browse/HDFS-14726) | Fix JN incompatibility issue in branch-2 due to backport of HDFS-10519 | Blocker | journal-node | Chen Liang | Chen Liang | +| [YARN-9806](https://issues.apache.org/jira/browse/YARN-9806) | TestNMSimulator#testNMSimulator fails in branch-2 | Major | . | Jonathan Hung | Jonathan Hung | +| [YARN-9820](https://issues.apache.org/jira/browse/YARN-9820) | RM logs InvalidStateTransitionException when app is submitted | Critical | . | Rohith Sharma K S | Prabhu Joseph | +| [HDFS-14303](https://issues.apache.org/jira/browse/HDFS-14303) | check block directory logic not correct when there is only meta file, print no meaning warn log | Minor | datanode, hdfs | qiang Liu | qiang Liu | +| [HADOOP-16582](https://issues.apache.org/jira/browse/HADOOP-16582) | LocalFileSystem's mkdirs() does not work as expected under viewfs. | Major | . | Kihwal Lee | Kihwal Lee | +| [HADOOP-16581](https://issues.apache.org/jira/browse/HADOOP-16581) | ValueQueue does not trigger an async refill when number of values falls below watermark | Major | common, kms | Yuval Degani | Yuval Degani | +| [HDFS-14853](https://issues.apache.org/jira/browse/HDFS-14853) | NPE in DFSNetworkTopology#chooseRandomWithStorageType() when the excludedNode is not present | Major | . | Ranith Sardar | Ranith Sardar | +| [YARN-9858](https://issues.apache.org/jira/browse/YARN-9858) | Optimize RMContext getExclusiveEnforcedPartitions | Major | . | Jonathan Hung | Jonathan Hung | +| [HDFS-14655](https://issues.apache.org/jira/browse/HDFS-14655) | [SBN Read] Namenode crashes if one of The JN is down | Critical | . | Harshakiran Reddy | Ayush Saxena | +| [HDFS-14245](https://issues.apache.org/jira/browse/HDFS-14245) | Class cast error in GetGroups with ObserverReadProxyProvider | Major | . | Shen Yinjie | Erik Krogen | +| [HDFS-14509](https://issues.apache.org/jira/browse/HDFS-14509) | DN throws InvalidToken due to inequality of password when upgrade NN 2.x to 3.x | Blocker | . | Yuxuan Wang | Yuxuan Wang | +| [HADOOP-16655](https://issues.apache.org/jira/browse/HADOOP-16655) | Change cipher suite when fetching tomcat tarball for branch-2 | Major | . | Jonathan Hung | Jonathan Hung | ### TESTS: @@ -351,22 +554,51 @@ | [HADOOP-15532](https://issues.apache.org/jira/browse/HADOOP-15532) | TestBasicDiskValidator fails with NoSuchFileException | Minor | . | Íñigo Goiri | Giovanni Matteo Fumarola | | [HDFS-13563](https://issues.apache.org/jira/browse/HDFS-13563) | TestDFSAdminWithHA times out on Windows | Minor | . | Anbang Hu | Lukas Majercak | | [HDFS-13681](https://issues.apache.org/jira/browse/HDFS-13681) | Fix TestStartup.testNNFailToStartOnReadOnlyNNDir test failure on Windows | Major | test | Xiao Liang | Xiao Liang | +| [YARN-8944](https://issues.apache.org/jira/browse/YARN-8944) | TestContainerAllocation.testUserLimitAllocationMultipleContainers failure after YARN-8896 | Minor | capacity scheduler | Wilfred Spiegelenburg | Wilfred Spiegelenburg | +| [HDFS-11950](https://issues.apache.org/jira/browse/HDFS-11950) | Disable libhdfs zerocopy test on Mac | Minor | libhdfs | John Zhuge | Akira Ajisaka | ### SUB-TASKS: | JIRA | Summary | Priority | Component | Reporter | Contributor | |:---- |:---- | :--- |:---- |:---- |:---- | +| [YARN-4081](https://issues.apache.org/jira/browse/YARN-4081) | Add support for multiple resource types in the Resource class | Major | resourcemanager | Varun Vasudev | Varun Vasudev | +| [YARN-4172](https://issues.apache.org/jira/browse/YARN-4172) | Extend DominantResourceCalculator to account for all resources | Major | resourcemanager | Varun Vasudev | Varun Vasudev | +| [YARN-4715](https://issues.apache.org/jira/browse/YARN-4715) | Add support to read resource types from a config file | Major | nodemanager, resourcemanager | Varun Vasudev | Varun Vasudev | +| [YARN-4829](https://issues.apache.org/jira/browse/YARN-4829) | Add support for binary units | Major | nodemanager, resourcemanager | Varun Vasudev | Varun Vasudev | +| [YARN-4830](https://issues.apache.org/jira/browse/YARN-4830) | Add support for resource types in the nodemanager | Major | nodemanager | Varun Vasudev | Varun Vasudev | +| [YARN-5242](https://issues.apache.org/jira/browse/YARN-5242) | Update DominantResourceCalculator to consider all resource types in calculations | Major | resourcemanager | Varun Vasudev | Varun Vasudev | +| [YARN-5586](https://issues.apache.org/jira/browse/YARN-5586) | Update the Resources class to consider all resource types | Major | nodemanager, resourcemanager | Varun Vasudev | Varun Vasudev | +| [YARN-6761](https://issues.apache.org/jira/browse/YARN-6761) | Fix build for YARN-3926 branch | Major | nodemanager, resourcemanager | Varun Vasudev | Varun Vasudev | +| [YARN-6786](https://issues.apache.org/jira/browse/YARN-6786) | ResourcePBImpl imports cleanup | Trivial | resourcemanager | Daniel Templeton | Yeliang Cang | +| [YARN-6788](https://issues.apache.org/jira/browse/YARN-6788) | Improve performance of resource profile branch | Blocker | nodemanager, resourcemanager | Sunil G | Sunil G | +| [YARN-6994](https://issues.apache.org/jira/browse/YARN-6994) | Remove last uses of Long from resource types code | Minor | resourcemanager | Daniel Templeton | Daniel Templeton | +| [YARN-6892](https://issues.apache.org/jira/browse/YARN-6892) | Improve API implementation in Resources and DominantResourceCalculator class | Major | nodemanager, resourcemanager | Sunil G | Sunil G | +| [YARN-6610](https://issues.apache.org/jira/browse/YARN-6610) | DominantResourceCalculator#getResourceAsValue dominant param is updated to handle multiple resources | Critical | resourcemanager | Daniel Templeton | Daniel Templeton | +| [YARN-7030](https://issues.apache.org/jira/browse/YARN-7030) | Performance optimizations in Resource and ResourceUtils class | Critical | nodemanager, resourcemanager | Wangda Tan | Wangda Tan | +| [YARN-7042](https://issues.apache.org/jira/browse/YARN-7042) | Clean up unit tests after YARN-6610 | Major | test | Daniel Templeton | Daniel Templeton | +| [YARN-6789](https://issues.apache.org/jira/browse/YARN-6789) | Add Client API to get all supported resource types from RM | Major | nodemanager, resourcemanager | Sunil G | Sunil G | +| [YARN-6781](https://issues.apache.org/jira/browse/YARN-6781) | ResourceUtils#initializeResourcesMap takes an unnecessary Map parameter | Minor | resourcemanager | Daniel Templeton | Yu-Tang Lin | +| [YARN-7067](https://issues.apache.org/jira/browse/YARN-7067) | Optimize ResourceType information display in UI | Critical | nodemanager, resourcemanager | Wangda Tan | Wangda Tan | +| [YARN-7039](https://issues.apache.org/jira/browse/YARN-7039) | Fix javac and javadoc errors in YARN-3926 branch | Major | nodemanager, resourcemanager | Sunil G | Sunil G | +| [YARN-7093](https://issues.apache.org/jira/browse/YARN-7093) | Improve log message in ResourceUtils | Trivial | nodemanager, resourcemanager | Sunil G | Sunil G | +| [YARN-6933](https://issues.apache.org/jira/browse/YARN-6933) | ResourceUtils.DISALLOWED\_NAMES check is duplicated | Major | resourcemanager | Daniel Templeton | Manikandan R | +| [YARN-7137](https://issues.apache.org/jira/browse/YARN-7137) | Move newly added APIs to unstable in YARN-3926 branch | Blocker | nodemanager, resourcemanager | Wangda Tan | Wangda Tan | | [HADOOP-14799](https://issues.apache.org/jira/browse/HADOOP-14799) | Update nimbus-jose-jwt to 4.41.1 | Major | . | Ray Chiang | Ray Chiang | +| [YARN-7345](https://issues.apache.org/jira/browse/YARN-7345) | GPU Isolation: Incorrect minor device numbers written to devices.deny file | Major | . | Jonathan Hung | Jonathan Hung | | [HADOOP-14997](https://issues.apache.org/jira/browse/HADOOP-14997) | Add hadoop-aliyun as dependency of hadoop-cloud-storage | Minor | fs/oss | Genmao Yu | Genmao Yu | +| [YARN-7143](https://issues.apache.org/jira/browse/YARN-7143) | FileNotFound handling in ResourceUtils is inconsistent | Major | resourcemanager | Daniel Templeton | Daniel Templeton | +| [YARN-6909](https://issues.apache.org/jira/browse/YARN-6909) | Use LightWeightedResource when number of resource types more than two | Critical | resourcemanager | Daniel Templeton | Sunil G | | [HDFS-12801](https://issues.apache.org/jira/browse/HDFS-12801) | RBF: Set MountTableResolver as default file resolver | Minor | . | Íñigo Goiri | Íñigo Goiri | | [YARN-7430](https://issues.apache.org/jira/browse/YARN-7430) | Enable user re-mapping for Docker containers by default | Blocker | security, yarn | Eric Yang | Eric Yang | | [HADOOP-15024](https://issues.apache.org/jira/browse/HADOOP-15024) | AliyunOSS: support user agent configuration and include that & Hadoop version information to oss server | Major | fs, fs/oss | Sammi Chen | Sammi Chen | | [HDFS-12858](https://issues.apache.org/jira/browse/HDFS-12858) | RBF: Add router admin commands usage in HDFS commands reference doc | Minor | documentation | Yiqun Lin | Yiqun Lin | | [HDFS-12835](https://issues.apache.org/jira/browse/HDFS-12835) | RBF: Fix Javadoc parameter errors | Minor | . | Wei Yan | Wei Yan | -| [HDFS-12396](https://issues.apache.org/jira/browse/HDFS-12396) | Webhdfs file system should get delegation token from kms provider. | Major | encryption, kms, webhdfs | Rushabh S Shah | Rushabh S Shah | +| [YARN-7573](https://issues.apache.org/jira/browse/YARN-7573) | Gpu Information page could be empty for nodes without GPU | Major | webapp, yarn-ui-v2 | Sunil G | Sunil G | +| [HDFS-12396](https://issues.apache.org/jira/browse/HDFS-12396) | Webhdfs file system should get delegation token from kms provider. | Major | encryption, kms, webhdfs | Rushabh Shah | Rushabh Shah | | [YARN-6704](https://issues.apache.org/jira/browse/YARN-6704) | Add support for work preserving NM restart when FederationInterceptor is enabled in AMRMProxyService | Major | . | Botong Huang | Botong Huang | | [HDFS-12875](https://issues.apache.org/jira/browse/HDFS-12875) | RBF: Complete logic for -readonly option of dfsrouteradmin add command | Major | . | Yiqun Lin | Íñigo Goiri | +| [YARN-7383](https://issues.apache.org/jira/browse/YARN-7383) | Node resource is not parsed correctly for resource names containing dot | Major | nodemanager, resourcemanager | Jonathan Hung | Gergely Novák | | [YARN-7630](https://issues.apache.org/jira/browse/YARN-7630) | Fix AMRMToken rollover handling in AMRMProxy | Minor | . | Botong Huang | Botong Huang | | [HDFS-12937](https://issues.apache.org/jira/browse/HDFS-12937) | RBF: Add more unit tests for router admin commands | Major | test | Yiqun Lin | Yiqun Lin | | [YARN-7032](https://issues.apache.org/jira/browse/YARN-7032) | [ATSv2] NPE while starting hbase co-processor when HBase authorization is enabled. | Critical | . | Rohith Sharma K S | Rohith Sharma K S | @@ -381,11 +613,14 @@ | [HDFS-12772](https://issues.apache.org/jira/browse/HDFS-12772) | RBF: Federation Router State State Store internal API | Major | . | Íñigo Goiri | Íñigo Goiri | | [HDFS-13042](https://issues.apache.org/jira/browse/HDFS-13042) | RBF: Heartbeat Router State | Major | . | Íñigo Goiri | Íñigo Goiri | | [HDFS-13049](https://issues.apache.org/jira/browse/HDFS-13049) | RBF: Inconsistent Router OPTS config in branch-2 and branch-3 | Minor | . | Wei Yan | Wei Yan | -| [HDFS-12574](https://issues.apache.org/jira/browse/HDFS-12574) | Add CryptoInputStream to WebHdfsFileSystem read call. | Major | encryption, kms, webhdfs | Rushabh S Shah | Rushabh S Shah | +| [YARN-7817](https://issues.apache.org/jira/browse/YARN-7817) | Add Resource reference to RM's NodeInfo object so REST API can get non memory/vcore resource usages. | Major | . | Sumana Sathish | Sunil G | +| [HDFS-12574](https://issues.apache.org/jira/browse/HDFS-12574) | Add CryptoInputStream to WebHdfsFileSystem read call. | Major | encryption, kms, webhdfs | Rushabh Shah | Rushabh Shah | | [HDFS-13044](https://issues.apache.org/jira/browse/HDFS-13044) | RBF: Add a safe mode for the Router | Major | . | Íñigo Goiri | Íñigo Goiri | | [HDFS-13043](https://issues.apache.org/jira/browse/HDFS-13043) | RBF: Expose the state of the Routers in the federation | Major | . | Íñigo Goiri | Íñigo Goiri | | [HDFS-13068](https://issues.apache.org/jira/browse/HDFS-13068) | RBF: Add router admin option to manage safe mode | Major | . | Íñigo Goiri | Yiqun Lin | +| [YARN-7860](https://issues.apache.org/jira/browse/YARN-7860) | Fix UT failure TestRMWebServiceAppsNodelabel#testAppsRunning | Major | . | Weiwei Yang | Sunil G | | [HDFS-13119](https://issues.apache.org/jira/browse/HDFS-13119) | RBF: Manage unavailable clusters | Major | . | Íñigo Goiri | Yiqun Lin | +| [YARN-7223](https://issues.apache.org/jira/browse/YARN-7223) | Document GPU isolation feature | Blocker | . | Wangda Tan | Wangda Tan | | [HDFS-13187](https://issues.apache.org/jira/browse/HDFS-13187) | RBF: Fix Routers information shown in the web UI | Minor | . | Wei Yan | Wei Yan | | [HDFS-13184](https://issues.apache.org/jira/browse/HDFS-13184) | RBF: Improve the unit test TestRouterRPCClientRetries | Minor | test | Yiqun Lin | Yiqun Lin | | [HDFS-13199](https://issues.apache.org/jira/browse/HDFS-13199) | RBF: Fix the hdfs router page missing label icon issue | Major | federation, hdfs | maobaolong | maobaolong | @@ -396,7 +631,7 @@ | [HADOOP-15267](https://issues.apache.org/jira/browse/HADOOP-15267) | S3A multipart upload fails when SSE-C encryption is enabled | Critical | fs/s3 | Anis Elleuch | Anis Elleuch | | [HDFS-13230](https://issues.apache.org/jira/browse/HDFS-13230) | RBF: ConnectionManager's cleanup task will compare each pool's own active conns with its total conns | Minor | . | Wei Yan | Chao Sun | | [HDFS-13233](https://issues.apache.org/jira/browse/HDFS-13233) | RBF: MountTableResolver doesn't return the correct mount point of the given path | Major | hdfs | wangzhiyuan | wangzhiyuan | -| [HDFS-13212](https://issues.apache.org/jira/browse/HDFS-13212) | RBF: Fix router location cache issue | Major | federation, hdfs | Weiwei Wu | Weiwei Wu | +| [HDFS-13212](https://issues.apache.org/jira/browse/HDFS-13212) | RBF: Fix router location cache issue | Major | federation, hdfs | Wu Weiwei | Wu Weiwei | | [HDFS-13232](https://issues.apache.org/jira/browse/HDFS-13232) | RBF: ConnectionPool should return first usable connection | Minor | . | Wei Yan | Ekanth Sethuramalingam | | [HDFS-13240](https://issues.apache.org/jira/browse/HDFS-13240) | RBF: Update some inaccurate document descriptions | Minor | . | Yiqun Lin | Yiqun Lin | | [HDFS-11399](https://issues.apache.org/jira/browse/HDFS-11399) | Many tests fails in Windows due to injecting disk failures | Major | . | Yiqun Lin | Yiqun Lin | @@ -407,6 +642,7 @@ | [HDFS-12773](https://issues.apache.org/jira/browse/HDFS-12773) | RBF: Improve State Store FS implementation | Major | . | Íñigo Goiri | Íñigo Goiri | | [HDFS-13198](https://issues.apache.org/jira/browse/HDFS-13198) | RBF: RouterHeartbeatService throws out CachedStateStore related exceptions when starting router | Minor | . | Wei Yan | Wei Yan | | [HDFS-13224](https://issues.apache.org/jira/browse/HDFS-13224) | RBF: Resolvers to support mount points across multiple subclusters | Major | . | Íñigo Goiri | Íñigo Goiri | +| [HDFS-13299](https://issues.apache.org/jira/browse/HDFS-13299) | RBF : Fix compilation error in branch-2 (TestMultipleDestinationResolver) | Blocker | . | Brahma Reddy Battula | Brahma Reddy Battula | | [HADOOP-15262](https://issues.apache.org/jira/browse/HADOOP-15262) | AliyunOSS: move files under a directory in parallel when rename a directory | Major | fs/oss | wujinhu | wujinhu | | [HDFS-13215](https://issues.apache.org/jira/browse/HDFS-13215) | RBF: Move Router to its own module | Major | . | Íñigo Goiri | Wei Yan | | [HDFS-13307](https://issues.apache.org/jira/browse/HDFS-13307) | RBF: Improve the use of setQuota command | Major | . | liuhongtong | liuhongtong | @@ -423,16 +659,17 @@ | [HDFS-13347](https://issues.apache.org/jira/browse/HDFS-13347) | RBF: Cache datanode reports | Minor | . | Íñigo Goiri | Íñigo Goiri | | [HDFS-13289](https://issues.apache.org/jira/browse/HDFS-13289) | RBF: TestConnectionManager#testCleanup() test case need correction | Minor | . | Dibyendu Karmakar | Dibyendu Karmakar | | [HDFS-13364](https://issues.apache.org/jira/browse/HDFS-13364) | RBF: Support NamenodeProtocol in the Router | Major | . | Íñigo Goiri | Íñigo Goiri | +| [HADOOP-14651](https://issues.apache.org/jira/browse/HADOOP-14651) | Update okhttp version to 2.7.5 | Major | fs/adl | Ray Chiang | Ray Chiang | | [YARN-6936](https://issues.apache.org/jira/browse/YARN-6936) | [Atsv2] Retrospect storing entities into sub application table from client perspective | Major | . | Rohith Sharma K S | Rohith Sharma K S | | [HDFS-13353](https://issues.apache.org/jira/browse/HDFS-13353) | RBF: TestRouterWebHDFSContractCreate failed | Major | test | Takanobu Asanuma | Takanobu Asanuma | | [YARN-8107](https://issues.apache.org/jira/browse/YARN-8107) | Give an informative message when incorrect format is used in ATSv2 filter attributes | Major | ATSv2 | Charan Hebri | Rohith Sharma K S | | [YARN-8110](https://issues.apache.org/jira/browse/YARN-8110) | AMRMProxy recover should catch for all throwable to avoid premature exit | Major | . | Botong Huang | Botong Huang | | [HDFS-13402](https://issues.apache.org/jira/browse/HDFS-13402) | RBF: Fix java doc for StateStoreFileSystemImpl | Minor | hdfs | Yiran Wu | Yiran Wu | -| [HDFS-13380](https://issues.apache.org/jira/browse/HDFS-13380) | RBF: mv/rm fail after the directory exceeded the quota limit | Major | . | Weiwei Wu | Yiqun Lin | +| [HDFS-13380](https://issues.apache.org/jira/browse/HDFS-13380) | RBF: mv/rm fail after the directory exceeded the quota limit | Major | . | Wu Weiwei | Yiqun Lin | | [HDFS-13410](https://issues.apache.org/jira/browse/HDFS-13410) | RBF: Support federation with no subclusters | Minor | . | Íñigo Goiri | Íñigo Goiri | | [HDFS-13384](https://issues.apache.org/jira/browse/HDFS-13384) | RBF: Improve timeout RPC call mechanism | Minor | . | Íñigo Goiri | Íñigo Goiri | | [HDFS-13045](https://issues.apache.org/jira/browse/HDFS-13045) | RBF: Improve error message returned from subcluster | Minor | . | Wei Yan | Íñigo Goiri | -| [HDFS-13428](https://issues.apache.org/jira/browse/HDFS-13428) | RBF: Remove LinkedList From StateStoreFileImpl.java | Trivial | federation | BELUGA BEHR | BELUGA BEHR | +| [HDFS-13428](https://issues.apache.org/jira/browse/HDFS-13428) | RBF: Remove LinkedList From StateStoreFileImpl.java | Trivial | federation | David Mollitor | David Mollitor | | [HADOOP-14999](https://issues.apache.org/jira/browse/HADOOP-14999) | AliyunOSS: provide one asynchronous multi-part based uploading mechanism | Major | fs/oss | Genmao Yu | Genmao Yu | | [YARN-7810](https://issues.apache.org/jira/browse/YARN-7810) | TestDockerContainerRuntime test failures due to UID lookup of a non-existent user | Major | . | Shane Kumpf | Shane Kumpf | | [HDFS-13435](https://issues.apache.org/jira/browse/HDFS-13435) | RBF: Improve the error loggings for printing the stack trace | Major | . | Yiqun Lin | Yiqun Lin | @@ -456,12 +693,14 @@ | [YARN-8130](https://issues.apache.org/jira/browse/YARN-8130) | Race condition when container events are published for KILLED applications | Major | ATSv2 | Charan Hebri | Rohith Sharma K S | | [YARN-7900](https://issues.apache.org/jira/browse/YARN-7900) | [AMRMProxy] AMRMClientRelayer for stateful FederationInterceptor | Major | . | Botong Huang | Botong Huang | | [HADOOP-15498](https://issues.apache.org/jira/browse/HADOOP-15498) | TestHadoopArchiveLogs (#testGenerateScript, #testPrepareWorkingDir) fails on Windows | Minor | . | Anbang Hu | Anbang Hu | +| [HADOOP-15497](https://issues.apache.org/jira/browse/HADOOP-15497) | TestTrash should use proper test path to avoid failing on Windows | Minor | . | Anbang Hu | Anbang Hu | | [HDFS-13637](https://issues.apache.org/jira/browse/HDFS-13637) | RBF: Router fails when threadIndex (in ConnectionPool) wraps around Integer.MIN\_VALUE | Critical | federation | CR Hota | CR Hota | | [YARN-4781](https://issues.apache.org/jira/browse/YARN-4781) | Support intra-queue preemption for fairness ordering policy. | Major | scheduler | Wangda Tan | Eric Payne | | [HADOOP-15506](https://issues.apache.org/jira/browse/HADOOP-15506) | Upgrade Azure Storage Sdk version to 7.0.0 and update corresponding code blocks | Minor | fs/azure | Esfandiar Manii | Esfandiar Manii | | [HADOOP-15529](https://issues.apache.org/jira/browse/HADOOP-15529) | ContainerLaunch#testInvalidEnvVariableSubstitutionType is not supported in Windows | Minor | . | Giovanni Matteo Fumarola | Giovanni Matteo Fumarola | | [HADOOP-15533](https://issues.apache.org/jira/browse/HADOOP-15533) | Make WASB listStatus messages consistent | Trivial | fs/azure | Esfandiar Manii | Esfandiar Manii | | [HADOOP-15458](https://issues.apache.org/jira/browse/HADOOP-15458) | TestLocalFileSystem#testFSOutputStreamBuilder fails on Windows | Minor | test | Xiao Liang | Xiao Liang | +| [YARN-8481](https://issues.apache.org/jira/browse/YARN-8481) | AMRMProxyPolicies should accept heartbeat response from new/unknown subclusters | Minor | amrmproxy, federation | Botong Huang | Botong Huang | | [HDFS-13528](https://issues.apache.org/jira/browse/HDFS-13528) | RBF: If a directory exceeds quota limit then quota usage is not refreshed for other mount entries | Major | . | Dibyendu Karmakar | Dibyendu Karmakar | | [HDFS-13710](https://issues.apache.org/jira/browse/HDFS-13710) | RBF: setQuota and getQuotaUsage should check the dfs.federation.router.quota.enable | Major | federation, hdfs | yanghuafeng | yanghuafeng | | [HDFS-13726](https://issues.apache.org/jira/browse/HDFS-13726) | RBF: Fix RBF configuration links | Minor | documentation | Takanobu Asanuma | Takanobu Asanuma | @@ -471,8 +710,58 @@ | [HDFS-13583](https://issues.apache.org/jira/browse/HDFS-13583) | RBF: Router admin clrQuota is not synchronized with nameservice | Major | . | Dibyendu Karmakar | Dibyendu Karmakar | | [HDFS-13750](https://issues.apache.org/jira/browse/HDFS-13750) | RBF: Router ID in RouterRpcClient is always null | Major | . | Takanobu Asanuma | Takanobu Asanuma | | [YARN-8129](https://issues.apache.org/jira/browse/YARN-8129) | Improve error message for invalid value in fields attribute | Minor | ATSv2 | Charan Hebri | Abhishek Modi | +| [YARN-8581](https://issues.apache.org/jira/browse/YARN-8581) | [AMRMProxy] Add sub-cluster timeout in LocalityMulticastAMRMProxyPolicy | Major | amrmproxy, federation | Botong Huang | Botong Huang | +| [YARN-8673](https://issues.apache.org/jira/browse/YARN-8673) | [AMRMProxy] More robust responseId resync after an YarnRM master slave switch | Major | amrmproxy | Botong Huang | Botong Huang | | [HDFS-13848](https://issues.apache.org/jira/browse/HDFS-13848) | Refactor NameNode failover proxy providers | Major | ha, hdfs-client | Konstantin Shvachko | Konstantin Shvachko | | [HDFS-13634](https://issues.apache.org/jira/browse/HDFS-13634) | RBF: Configurable value in xml for async connection request queue size. | Major | federation | CR Hota | CR Hota | +| [HADOOP-15731](https://issues.apache.org/jira/browse/HADOOP-15731) | TestDistributedShell fails on Windows | Major | . | Botong Huang | Botong Huang | +| [HADOOP-15759](https://issues.apache.org/jira/browse/HADOOP-15759) | AliyunOSS: update oss-sdk version to 3.0.0 | Major | fs/oss | wujinhu | wujinhu | +| [HADOOP-15748](https://issues.apache.org/jira/browse/HADOOP-15748) | S3 listing inconsistency can raise NPE in globber | Major | fs | Steve Loughran | Steve Loughran | +| [YARN-8696](https://issues.apache.org/jira/browse/YARN-8696) | [AMRMProxy] FederationInterceptor upgrade: home sub-cluster heartbeat async | Major | nodemanager | Botong Huang | Botong Huang | +| [HADOOP-15671](https://issues.apache.org/jira/browse/HADOOP-15671) | AliyunOSS: Support Assume Roles in AliyunOSS | Major | fs/oss | wujinhu | wujinhu | +| [HDFS-13790](https://issues.apache.org/jira/browse/HDFS-13790) | RBF: Move ClientProtocol APIs to its own module | Major | . | Íñigo Goiri | Chao Sun | +| [YARN-7652](https://issues.apache.org/jira/browse/YARN-7652) | Handle AM register requests asynchronously in FederationInterceptor | Major | amrmproxy, federation | Subramaniam Krishnan | Botong Huang | +| [YARN-6989](https://issues.apache.org/jira/browse/YARN-6989) | Ensure timeline service v2 codebase gets UGI from HttpServletRequest in a consistent way | Major | timelineserver | Vrushali C | Abhishek Modi | +| [YARN-3879](https://issues.apache.org/jira/browse/YARN-3879) | [Storage implementation] Create HDFS backing storage implementation for ATS reads | Major | timelineserver | Tsuyoshi Ozawa | Abhishek Modi | +| [HADOOP-15837](https://issues.apache.org/jira/browse/HADOOP-15837) | DynamoDB table Update can fail S3A FS init | Major | fs/s3 | Steve Loughran | Steve Loughran | +| [HADOOP-15607](https://issues.apache.org/jira/browse/HADOOP-15607) | AliyunOSS: fix duplicated partNumber issue in AliyunOSSBlockOutputStream | Critical | . | wujinhu | wujinhu | +| [HADOOP-15868](https://issues.apache.org/jira/browse/HADOOP-15868) | AliyunOSS: update document for properties of multiple part download, multiple part upload and directory copy | Major | fs/oss | wujinhu | wujinhu | +| [YARN-8893](https://issues.apache.org/jira/browse/YARN-8893) | [AMRMProxy] Fix thread leak in AMRMClientRelayer and UAM client | Major | amrmproxy, federation | Botong Huang | Botong Huang | +| [YARN-8905](https://issues.apache.org/jira/browse/YARN-8905) | [Router] Add JvmMetricsInfo and pause monitor | Minor | . | Bibin Chundatt | Bilwa S T | +| [HADOOP-15917](https://issues.apache.org/jira/browse/HADOOP-15917) | AliyunOSS: fix incorrect ReadOps and WriteOps in statistics | Major | fs/oss | wujinhu | wujinhu | +| [HADOOP-16009](https://issues.apache.org/jira/browse/HADOOP-16009) | Replace the url of the repository in Apache Hadoop source code | Major | documentation | Akira Ajisaka | Akira Ajisaka | +| [HADOOP-15323](https://issues.apache.org/jira/browse/HADOOP-15323) | AliyunOSS: Improve copy file performance for AliyunOSSFileSystemStore | Major | fs/oss | wujinhu | wujinhu | +| [YARN-9182](https://issues.apache.org/jira/browse/YARN-9182) | Backport YARN-6445 resource profile performance improvements to branch-2 | Major | . | Jonathan Hung | Jonathan Hung | +| [YARN-9181](https://issues.apache.org/jira/browse/YARN-9181) | Backport YARN-6232 for generic resource type usage to branch-2 | Major | . | Jonathan Hung | Jonathan Hung | +| [YARN-9177](https://issues.apache.org/jira/browse/YARN-9177) | Use resource map for app metrics in TestCombinedSystemMetricsPublisher for branch-2 | Major | . | Jonathan Hung | Jonathan Hung | +| [YARN-9188](https://issues.apache.org/jira/browse/YARN-9188) | Port YARN-7136 to branch-2 | Major | . | Jonathan Hung | Jonathan Hung | +| [YARN-9187](https://issues.apache.org/jira/browse/YARN-9187) | Backport YARN-6852 for GPU-specific native changes to branch-2 | Major | . | Jonathan Hung | Jonathan Hung | +| [YARN-9180](https://issues.apache.org/jira/browse/YARN-9180) | Port YARN-7033 NM recovery of assigned resources to branch-3.0/branch-2 | Major | . | Jonathan Hung | Jonathan Hung | +| [YARN-9280](https://issues.apache.org/jira/browse/YARN-9280) | Backport YARN-6620 to YARN-8200/branch-2 | Major | . | Jonathan Hung | Jonathan Hung | +| [YARN-9174](https://issues.apache.org/jira/browse/YARN-9174) | Backport YARN-7224 for refactoring of GpuDevice class | Major | . | Jonathan Hung | Jonathan Hung | +| [YARN-9289](https://issues.apache.org/jira/browse/YARN-9289) | Backport YARN-7330 for GPU in UI to branch-2 | Major | . | Jonathan Hung | Jonathan Hung | +| [HDFS-14262](https://issues.apache.org/jira/browse/HDFS-14262) | [SBN read] Unclear Log.WARN message in GlobalStateIdContext | Major | hdfs | Shweta | Shweta | +| [YARN-8549](https://issues.apache.org/jira/browse/YARN-8549) | Adding a NoOp timeline writer and reader plugin classes for ATSv2 | Minor | ATSv2, timelineclient, timelineserver | Prabha Manepalli | Prabha Manepalli | +| [HADOOP-16109](https://issues.apache.org/jira/browse/HADOOP-16109) | Parquet reading S3AFileSystem causes EOF | Blocker | fs/s3 | Dave Christianson | Steve Loughran | +| [YARN-9397](https://issues.apache.org/jira/browse/YARN-9397) | Fix empty NMResourceInfo object test failures in branch-2 | Major | . | Jonathan Hung | Jonathan Hung | +| [HADOOP-16191](https://issues.apache.org/jira/browse/HADOOP-16191) | AliyunOSS: improvements for copyFile/copyDirectory and logging | Major | fs/oss | wujinhu | wujinhu | +| [YARN-9271](https://issues.apache.org/jira/browse/YARN-9271) | Backport YARN-6927 for resource type support in MapReduce | Major | . | Jonathan Hung | Jonathan Hung | +| [YARN-9291](https://issues.apache.org/jira/browse/YARN-9291) | Backport YARN-7637 to branch-2 | Major | . | Jonathan Hung | Jonathan Hung | +| [YARN-9409](https://issues.apache.org/jira/browse/YARN-9409) | Port resource type changes from YARN-7237 to branch-3.0/branch-2 | Major | . | Jonathan Hung | Jonathan Hung | +| [YARN-9272](https://issues.apache.org/jira/browse/YARN-9272) | Backport YARN-7738 for refreshing max allocation for multiple resource types | Major | . | Jonathan Hung | Jonathan Hung | +| [HADOOP-16205](https://issues.apache.org/jira/browse/HADOOP-16205) | Backporting ABFS driver from trunk to branch 2.0 | Major | fs/azure | Esfandiar Manii | Yuan Gao | +| [HADOOP-16269](https://issues.apache.org/jira/browse/HADOOP-16269) | ABFS: add listFileStatus with StartFrom | Major | fs/azure | Da Zhou | Da Zhou | +| [HADOOP-16306](https://issues.apache.org/jira/browse/HADOOP-16306) | AliyunOSS: Remove temporary files when upload small files to OSS | Major | fs/oss | wujinhu | wujinhu | +| [HDFS-14034](https://issues.apache.org/jira/browse/HDFS-14034) | Support getQuotaUsage API in WebHDFS | Major | fs, webhdfs | Erik Krogen | Chao Sun | +| [YARN-9775](https://issues.apache.org/jira/browse/YARN-9775) | RMWebServices /scheduler-conf GET returns all hadoop configurations for ZKConfigurationStore | Major | restapi | Prabhu Joseph | Prabhu Joseph | +| [HDFS-14771](https://issues.apache.org/jira/browse/HDFS-14771) | Backport HDFS-14617 to branch-2 (Improve fsimage load time by writing sub-sections to the fsimage index) | Major | namenode | Xiaoqiao He | Xiaoqiao He | +| [HDFS-14822](https://issues.apache.org/jira/browse/HDFS-14822) | [SBN read] Revisit GlobalStateIdContext locking when getting server state id | Major | hdfs | Chen Liang | Chen Liang | +| [HDFS-14785](https://issues.apache.org/jira/browse/HDFS-14785) | [SBN read] Change client logging to be less aggressive | Major | hdfs | Chen Liang | Chen Liang | +| [HDFS-14858](https://issues.apache.org/jira/browse/HDFS-14858) | [SBN read] Allow configurably enable/disable AlignmentContext on NameNode | Major | hdfs | Chen Liang | Chen Liang | +| [HDFS-12979](https://issues.apache.org/jira/browse/HDFS-12979) | StandbyNode should upload FsImage to ObserverNode after checkpointing. | Major | hdfs | Konstantin Shvachko | Chen Liang | +| [HADOOP-16630](https://issues.apache.org/jira/browse/HADOOP-16630) | Backport HADOOP-16548 - "ABFS: Config to enable/disable flush operation" to branch-2 | Minor | fs/azure | Sneha Vijayarajan | Sneha Vijayarajan | +| [HADOOP-16631](https://issues.apache.org/jira/browse/HADOOP-16631) | Backport HADOOP-16578 - "ABFS: fileSystemExists() should not call container level apis" to Branch-2 | Major | fs/azure | Sneha Vijayarajan | Sneha Vijayarajan | +| [HDFS-14162](https://issues.apache.org/jira/browse/HDFS-14162) | Balancer should work with ObserverNode | Major | . | Konstantin Shvachko | Erik Krogen | ### OTHER: @@ -482,3 +771,18 @@ | [HADOOP-15149](https://issues.apache.org/jira/browse/HADOOP-15149) | CryptoOutputStream should implement StreamCapabilities | Major | fs | Mike Drob | Xiao Chen | | [HADOOP-15177](https://issues.apache.org/jira/browse/HADOOP-15177) | Update the release year to 2018 | Blocker | build | Akira Ajisaka | Bharat Viswanadham | | [YARN-8412](https://issues.apache.org/jira/browse/YARN-8412) | Move ResourceRequest.clone logic everywhere into a proper API | Minor | . | Botong Huang | Botong Huang | +| [HDFS-13870](https://issues.apache.org/jira/browse/HDFS-13870) | WebHDFS: Document ALLOWSNAPSHOT and DISALLOWSNAPSHOT API doc | Minor | documentation, webhdfs | Siyao Meng | Siyao Meng | +| [HDFS-12729](https://issues.apache.org/jira/browse/HDFS-12729) | Document special paths in HDFS | Major | documentation | Christopher Douglas | Masatake Iwasaki | +| [HADOOP-15711](https://issues.apache.org/jira/browse/HADOOP-15711) | Move branch-2 precommit/nightly test builds to java 8 | Critical | . | Jonathan Hung | Jonathan Hung | +| [HDFS-14510](https://issues.apache.org/jira/browse/HDFS-14510) | Backport HDFS-13087 to branch-2 (Snapshotted encryption zone information should be immutable) | Major | encryption, snapshots | Wei-Chiu Chuang | Siyao Meng | +| [HDFS-14585](https://issues.apache.org/jira/browse/HDFS-14585) | Backport HDFS-8901 Use ByteBuffer in DFSInputStream#read to branch2.9 | Major | . | Lisheng Sun | Lisheng Sun | +| [HDFS-14483](https://issues.apache.org/jira/browse/HDFS-14483) | Backport HDFS-14111,HDFS-3246 ByteBuffer pread interface to branch-2.9 | Major | . | Zheng Hu | Lisheng Sun | +| [YARN-9559](https://issues.apache.org/jira/browse/YARN-9559) | Create AbstractContainersLauncher for pluggable ContainersLauncher logic | Major | . | Jonathan Hung | Jonathan Hung | +| [HDFS-14725](https://issues.apache.org/jira/browse/HDFS-14725) | Backport HDFS-12914 to branch-2 (Block report leases cause missing blocks until next report) | Major | namenode | Wei-Chiu Chuang | Xiaoqiao He | +| [YARN-8200](https://issues.apache.org/jira/browse/YARN-8200) | Backport resource types/GPU features to branch-3.0/branch-2 | Major | . | Jonathan Hung | Jonathan Hung | +| [HADOOP-16555](https://issues.apache.org/jira/browse/HADOOP-16555) | Update commons-compress to 1.19 | Major | . | Wei-Chiu Chuang | YiSheng Lien | +| [YARN-9730](https://issues.apache.org/jira/browse/YARN-9730) | Support forcing configured partitions to be exclusive based on app node label | Major | . | Jonathan Hung | Jonathan Hung | +| [HADOOP-16544](https://issues.apache.org/jira/browse/HADOOP-16544) | update io.netty in branch-2 | Major | . | Wei-Chiu Chuang | Masatake Iwasaki | +| [HADOOP-16588](https://issues.apache.org/jira/browse/HADOOP-16588) | Update commons-beanutils version to 1.9.4 in branch-2 | Critical | . | Wei-Chiu Chuang | Wei-Chiu Chuang | + + diff --git a/hadoop-common-project/hadoop-common/src/site/markdown/release/2.10.0/CHANGES.2.10.0.md b/hadoop-common-project/hadoop-common/src/site/markdown/release/2.10.0/CHANGES.2.10.0.md deleted file mode 100644 index d8dd2daca09cc..0000000000000 --- a/hadoop-common-project/hadoop-common/src/site/markdown/release/2.10.0/CHANGES.2.10.0.md +++ /dev/null @@ -1,788 +0,0 @@ - - -# "Apache Hadoop" Changelog - -## Release 2.10.0 - 2019-10-22 - -### INCOMPATIBLE CHANGES: - -| JIRA | Summary | Priority | Component | Reporter | Contributor | -|:---- |:---- | :--- |:---- |:---- |:---- | -| [HDFS-12883](https://issues.apache.org/jira/browse/HDFS-12883) | RBF: Document Router and State Store metrics | Major | documentation | Yiqun Lin | Yiqun Lin | -| [HDFS-12895](https://issues.apache.org/jira/browse/HDFS-12895) | RBF: Add ACL support for mount table | Major | . | Yiqun Lin | Yiqun Lin | -| [HDFS-13099](https://issues.apache.org/jira/browse/HDFS-13099) | RBF: Use the ZooKeeper as the default State Store | Minor | documentation | Yiqun Lin | Yiqun Lin | -| [HADOOP-16055](https://issues.apache.org/jira/browse/HADOOP-16055) | Upgrade AWS SDK to 1.11.271 in branch-2 | Blocker | fs/s3 | Akira Ajisaka | Akira Ajisaka | -| [HADOOP-16053](https://issues.apache.org/jira/browse/HADOOP-16053) | Backport HADOOP-14816 to branch-2 | Major | build | Akira Ajisaka | Akira Ajisaka | - - -### IMPORTANT ISSUES: - -| JIRA | Summary | Priority | Component | Reporter | Contributor | -|:---- |:---- | :--- |:---- |:---- |:---- | -| [HDFS-13083](https://issues.apache.org/jira/browse/HDFS-13083) | RBF: Fix doc error setting up client | Major | federation | tartarus | tartarus | - - -### NEW FEATURES: - -| JIRA | Summary | Priority | Component | Reporter | Contributor | -|:---- |:---- | :--- |:---- |:---- |:---- | -| [HDFS-13283](https://issues.apache.org/jira/browse/HDFS-13283) | Percentage based Reserved Space Calculation for DataNode | Major | datanode, hdfs | Lukas Majercak | Lukas Majercak | -| [HDFS-13553](https://issues.apache.org/jira/browse/HDFS-13553) | RBF: Support global quota | Major | . | Íñigo Goiri | Yiqun Lin | -| [HADOOP-15950](https://issues.apache.org/jira/browse/HADOOP-15950) | Failover for LdapGroupsMapping | Major | common, security | Lukas Majercak | Lukas Majercak | -| [YARN-9761](https://issues.apache.org/jira/browse/YARN-9761) | Allow overriding application submissions based on server side configs | Major | . | Jonathan Hung | pralabhkumar | -| [YARN-9760](https://issues.apache.org/jira/browse/YARN-9760) | Support configuring application priorities on a workflow level | Major | . | Jonathan Hung | Varun Saxena | - - -### IMPROVEMENTS: - -| JIRA | Summary | Priority | Component | Reporter | Contributor | -|:---- |:---- | :--- |:---- |:---- |:---- | -| [HADOOP-14987](https://issues.apache.org/jira/browse/HADOOP-14987) | Improve KMSClientProvider log around delegation token checking | Major | . | Xiaoyu Yao | Xiaoyu Yao | -| [HADOOP-14872](https://issues.apache.org/jira/browse/HADOOP-14872) | CryptoInputStream should implement unbuffer | Major | fs, security | John Zhuge | John Zhuge | -| [HADOOP-14960](https://issues.apache.org/jira/browse/HADOOP-14960) | Add GC time percentage monitor/alerter | Major | . | Misha Dmitriev | Misha Dmitriev | -| [HADOOP-15023](https://issues.apache.org/jira/browse/HADOOP-15023) | ValueQueue should also validate (lowWatermark \* numValues) \> 0 on construction | Minor | . | Xiao Chen | Xiao Chen | -| [YARN-6851](https://issues.apache.org/jira/browse/YARN-6851) | Capacity Scheduler: document configs for controlling # containers allowed to be allocated per node heartbeat | Minor | . | Wei Yan | Wei Yan | -| [YARN-7495](https://issues.apache.org/jira/browse/YARN-7495) | Improve robustness of the AggregatedLogDeletionService | Major | log-aggregation | Jonathan Turner Eagles | Jonathan Turner Eagles | -| [HADOOP-15056](https://issues.apache.org/jira/browse/HADOOP-15056) | Fix TestUnbuffer#testUnbufferException failure | Minor | test | Jack Bearden | Jack Bearden | -| [HADOOP-15012](https://issues.apache.org/jira/browse/HADOOP-15012) | Add readahead, dropbehind, and unbuffer to StreamCapabilities | Major | fs | John Zhuge | John Zhuge | -| [HADOOP-15104](https://issues.apache.org/jira/browse/HADOOP-15104) | AliyunOSS: change the default value of max error retry | Major | fs/oss | wujinhu | wujinhu | -| [YARN-7274](https://issues.apache.org/jira/browse/YARN-7274) | Ability to disable elasticity at leaf queue level | Major | capacityscheduler | Scott Brokaw | Zian Chen | -| [YARN-7642](https://issues.apache.org/jira/browse/YARN-7642) | Add test case to verify context update after container promotion or demotion with or without auto update | Minor | nodemanager | Weiwei Yang | Weiwei Yang | -| [HADOOP-15111](https://issues.apache.org/jira/browse/HADOOP-15111) | AliyunOSS: backport HADOOP-14993 to branch-2 | Major | fs/oss | Genmao Yu | Genmao Yu | -| [HDFS-12818](https://issues.apache.org/jira/browse/HDFS-12818) | Support multiple storages in DataNodeCluster / SimulatedFSDataset | Minor | datanode, test | Erik Krogen | Erik Krogen | -| [HDFS-9023](https://issues.apache.org/jira/browse/HDFS-9023) | When NN is not able to identify DN for replication, reason behind it can be logged | Critical | hdfs-client, namenode | Surendra Singh Lilhore | Xiao Chen | -| [YARN-7678](https://issues.apache.org/jira/browse/YARN-7678) | Ability to enable logging of container memory stats | Major | nodemanager | Jim Brennan | Jim Brennan | -| [HDFS-12945](https://issues.apache.org/jira/browse/HDFS-12945) | Switch to ClientProtocol instead of NamenodeProtocols in NamenodeWebHdfsMethods | Minor | . | Wei Yan | Wei Yan | -| [YARN-7622](https://issues.apache.org/jira/browse/YARN-7622) | Allow fair-scheduler configuration on HDFS | Minor | fairscheduler, resourcemanager | Greg Phillips | Greg Phillips | -| [YARN-7590](https://issues.apache.org/jira/browse/YARN-7590) | Improve container-executor validation check | Major | security, yarn | Eric Yang | Eric Yang | -| [MAPREDUCE-7029](https://issues.apache.org/jira/browse/MAPREDUCE-7029) | FileOutputCommitter is slow on filesystems lacking recursive delete | Minor | . | Karthik Palaniappan | Karthik Palaniappan | -| [MAPREDUCE-6984](https://issues.apache.org/jira/browse/MAPREDUCE-6984) | MR AM to clean up temporary files from previous attempt in case of no recovery | Major | applicationmaster | Gergo Repas | Gergo Repas | -| [HADOOP-15189](https://issues.apache.org/jira/browse/HADOOP-15189) | backport HADOOP-15039 to branch-2 and branch-3 | Blocker | . | Genmao Yu | Genmao Yu | -| [HADOOP-15212](https://issues.apache.org/jira/browse/HADOOP-15212) | Add independent secret manager method for logging expired tokens | Major | security | Daryn Sharp | Daryn Sharp | -| [YARN-7728](https://issues.apache.org/jira/browse/YARN-7728) | Expose container preemptions related information in Capacity Scheduler queue metrics | Major | . | Eric Payne | Eric Payne | -| [MAPREDUCE-7048](https://issues.apache.org/jira/browse/MAPREDUCE-7048) | Uber AM can crash due to unknown task in statusUpdate | Major | mr-am | Peter Bacsko | Peter Bacsko | -| [HADOOP-13972](https://issues.apache.org/jira/browse/HADOOP-13972) | ADLS to support per-store configuration | Major | fs/adl | John Zhuge | Sharad Sonker | -| [YARN-7813](https://issues.apache.org/jira/browse/YARN-7813) | Capacity Scheduler Intra-queue Preemption should be configurable for each queue | Major | capacity scheduler, scheduler preemption | Eric Payne | Eric Payne | -| [HADOOP-15235](https://issues.apache.org/jira/browse/HADOOP-15235) | Authentication Tokens should use HMAC instead of MAC | Major | security | Robert Kanter | Robert Kanter | -| [HDFS-11187](https://issues.apache.org/jira/browse/HDFS-11187) | Optimize disk access for last partial chunk checksum of Finalized replica | Major | datanode | Wei-Chiu Chuang | Gabor Bota | -| [HADOOP-15266](https://issues.apache.org/jira/browse/HADOOP-15266) | [branch-2] Upper/Lower case conversion support for group names in LdapGroupsMapping | Major | . | Nanda kumar | Nanda kumar | -| [HADOOP-15279](https://issues.apache.org/jira/browse/HADOOP-15279) | increase maven heap size recommendations | Minor | build, documentation, test | Allen Wittenauer | Allen Wittenauer | -| [HDFS-12884](https://issues.apache.org/jira/browse/HDFS-12884) | BlockUnderConstructionFeature.truncateBlock should be of type BlockInfo | Major | namenode | Konstantin Shvachko | chencan | -| [HADOOP-15334](https://issues.apache.org/jira/browse/HADOOP-15334) | Upgrade Maven surefire plugin | Major | build | Arpit Agarwal | Arpit Agarwal | -| [HADOOP-15312](https://issues.apache.org/jira/browse/HADOOP-15312) | Undocumented KeyProvider configuration keys | Major | . | Wei-Chiu Chuang | LiXin Ge | -| [YARN-7623](https://issues.apache.org/jira/browse/YARN-7623) | Fix the CapacityScheduler Queue configuration documentation | Major | . | Arun Suresh | Jonathan Hung | -| [HDFS-13314](https://issues.apache.org/jira/browse/HDFS-13314) | NameNode should optionally exit if it detects FsImage corruption | Major | namenode | Arpit Agarwal | Arpit Agarwal | -| [HDFS-13418](https://issues.apache.org/jira/browse/HDFS-13418) | NetworkTopology should be configurable when enable DFSNetworkTopology | Major | . | Tao Jie | Tao Jie | -| [HADOOP-15394](https://issues.apache.org/jira/browse/HADOOP-15394) | Backport PowerShell NodeFencer HADOOP-14309 to branch-2 | Minor | . | Íñigo Goiri | Íñigo Goiri | -| [HDFS-13462](https://issues.apache.org/jira/browse/HDFS-13462) | Add BIND\_HOST configuration for JournalNode's HTTP and RPC Servers | Major | hdfs, journal-node | Lukas Majercak | Lukas Majercak | -| [HDFS-13492](https://issues.apache.org/jira/browse/HDFS-13492) | Limit httpfs binds to certain IP addresses in branch-2 | Major | httpfs | Wei-Chiu Chuang | Wei-Chiu Chuang | -| [HDFS-12981](https://issues.apache.org/jira/browse/HDFS-12981) | renameSnapshot a Non-Existent snapshot to itself should throw error | Minor | hdfs | Sailesh Patel | Kitti Nanasi | -| [HADOOP-15441](https://issues.apache.org/jira/browse/HADOOP-15441) | Log kms url and token service at debug level. | Minor | . | Wei-Chiu Chuang | Gabor Bota | -| [HDFS-13544](https://issues.apache.org/jira/browse/HDFS-13544) | Improve logging for JournalNode in federated cluster | Major | federation, hdfs | Hanisha Koneru | Hanisha Koneru | -| [YARN-8249](https://issues.apache.org/jira/browse/YARN-8249) | Few REST api's in RMWebServices are missing static user check | Critical | webapp, yarn | Sunil G | Sunil G | -| [HADOOP-15486](https://issues.apache.org/jira/browse/HADOOP-15486) | Make NetworkTopology#netLock fair | Major | net | Nanda kumar | Nanda kumar | -| [HADOOP-15449](https://issues.apache.org/jira/browse/HADOOP-15449) | Increase default timeout of ZK session to avoid frequent NameNode failover | Critical | common | Karthik Palanisamy | Karthik Palanisamy | -| [HDFS-13602](https://issues.apache.org/jira/browse/HDFS-13602) | Add checkOperation(WRITE) checks in FSNamesystem | Major | ha, namenode | Erik Krogen | Chao Sun | -| [HDFS-13644](https://issues.apache.org/jira/browse/HDFS-13644) | Backport HDFS-10376 to branch-2 | Major | . | Yiqun Lin | Zsolt Venczel | -| [HDFS-13653](https://issues.apache.org/jira/browse/HDFS-13653) | Make dfs.client.failover.random.order a per nameservice configuration | Major | federation | Ekanth Sethuramalingam | Ekanth Sethuramalingam | -| [HDFS-13686](https://issues.apache.org/jira/browse/HDFS-13686) | Add overall metrics for FSNamesystemLock | Major | hdfs, namenode | Lukas Majercak | Lukas Majercak | -| [HDFS-13714](https://issues.apache.org/jira/browse/HDFS-13714) | Fix TestNameNodePrunesMissingStorages test failures on Windows | Major | hdfs, namenode, test | Lukas Majercak | Lukas Majercak | -| [HDFS-13719](https://issues.apache.org/jira/browse/HDFS-13719) | Docs around dfs.image.transfer.timeout are misleading | Major | documentation | Kitti Nanasi | Kitti Nanasi | -| [HDFS-11060](https://issues.apache.org/jira/browse/HDFS-11060) | make DEFAULT\_MAX\_CORRUPT\_FILEBLOCKS\_RETURNED configurable | Minor | hdfs | Lantao Jin | Lantao Jin | -| [YARN-8155](https://issues.apache.org/jira/browse/YARN-8155) | Improve ATSv2 client logging in RM and NM publisher | Major | . | Rohith Sharma K S | Abhishek Modi | -| [HDFS-13814](https://issues.apache.org/jira/browse/HDFS-13814) | Remove super user privilege requirement for NameNode.getServiceStatus | Minor | namenode | Chao Sun | Chao Sun | -| [YARN-8559](https://issues.apache.org/jira/browse/YARN-8559) | Expose mutable-conf scheduler's configuration in RM /scheduler-conf endpoint | Major | resourcemanager | Anna Savarin | Weiwei Yang | -| [HDFS-13813](https://issues.apache.org/jira/browse/HDFS-13813) | Exit NameNode if dangling child inode is detected when saving FsImage | Major | hdfs, namenode | Siyao Meng | Siyao Meng | -| [HDFS-13821](https://issues.apache.org/jira/browse/HDFS-13821) | RBF: Add dfs.federation.router.mount-table.cache.enable so that users can disable cache | Major | hdfs | Fei Hui | Fei Hui | -| [HADOOP-15689](https://issues.apache.org/jira/browse/HADOOP-15689) | Add "\*.patch" into .gitignore file of branch-2 | Major | . | Rui Gao | Rui Gao | -| [HDFS-13831](https://issues.apache.org/jira/browse/HDFS-13831) | Make block increment deletion number configurable | Major | . | Yiqun Lin | Ryan Wu | -| [YARN-8051](https://issues.apache.org/jira/browse/YARN-8051) | TestRMEmbeddedElector#testCallbackSynchronization is flakey | Major | test | Robert Kanter | Robert Kanter | -| [HADOOP-15547](https://issues.apache.org/jira/browse/HADOOP-15547) | WASB: improve listStatus performance | Major | fs/azure | Thomas Marqardt | Thomas Marqardt | -| [HDFS-13857](https://issues.apache.org/jira/browse/HDFS-13857) | RBF: Choose to enable the default nameservice to read/write files | Major | federation, hdfs | yanghuafeng | yanghuafeng | -| [HDFS-13812](https://issues.apache.org/jira/browse/HDFS-13812) | Fix the inconsistent default refresh interval on Caching documentation | Trivial | documentation | David Mollitor | Hrishikesh Gadre | -| [HADOOP-15657](https://issues.apache.org/jira/browse/HADOOP-15657) | Registering MutableQuantiles via Metric annotation | Major | metrics | Sushil Ks | Sushil Ks | -| [HDFS-13902](https://issues.apache.org/jira/browse/HDFS-13902) | Add JMX, conf and stacks menus to the datanode page | Minor | datanode | fengchuang | fengchuang | -| [HADOOP-15726](https://issues.apache.org/jira/browse/HADOOP-15726) | Create utility to limit frequency of log statements | Major | common, util | Erik Krogen | Erik Krogen | -| [YARN-7974](https://issues.apache.org/jira/browse/YARN-7974) | Allow updating application tracking url after registration | Major | . | Jonathan Hung | Jonathan Hung | -| [YARN-8750](https://issues.apache.org/jira/browse/YARN-8750) | Refactor TestQueueMetrics | Minor | resourcemanager | Szilard Nemeth | Szilard Nemeth | -| [YARN-8896](https://issues.apache.org/jira/browse/YARN-8896) | Limit the maximum number of container assignments per heartbeat | Major | . | Weiwei Yang | Zhankun Tang | -| [HADOOP-15804](https://issues.apache.org/jira/browse/HADOOP-15804) | upgrade to commons-compress 1.18 | Major | . | PJ Fanning | Akira Ajisaka | -| [YARN-8915](https://issues.apache.org/jira/browse/YARN-8915) | Update the doc about the default value of "maximum-container-assignments" for capacity scheduler | Minor | . | Zhankun Tang | Zhankun Tang | -| [YARN-7225](https://issues.apache.org/jira/browse/YARN-7225) | Add queue and partition info to RM audit log | Major | resourcemanager | Jonathan Hung | Eric Payne | -| [HADOOP-15919](https://issues.apache.org/jira/browse/HADOOP-15919) | AliyunOSS: Enable Yarn to use OSS | Major | fs/oss | wujinhu | wujinhu | -| [HADOOP-15943](https://issues.apache.org/jira/browse/HADOOP-15943) | AliyunOSS: add missing owner & group attributes for oss FileStatus | Major | fs/oss | wujinhu | wujinhu | -| [YARN-9036](https://issues.apache.org/jira/browse/YARN-9036) | Escape newlines in health report in YARN UI | Major | . | Jonathan Hung | Keqiu Hu | -| [YARN-9085](https://issues.apache.org/jira/browse/YARN-9085) | Add Guaranteed and MaxCapacity to CSQueueMetrics | Major | . | Jonathan Hung | Jonathan Hung | -| [HDFS-14171](https://issues.apache.org/jira/browse/HDFS-14171) | Performance improvement in Tailing EditLog | Major | namenode | Kenneth Yang | Kenneth Yang | -| [HADOOP-15481](https://issues.apache.org/jira/browse/HADOOP-15481) | Emit FairCallQueue stats as metrics | Major | metrics, rpc-server | Erik Krogen | Christopher Gregorian | -| [HADOOP-15617](https://issues.apache.org/jira/browse/HADOOP-15617) | Node.js and npm package loading in the Dockerfile failing on branch-2 | Major | build | Allen Wittenauer | Akhil PB | -| [HADOOP-16089](https://issues.apache.org/jira/browse/HADOOP-16089) | AliyunOSS: update oss-sdk version to 3.4.1 | Major | fs/oss | wujinhu | wujinhu | -| [YARN-7171](https://issues.apache.org/jira/browse/YARN-7171) | RM UI should sort memory / cores numerically | Major | . | Eric Maynard | Ahmed Hussein | -| [YARN-9282](https://issues.apache.org/jira/browse/YARN-9282) | Typo in javadoc of class LinuxContainerExecutor: hadoop.security.authetication should be 'authentication' | Trivial | . | Szilard Nemeth | Charan Hebri | -| [HADOOP-16126](https://issues.apache.org/jira/browse/HADOOP-16126) | ipc.Client.stop() may sleep too long to wait for all connections | Major | ipc | Tsz-wo Sze | Tsz-wo Sze | -| [HDFS-14247](https://issues.apache.org/jira/browse/HDFS-14247) | Repeat adding node description into network topology | Minor | datanode | HuangTao | HuangTao | -| [YARN-9150](https://issues.apache.org/jira/browse/YARN-9150) | Making TimelineSchemaCreator support different backends for Timeline Schema Creation in ATSv2 | Major | ATSv2 | Sushil Ks | Sushil Ks | -| [HDFS-14366](https://issues.apache.org/jira/browse/HDFS-14366) | Improve HDFS append performance | Major | hdfs | Chao Sun | Chao Sun | -| [HDFS-14205](https://issues.apache.org/jira/browse/HDFS-14205) | Backport HDFS-6440 to branch-2 | Major | . | Chen Liang | Chao Sun | -| [HDFS-14391](https://issues.apache.org/jira/browse/HDFS-14391) | Backport HDFS-9659 to branch-2 | Minor | . | Chao Sun | Chao Sun | -| [HDFS-14415](https://issues.apache.org/jira/browse/HDFS-14415) | Backport HDFS-13799 to branch-2 | Trivial | . | Chao Sun | Chao Sun | -| [HDFS-14432](https://issues.apache.org/jira/browse/HDFS-14432) | dfs.datanode.shared.file.descriptor.paths duplicated in hdfs-default.xml | Minor | hdfs | puleya7 | puleya7 | -| [YARN-9529](https://issues.apache.org/jira/browse/YARN-9529) | Log correct cpu controller path on error while initializing CGroups. | Major | nodemanager | Jonathan Hung | Jonathan Hung | -| [HDFS-14502](https://issues.apache.org/jira/browse/HDFS-14502) | keepResults option in NNThroughputBenchmark should call saveNamespace() | Major | benchmarks, hdfs | Konstantin Shvachko | Konstantin Shvachko | -| [HADOOP-16323](https://issues.apache.org/jira/browse/HADOOP-16323) | https everywhere in Maven settings | Minor | build | Akira Ajisaka | Akira Ajisaka | -| [YARN-9563](https://issues.apache.org/jira/browse/YARN-9563) | Resource report REST API could return NaN or Inf | Minor | . | Ahmed Hussein | Ahmed Hussein | -| [HDFS-14513](https://issues.apache.org/jira/browse/HDFS-14513) | FSImage which is saving should be clean while NameNode shutdown | Major | namenode | Xiaoqiao He | Xiaoqiao He | -| [HADOOP-16369](https://issues.apache.org/jira/browse/HADOOP-16369) | Fix zstandard shortname misspelled as zts | Major | . | Jonathan Turner Eagles | Jonathan Turner Eagles | -| [HADOOP-16359](https://issues.apache.org/jira/browse/HADOOP-16359) | Bundle ZSTD native in branch-2 | Major | native | Chao Sun | Chao Sun | -| [HADOOP-15253](https://issues.apache.org/jira/browse/HADOOP-15253) | Should update maxQueueSize when refresh call queue | Minor | . | Tao Jie | Tao Jie | -| [HADOOP-16266](https://issues.apache.org/jira/browse/HADOOP-16266) | Add more fine-grained processing time metrics to the RPC layer | Minor | ipc | Christopher Gregorian | Erik Krogen | -| [HDFS-13694](https://issues.apache.org/jira/browse/HDFS-13694) | Making md5 computing being in parallel with image loading | Major | . | zhouyingchao | Lisheng Sun | -| [HDFS-14632](https://issues.apache.org/jira/browse/HDFS-14632) | Reduce useless #getNumLiveDataNodes call in SafeModeMonitor | Major | namenode | Xiaoqiao He | Xiaoqiao He | -| [HDFS-14547](https://issues.apache.org/jira/browse/HDFS-14547) | DirectoryWithQuotaFeature.quota costs additional memory even the storage type quota is not set. | Major | . | Jinglun | Jinglun | -| [HDFS-14697](https://issues.apache.org/jira/browse/HDFS-14697) | Backport HDFS-14513 to branch-2 | Minor | namenode | Xiaoqiao He | Xiaoqiao He | -| [YARN-8045](https://issues.apache.org/jira/browse/YARN-8045) | Reduce log output from container status calls | Major | . | Shane Kumpf | Craig Condit | -| [HDFS-14313](https://issues.apache.org/jira/browse/HDFS-14313) | Get hdfs used space from FsDatasetImpl#volumeMap#ReplicaInfo in memory instead of df/du | Major | datanode, performance | Lisheng Sun | Lisheng Sun | -| [HDFS-14696](https://issues.apache.org/jira/browse/HDFS-14696) | Backport HDFS-11273 to branch-2 (Move TransferFsImage#doGetUrl function to a Util class) | Major | . | Siyao Meng | Siyao Meng | -| [HDFS-14370](https://issues.apache.org/jira/browse/HDFS-14370) | Edit log tailing fast-path should allow for backoff | Major | namenode, qjm | Erik Krogen | Erik Krogen | -| [YARN-9442](https://issues.apache.org/jira/browse/YARN-9442) | container working directory has group read permissions | Minor | yarn | Jim Brennan | Jim Brennan | -| [HADOOP-16459](https://issues.apache.org/jira/browse/HADOOP-16459) | Backport [HADOOP-16266] "Add more fine-grained processing time metrics to the RPC layer" to branch-2 | Major | . | Erik Krogen | Erik Krogen | -| [HDFS-14707](https://issues.apache.org/jira/browse/HDFS-14707) | Add JAVA\_LIBRARY\_PATH to HTTPFS startup options in branch-2 | Major | httpfs | Masatake Iwasaki | Masatake Iwasaki | -| [HDFS-14723](https://issues.apache.org/jira/browse/HDFS-14723) | Add helper method FSNamesystem#setBlockManagerForTesting() in branch-2 | Blocker | namenode | Wei-Chiu Chuang | Wei-Chiu Chuang | -| [HDFS-14276](https://issues.apache.org/jira/browse/HDFS-14276) | [SBN read] Reduce tailing overhead | Major | ha, namenode | Wei-Chiu Chuang | Ayush Saxena | -| [HDFS-14617](https://issues.apache.org/jira/browse/HDFS-14617) | Improve fsimage load time by writing sub-sections to the fsimage index | Major | namenode | Stephen O'Donnell | Stephen O'Donnell | -| [YARN-9756](https://issues.apache.org/jira/browse/YARN-9756) | Create metric that sums total memory/vcores preempted per round | Major | capacity scheduler | Eric Payne | Manikandan R | -| [HDFS-14633](https://issues.apache.org/jira/browse/HDFS-14633) | The StorageType quota and consume in QuotaFeature is not handled for rename | Major | . | Jinglun | Jinglun | -| [HADOOP-16439](https://issues.apache.org/jira/browse/HADOOP-16439) | Upgrade bundled Tomcat in branch-2 | Major | httpfs, kms | Masatake Iwasaki | Masatake Iwasaki | -| [YARN-9810](https://issues.apache.org/jira/browse/YARN-9810) | Add queue capacity/maxcapacity percentage metrics | Major | . | Jonathan Hung | Shubham Gupta | -| [YARN-9763](https://issues.apache.org/jira/browse/YARN-9763) | Print application tags in application summary | Major | . | Jonathan Hung | Manoj Kumar | -| [YARN-9764](https://issues.apache.org/jira/browse/YARN-9764) | Print application submission context label in application summary | Major | . | Jonathan Hung | Manoj Kumar | -| [HADOOP-16530](https://issues.apache.org/jira/browse/HADOOP-16530) | Update xercesImpl in branch-2 | Major | . | Wei-Chiu Chuang | Wei-Chiu Chuang | -| [YARN-9824](https://issues.apache.org/jira/browse/YARN-9824) | Fall back to configured queue ordering policy class name | Major | . | Jonathan Hung | Jonathan Hung | -| [YARN-9825](https://issues.apache.org/jira/browse/YARN-9825) | Changes for initializing placement rules with ResourceScheduler in branch-2 | Major | . | Jonathan Hung | Jonathan Hung | -| [YARN-9762](https://issues.apache.org/jira/browse/YARN-9762) | Add submission context label to audit logs | Major | . | Jonathan Hung | Manoj Kumar | -| [HDFS-14667](https://issues.apache.org/jira/browse/HDFS-14667) | Backport [HDFS-14403] "Cost-based FairCallQueue" to branch-2 | Major | . | Erik Krogen | Erik Krogen | - - -### BUG FIXES: - -| JIRA | Summary | Priority | Component | Reporter | Contributor | -|:---- |:---- | :--- |:---- |:---- |:---- | -| [MAPREDUCE-6896](https://issues.apache.org/jira/browse/MAPREDUCE-6896) | Document wrong spelling in usage of MapredTestDriver tools. | Major | documentation | LiXin Ge | LiXin Ge | -| [HDFS-12052](https://issues.apache.org/jira/browse/HDFS-12052) | Set SWEBHDFS delegation token kind when ssl is enabled in HttpFS | Major | httpfs, webhdfs | Zoran Dimitrijevic | Zoran Dimitrijevic | -| [HDFS-12318](https://issues.apache.org/jira/browse/HDFS-12318) | Fix IOException condition for openInfo in DFSInputStream | Major | . | legend | legend | -| [HDFS-12614](https://issues.apache.org/jira/browse/HDFS-12614) | FSPermissionChecker#getINodeAttrs() throws NPE when INodeAttributesProvider configured | Major | . | Manoj Govindassamy | Manoj Govindassamy | -| [YARN-7396](https://issues.apache.org/jira/browse/YARN-7396) | NPE when accessing container logs due to null dirsHandler | Major | . | Jonathan Hung | Jonathan Hung | -| [YARN-7370](https://issues.apache.org/jira/browse/YARN-7370) | Preemption properties should be refreshable | Major | capacity scheduler, scheduler preemption | Eric Payne | Gergely Novák | -| [YARN-7428](https://issues.apache.org/jira/browse/YARN-7428) | Add containerId to Localizer failed logs | Minor | nodemanager | Prabhu Joseph | Prabhu Joseph | -| [YARN-7410](https://issues.apache.org/jira/browse/YARN-7410) | Cleanup FixedValueResource to avoid dependency to ResourceUtils | Major | resourcemanager | Sunil G | Wangda Tan | -| [HDFS-12783](https://issues.apache.org/jira/browse/HDFS-12783) | [branch-2] "dfsrouter" should use hdfsScript | Major | . | Brahma Reddy Battula | Brahma Reddy Battula | -| [HDFS-12788](https://issues.apache.org/jira/browse/HDFS-12788) | Reset the upload button when file upload fails | Critical | ui, webhdfs | Brahma Reddy Battula | Brahma Reddy Battula | -| [YARN-7469](https://issues.apache.org/jira/browse/YARN-7469) | Capacity Scheduler Intra-queue preemption: User can starve if newest app is exactly at user limit | Major | capacity scheduler, yarn | Eric Payne | Eric Payne | -| [HADOOP-14982](https://issues.apache.org/jira/browse/HADOOP-14982) | Clients using FailoverOnNetworkExceptionRetry can go into a loop if they're used without authenticating with kerberos in HA env | Major | common | Peter Bacsko | Peter Bacsko | -| [YARN-7489](https://issues.apache.org/jira/browse/YARN-7489) | ConcurrentModificationException in RMAppImpl#getRMAppMetrics | Major | capacityscheduler | Tao Yang | Tao Yang | -| [YARN-7525](https://issues.apache.org/jira/browse/YARN-7525) | Incorrect query parameters in cluster nodes REST API document | Minor | documentation | Tao Yang | Tao Yang | -| [HDFS-12813](https://issues.apache.org/jira/browse/HDFS-12813) | RequestHedgingProxyProvider can hide Exception thrown from the Namenode for proxy size of 1 | Major | ha | Mukul Kumar Singh | Mukul Kumar Singh | -| [HADOOP-15045](https://issues.apache.org/jira/browse/HADOOP-15045) | ISA-L build options are documented in branch-2 | Major | build, documentation | Akira Ajisaka | Akira Ajisaka | -| [HADOOP-15067](https://issues.apache.org/jira/browse/HADOOP-15067) | GC time percentage reported in JvmMetrics should be a gauge, not counter | Major | . | Misha Dmitriev | Misha Dmitriev | -| [YARN-7363](https://issues.apache.org/jira/browse/YARN-7363) | ContainerLocalizer doesn't have a valid log4j config when using LinuxContainerExecutor | Major | nodemanager | Yufei Gu | Yufei Gu | -| [HDFS-12754](https://issues.apache.org/jira/browse/HDFS-12754) | Lease renewal can hit a deadlock | Major | . | Kuhu Shukla | Kuhu Shukla | -| [HDFS-12832](https://issues.apache.org/jira/browse/HDFS-12832) | INode.getFullPathName may throw ArrayIndexOutOfBoundsException lead to NameNode exit | Critical | namenode | DENG FEI | Konstantin Shvachko | -| [HDFS-11754](https://issues.apache.org/jira/browse/HDFS-11754) | Make FsServerDefaults cache configurable. | Minor | . | Rushabh Shah | Mikhail Erofeev | -| [HADOOP-15042](https://issues.apache.org/jira/browse/HADOOP-15042) | Azure PageBlobInputStream.skip() can return negative value when numberOfPagesRemaining is 0 | Minor | fs/azure | Rajesh Balamohan | Rajesh Balamohan | -| [HDFS-12638](https://issues.apache.org/jira/browse/HDFS-12638) | Delete copy-on-truncate block along with the original block, when deleting a file being truncated | Blocker | hdfs | Jiandan Yang | Konstantin Shvachko | -| [YARN-4813](https://issues.apache.org/jira/browse/YARN-4813) | TestRMWebServicesDelegationTokenAuthentication.testDoAs fails intermittently | Major | resourcemanager | Daniel Templeton | Gergo Repas | -| [MAPREDUCE-5124](https://issues.apache.org/jira/browse/MAPREDUCE-5124) | AM lacks flow control for task events | Major | mr-am | Jason Darrell Lowe | Peter Bacsko | -| [YARN-7455](https://issues.apache.org/jira/browse/YARN-7455) | quote\_and\_append\_arg can overflow buffer | Major | nodemanager | Jason Darrell Lowe | Jim Brennan | -| [YARN-7594](https://issues.apache.org/jira/browse/YARN-7594) | TestNMWebServices#testGetNMResourceInfo fails on trunk | Major | nodemanager, webapp | Gergely Novák | Gergely Novák | -| [YARN-5594](https://issues.apache.org/jira/browse/YARN-5594) | Handle old RMDelegationToken format when recovering RM | Major | resourcemanager | Tatyana But | Robert Kanter | -| [HADOOP-14985](https://issues.apache.org/jira/browse/HADOOP-14985) | Remove subversion related code from VersionInfoMojo.java | Minor | build | Akira Ajisaka | Ajay Kumar | -| [HDFS-11751](https://issues.apache.org/jira/browse/HDFS-11751) | DFSZKFailoverController daemon exits with wrong status code | Major | auto-failover | Doris Gu | Bharat Viswanadham | -| [HDFS-12889](https://issues.apache.org/jira/browse/HDFS-12889) | Router UI is missing robots.txt file | Major | . | Bharat Viswanadham | Bharat Viswanadham | -| [YARN-7607](https://issues.apache.org/jira/browse/YARN-7607) | Remove the trailing duplicated timestamp in container diagnostics message | Minor | nodemanager | Weiwei Yang | Weiwei Yang | -| [HADOOP-15080](https://issues.apache.org/jira/browse/HADOOP-15080) | Aliyun OSS: update oss sdk from 2.8.1 to 2.8.3 to remove its dependency on Cat-x "json-lib" | Blocker | fs/oss | Christopher Douglas | Sammi Chen | -| [YARN-7608](https://issues.apache.org/jira/browse/YARN-7608) | Incorrect sTarget column causing DataTable warning on RM application and scheduler web page | Major | resourcemanager, webapp | Weiwei Yang | Gergely Novák | -| [HDFS-12833](https://issues.apache.org/jira/browse/HDFS-12833) | Distcp : Update the usage of delete option for dependency with update and overwrite option | Minor | distcp, hdfs | Harshakiran Reddy | usharani | -| [YARN-7647](https://issues.apache.org/jira/browse/YARN-7647) | NM print inappropriate error log when node-labels is enabled | Minor | . | Yang Wang | Yang Wang | -| [HDFS-12907](https://issues.apache.org/jira/browse/HDFS-12907) | Allow read-only access to reserved raw for non-superusers | Major | namenode | Daryn Sharp | Rushabh Shah | -| [HDFS-12881](https://issues.apache.org/jira/browse/HDFS-12881) | Output streams closed with IOUtils suppressing write errors | Major | . | Jason Darrell Lowe | Ajay Kumar | -| [YARN-7595](https://issues.apache.org/jira/browse/YARN-7595) | Container launching code suppresses close exceptions after writes | Major | nodemanager | Jason Darrell Lowe | Jim Brennan | -| [HADOOP-15085](https://issues.apache.org/jira/browse/HADOOP-15085) | Output streams closed with IOUtils suppressing write errors | Major | . | Jason Darrell Lowe | Jim Brennan | -| [HADOOP-15123](https://issues.apache.org/jira/browse/HADOOP-15123) | KDiag tries to load krb5.conf from KRB5CCNAME instead of KRB5\_CONFIG | Minor | security | Vipin Rathor | Vipin Rathor | -| [YARN-7661](https://issues.apache.org/jira/browse/YARN-7661) | NodeManager metrics return wrong value after update node resource | Major | . | Yang Wang | Yang Wang | -| [HDFS-12347](https://issues.apache.org/jira/browse/HDFS-12347) | TestBalancerRPCDelay#testBalancerRPCDelay fails very frequently | Critical | test | Xiao Chen | Bharat Viswanadham | -| [YARN-7662](https://issues.apache.org/jira/browse/YARN-7662) | [Atsv2] Define new set of configurations for reader and collectors to bind. | Major | . | Rohith Sharma K S | Rohith Sharma K S | -| [YARN-7674](https://issues.apache.org/jira/browse/YARN-7674) | Update Timeline Reader web app address in UI2 | Major | . | Rohith Sharma K S | Sunil G | -| [YARN-7542](https://issues.apache.org/jira/browse/YARN-7542) | Fix issue that causes some Running Opportunistic Containers to be recovered as PAUSED | Major | . | Arun Suresh | Sampada Dehankar | -| [HADOOP-15143](https://issues.apache.org/jira/browse/HADOOP-15143) | NPE due to Invalid KerberosTicket in UGI | Major | . | Jitendra Nath Pandey | Mukul Kumar Singh | -| [YARN-7692](https://issues.apache.org/jira/browse/YARN-7692) | Skip validating priority acls while recovering applications | Blocker | resourcemanager | Charan Hebri | Sunil G | -| [MAPREDUCE-7028](https://issues.apache.org/jira/browse/MAPREDUCE-7028) | Concurrent task progress updates causing NPE in Application Master | Blocker | mr-am | Gergo Repas | Gergo Repas | -| [YARN-7619](https://issues.apache.org/jira/browse/YARN-7619) | Max AM Resource value in Capacity Scheduler UI has to be refreshed for every user | Major | capacity scheduler, yarn | Eric Payne | Eric Payne | -| [YARN-7699](https://issues.apache.org/jira/browse/YARN-7699) | queueUsagePercentage is coming as INF for getApp REST api call | Major | webapp | Sunil G | Sunil G | -| [HDFS-12985](https://issues.apache.org/jira/browse/HDFS-12985) | NameNode crashes during restart after an OpenForWrite file present in the Snapshot got deleted | Major | hdfs | Manoj Govindassamy | Manoj Govindassamy | -| [YARN-4227](https://issues.apache.org/jira/browse/YARN-4227) | Ignore expired containers from removed nodes in FairScheduler | Critical | fairscheduler | Wilfred Spiegelenburg | Wilfred Spiegelenburg | -| [YARN-7508](https://issues.apache.org/jira/browse/YARN-7508) | NPE in FiCaSchedulerApp when debug log enabled in async-scheduling mode | Major | capacityscheduler | Tao Yang | Tao Yang | -| [YARN-7663](https://issues.apache.org/jira/browse/YARN-7663) | RMAppImpl:Invalid event: START at KILLED | Major | resourcemanager | lujie | lujie | -| [YARN-6948](https://issues.apache.org/jira/browse/YARN-6948) | Invalid event: ATTEMPT\_ADDED at FINAL\_SAVING | Major | yarn | lujie | lujie | -| [HADOOP-15060](https://issues.apache.org/jira/browse/HADOOP-15060) | TestShellBasedUnixGroupsMapping.testFiniteGroupResolutionTime flaky | Major | . | Miklos Szegedi | Miklos Szegedi | -| [YARN-7735](https://issues.apache.org/jira/browse/YARN-7735) | Fix typo in YARN documentation | Minor | documentation | Takanobu Asanuma | Takanobu Asanuma | -| [YARN-7727](https://issues.apache.org/jira/browse/YARN-7727) | Incorrect log levels in few logs with QueuePriorityContainerCandidateSelector | Minor | yarn | Prabhu Joseph | Prabhu Joseph | -| [HDFS-11915](https://issues.apache.org/jira/browse/HDFS-11915) | Sync rbw dir on the first hsync() to avoid file lost on power failure | Critical | . | Kanaka Kumar Avvaru | Vinayakumar B | -| [YARN-7705](https://issues.apache.org/jira/browse/YARN-7705) | Create the container log directory with correct sticky bit in C code | Major | nodemanager | Yufei Gu | Yufei Gu | -| [HDFS-9049](https://issues.apache.org/jira/browse/HDFS-9049) | Make Datanode Netty reverse proxy port to be configurable | Major | datanode | Vinayakumar B | Vinayakumar B | -| [YARN-7758](https://issues.apache.org/jira/browse/YARN-7758) | Add an additional check to the validity of container and application ids passed to container-executor | Major | nodemanager | Miklos Szegedi | Yufei Gu | -| [HADOOP-15150](https://issues.apache.org/jira/browse/HADOOP-15150) | in FsShell, UGI params should be overidden through env vars(-D arg) | Major | . | Brahma Reddy Battula | Brahma Reddy Battula | -| [HADOOP-15181](https://issues.apache.org/jira/browse/HADOOP-15181) | Typo in SecureMode.md | Trivial | documentation | Masahiro Tanaka | Masahiro Tanaka | -| [YARN-7806](https://issues.apache.org/jira/browse/YARN-7806) | Distributed Shell should use timeline async api's | Major | distributed-shell | Sumana Sathish | Rohith Sharma K S | -| [HADOOP-15121](https://issues.apache.org/jira/browse/HADOOP-15121) | Encounter NullPointerException when using DecayRpcScheduler | Major | . | Tao Jie | Tao Jie | -| [MAPREDUCE-7015](https://issues.apache.org/jira/browse/MAPREDUCE-7015) | Possible race condition in JHS if the job is not loaded | Major | jobhistoryserver | Peter Bacsko | Peter Bacsko | -| [YARN-7737](https://issues.apache.org/jira/browse/YARN-7737) | prelaunch.err file not found exception on container failure | Major | . | Jonathan Hung | Keqiu Hu | -| [HDFS-13063](https://issues.apache.org/jira/browse/HDFS-13063) | Fix the incorrect spelling in HDFSHighAvailabilityWithQJM.md | Trivial | documentation | Jianfei Jiang | Jianfei Jiang | -| [YARN-7102](https://issues.apache.org/jira/browse/YARN-7102) | NM heartbeat stuck when responseId overflows MAX\_INT | Critical | . | Botong Huang | Botong Huang | -| [MAPREDUCE-7041](https://issues.apache.org/jira/browse/MAPREDUCE-7041) | MR should not try to clean up at first job attempt | Major | . | Takanobu Asanuma | Gergo Repas | -| [MAPREDUCE-7020](https://issues.apache.org/jira/browse/MAPREDUCE-7020) | Task timeout in uber mode can crash AM | Major | mr-am | Akira Ajisaka | Peter Bacsko | -| [YARN-7765](https://issues.apache.org/jira/browse/YARN-7765) | [Atsv2] GSSException: No valid credentials provided - Failed to find any Kerberos tgt thrown by Timelinev2Client & HBaseClient in NM | Blocker | . | Sumana Sathish | Rohith Sharma K S | -| [HDFS-12974](https://issues.apache.org/jira/browse/HDFS-12974) | Exception message is not printed when creating an encryption zone fails with AuthorizationException | Minor | encryption | fang zhenyi | fang zhenyi | -| [YARN-7698](https://issues.apache.org/jira/browse/YARN-7698) | A misleading variable's name in ApplicationAttemptEventDispatcher | Minor | resourcemanager | Jinjiang Ling | Jinjiang Ling | -| [HDFS-12528](https://issues.apache.org/jira/browse/HDFS-12528) | Add an option to not disable short-circuit reads on failures | Major | hdfs-client, performance | Andre Araujo | Xiao Chen | -| [HDFS-13100](https://issues.apache.org/jira/browse/HDFS-13100) | Handle IllegalArgumentException when GETSERVERDEFAULTS is not implemented in webhdfs. | Critical | hdfs, webhdfs | Yongjun Zhang | Yongjun Zhang | -| [YARN-7849](https://issues.apache.org/jira/browse/YARN-7849) | TestMiniYarnClusterNodeUtilization#testUpdateNodeUtilization fails due to heartbeat sync error | Major | test | Jason Darrell Lowe | Botong Huang | -| [YARN-7801](https://issues.apache.org/jira/browse/YARN-7801) | AmFilterInitializer should addFilter after fill all parameters | Critical | . | Sumana Sathish | Wangda Tan | -| [YARN-7890](https://issues.apache.org/jira/browse/YARN-7890) | NPE during container relaunch | Major | . | Billie Rinaldi | Jason Darrell Lowe | -| [HDFS-13115](https://issues.apache.org/jira/browse/HDFS-13115) | In getNumUnderConstructionBlocks(), ignore the inodeIds for which the inodes have been deleted | Major | . | Yongjun Zhang | Yongjun Zhang | -| [HDFS-12935](https://issues.apache.org/jira/browse/HDFS-12935) | Get ambiguous result for DFSAdmin command in HA mode when only one namenode is up | Major | tools | Jianfei Jiang | Jianfei Jiang | -| [HDFS-13120](https://issues.apache.org/jira/browse/HDFS-13120) | Snapshot diff could be corrupted after concat | Major | namenode, snapshots | Xiaoyu Yao | Xiaoyu Yao | -| [HDFS-10453](https://issues.apache.org/jira/browse/HDFS-10453) | ReplicationMonitor thread could stuck for long time due to the race between replication and delete of same file in a large cluster. | Major | namenode | Xiaoqiao He | Xiaoqiao He | -| [HDFS-8693](https://issues.apache.org/jira/browse/HDFS-8693) | refreshNamenodes does not support adding a new standby to a running DN | Critical | datanode, ha | Jian Fang | Ajith S | -| [MAPREDUCE-7052](https://issues.apache.org/jira/browse/MAPREDUCE-7052) | TestFixedLengthInputFormat#testFormatCompressedIn is flaky | Major | client, test | Peter Bacsko | Peter Bacsko | -| [HDFS-13112](https://issues.apache.org/jira/browse/HDFS-13112) | Token expiration edits may cause log corruption or deadlock | Critical | namenode | Daryn Sharp | Daryn Sharp | -| [MAPREDUCE-7053](https://issues.apache.org/jira/browse/MAPREDUCE-7053) | Timed out tasks can fail to produce thread dump | Major | . | Jason Darrell Lowe | Jason Darrell Lowe | -| [HADOOP-15206](https://issues.apache.org/jira/browse/HADOOP-15206) | BZip2 drops and duplicates records when input split size is small | Major | . | Aki Tanaka | Aki Tanaka | -| [YARN-7937](https://issues.apache.org/jira/browse/YARN-7937) | Fix http method name in Cluster Application Timeout Update API example request | Minor | docs, documentation | Charan Hebri | Charan Hebri | -| [YARN-7947](https://issues.apache.org/jira/browse/YARN-7947) | Capacity Scheduler intra-queue preemption can NPE for non-schedulable apps | Major | capacity scheduler, scheduler preemption | Eric Payne | Eric Payne | -| [YARN-7945](https://issues.apache.org/jira/browse/YARN-7945) | Java Doc error in UnmanagedAMPoolManager for branch-2 | Major | . | Rohith Sharma K S | Botong Huang | -| [HADOOP-14903](https://issues.apache.org/jira/browse/HADOOP-14903) | Add json-smart explicitly to pom.xml | Major | common | Ray Chiang | Ray Chiang | -| [HADOOP-15236](https://issues.apache.org/jira/browse/HADOOP-15236) | Fix typo in RequestHedgingProxyProvider and RequestHedgingRMFailoverProxyProvider | Trivial | documentation | Akira Ajisaka | Gabor Bota | -| [MAPREDUCE-7027](https://issues.apache.org/jira/browse/MAPREDUCE-7027) | HadoopArchiveLogs shouldn't delete the original logs if the HAR creation fails | Critical | harchive | Gergely Novák | Gergely Novák | -| [HDFS-12781](https://issues.apache.org/jira/browse/HDFS-12781) | After Datanode down, In Namenode UI Datanode tab is throwing warning message. | Major | datanode | Harshakiran Reddy | Brahma Reddy Battula | -| [HDFS-12070](https://issues.apache.org/jira/browse/HDFS-12070) | Failed block recovery leaves files open indefinitely and at risk for data loss | Major | . | Daryn Sharp | Kihwal Lee | -| [HADOOP-15251](https://issues.apache.org/jira/browse/HADOOP-15251) | Backport HADOOP-13514 (surefire upgrade) to branch-2 | Major | test | Christopher Douglas | Christopher Douglas | -| [HDFS-13194](https://issues.apache.org/jira/browse/HDFS-13194) | CachePool permissions incorrectly checked | Major | . | Yiqun Lin | Jianfei Jiang | -| [HADOOP-15276](https://issues.apache.org/jira/browse/HADOOP-15276) | branch-2 site not building after ADL troubleshooting doc added | Major | documentation | Steve Loughran | Steve Loughran | -| [YARN-7835](https://issues.apache.org/jira/browse/YARN-7835) | [Atsv2] Race condition in NM while publishing events if second attempt is launched on the same node | Critical | . | Rohith Sharma K S | Rohith Sharma K S | -| [HADOOP-15275](https://issues.apache.org/jira/browse/HADOOP-15275) | Incorrect javadoc for return type of RetryPolicy#shouldRetry | Minor | documentation | Nanda kumar | Nanda kumar | -| [YARN-7511](https://issues.apache.org/jira/browse/YARN-7511) | NPE in ContainerLocalizer when localization failed for running container | Major | nodemanager | Tao Yang | Tao Yang | -| [MAPREDUCE-7023](https://issues.apache.org/jira/browse/MAPREDUCE-7023) | TestHadoopArchiveLogs.testCheckFilesAndSeedApps fails on rerun | Minor | test | Gergely Novák | Gergely Novák | -| [HADOOP-15283](https://issues.apache.org/jira/browse/HADOOP-15283) | Upgrade from findbugs 3.0.1 to spotbugs 3.1.2 in branch-2 to fix docker image build | Major | . | Xiao Chen | Akira Ajisaka | -| [HADOOP-15286](https://issues.apache.org/jira/browse/HADOOP-15286) | Remove unused imports from TestKMSWithZK.java | Minor | test | Akira Ajisaka | Ajay Kumar | -| [HDFS-13040](https://issues.apache.org/jira/browse/HDFS-13040) | Kerberized inotify client fails despite kinit properly | Major | namenode | Wei-Chiu Chuang | Xiao Chen | -| [YARN-7736](https://issues.apache.org/jira/browse/YARN-7736) | Fix itemization in YARN federation document | Minor | documentation | Akira Ajisaka | Sen Zhao | -| [HDFS-13164](https://issues.apache.org/jira/browse/HDFS-13164) | File not closed if streamer fail with DSQuotaExceededException | Major | hdfs-client | Xiao Chen | Xiao Chen | -| [HDFS-13109](https://issues.apache.org/jira/browse/HDFS-13109) | Support fully qualified hdfs path in EZ commands | Major | hdfs | Hanisha Koneru | Hanisha Koneru | -| [MAPREDUCE-6930](https://issues.apache.org/jira/browse/MAPREDUCE-6930) | mapreduce.map.cpu.vcores and mapreduce.reduce.cpu.vcores are both present twice in mapred-default.xml | Major | mrv2 | Daniel Templeton | Sen Zhao | -| [HDFS-10618](https://issues.apache.org/jira/browse/HDFS-10618) | TestPendingReconstruction#testPendingAndInvalidate is flaky due to race condition | Major | . | Eric Badger | Eric Badger | -| [HDFS-10803](https://issues.apache.org/jira/browse/HDFS-10803) | TestBalancerWithMultipleNameNodes#testBalancing2OutOf3Blockpools fails intermittently due to no free space available | Major | . | Yiqun Lin | Yiqun Lin | -| [HDFS-12156](https://issues.apache.org/jira/browse/HDFS-12156) | TestFSImage fails without -Pnative | Major | test | Akira Ajisaka | Akira Ajisaka | -| [HDFS-13261](https://issues.apache.org/jira/browse/HDFS-13261) | Fix incorrect null value check | Minor | hdfs | Jianfei Jiang | Jianfei Jiang | -| [HDFS-12886](https://issues.apache.org/jira/browse/HDFS-12886) | Ignore minReplication for block recovery | Major | hdfs, namenode | Lukas Majercak | Lukas Majercak | -| [YARN-8039](https://issues.apache.org/jira/browse/YARN-8039) | Clean up log dir configuration in TestLinuxContainerExecutorWithMocks.testStartLocalizer | Minor | . | Miklos Szegedi | Miklos Szegedi | -| [HDFS-13296](https://issues.apache.org/jira/browse/HDFS-13296) | GenericTestUtils generates paths with drive letter in Windows and fail webhdfs related test cases | Major | . | Xiao Liang | Xiao Liang | -| [HDFS-13268](https://issues.apache.org/jira/browse/HDFS-13268) | TestWebHdfsFileContextMainOperations fails on Windows | Major | . | Íñigo Goiri | Xiao Liang | -| [YARN-8054](https://issues.apache.org/jira/browse/YARN-8054) | Improve robustness of the LocalDirsHandlerService MonitoringTimerTask thread | Major | . | Jonathan Turner Eagles | Jonathan Turner Eagles | -| [YARN-7873](https://issues.apache.org/jira/browse/YARN-7873) | Revert YARN-6078 | Blocker | . | Billie Rinaldi | Billie Rinaldi | -| [HDFS-13195](https://issues.apache.org/jira/browse/HDFS-13195) | DataNode conf page cannot display the current value after reconfig | Minor | datanode | maobaolong | maobaolong | -| [YARN-8063](https://issues.apache.org/jira/browse/YARN-8063) | DistributedShellTimelinePlugin wrongly check for entityId instead of entityType | Major | . | Rohith Sharma K S | Rohith Sharma K S | -| [YARN-8068](https://issues.apache.org/jira/browse/YARN-8068) | Application Priority field causes NPE in app timeline publish when Hadoop 2.7 based clients to 2.8+ | Blocker | yarn | Sunil G | Sunil G | -| [HADOOP-12862](https://issues.apache.org/jira/browse/HADOOP-12862) | LDAP Group Mapping over SSL can not specify trust store | Major | . | Wei-Chiu Chuang | Wei-Chiu Chuang | -| [HADOOP-15317](https://issues.apache.org/jira/browse/HADOOP-15317) | Improve NetworkTopology chooseRandom's loop | Major | . | Xiao Chen | Xiao Chen | -| [HADOOP-15355](https://issues.apache.org/jira/browse/HADOOP-15355) | TestCommonConfigurationFields is broken by HADOOP-15312 | Major | test | Konstantin Shvachko | LiXin Ge | -| [HDFS-13176](https://issues.apache.org/jira/browse/HDFS-13176) | WebHdfs file path gets truncated when having semicolon (;) inside | Major | webhdfs | Zsolt Venczel | Zsolt Venczel | -| [HADOOP-15375](https://issues.apache.org/jira/browse/HADOOP-15375) | Branch-2 pre-commit failed to build docker image | Major | . | Xiao Chen | Xiao Chen | -| [HADOOP-15357](https://issues.apache.org/jira/browse/HADOOP-15357) | Configuration.getPropsWithPrefix no longer does variable substitution | Major | . | Jim Brennan | Jim Brennan | -| [MAPREDUCE-7062](https://issues.apache.org/jira/browse/MAPREDUCE-7062) | Update mapreduce.job.tags description for making use for ATSv2 purpose. | Major | . | Charan Hebri | Charan Hebri | -| [YARN-8073](https://issues.apache.org/jira/browse/YARN-8073) | TimelineClientImpl doesn't honor yarn.timeline-service.versions configuration | Major | . | Rohith Sharma K S | Rohith Sharma K S | -| [YARN-6629](https://issues.apache.org/jira/browse/YARN-6629) | NPE occurred when container allocation proposal is applied but its resource requests are removed before | Critical | . | Tao Yang | Tao Yang | -| [HDFS-13427](https://issues.apache.org/jira/browse/HDFS-13427) | Fix the section titles of transparent encryption document | Minor | documentation | Akira Ajisaka | Akira Ajisaka | -| [YARN-7527](https://issues.apache.org/jira/browse/YARN-7527) | Over-allocate node resource in async-scheduling mode of CapacityScheduler | Major | capacityscheduler | Tao Yang | Tao Yang | -| [HDFS-7101](https://issues.apache.org/jira/browse/HDFS-7101) | Potential null dereference in DFSck#doWork() | Minor | . | Ted Yu | skrho | -| [YARN-8120](https://issues.apache.org/jira/browse/YARN-8120) | JVM can crash with SIGSEGV when exiting due to custom leveldb logger | Major | nodemanager, resourcemanager | Jason Darrell Lowe | Jason Darrell Lowe | -| [YARN-8147](https://issues.apache.org/jira/browse/YARN-8147) | TestClientRMService#testGetApplications sporadically fails | Major | test | Jason Darrell Lowe | Jason Darrell Lowe | -| [HADOOP-14970](https://issues.apache.org/jira/browse/HADOOP-14970) | MiniHadoopClusterManager doesn't respect lack of format option | Minor | . | Erik Krogen | Erik Krogen | -| [YARN-8156](https://issues.apache.org/jira/browse/YARN-8156) | Increase the default value of yarn.timeline-service.app-collector.linger-period.ms | Major | . | Rohith Sharma K S | Charan Hebri | -| [YARN-8165](https://issues.apache.org/jira/browse/YARN-8165) | Incorrect queue name logging in AbstractContainerAllocator | Trivial | capacityscheduler | Weiwei Yang | Weiwei Yang | -| [YARN-8164](https://issues.apache.org/jira/browse/YARN-8164) | Fix a potential NPE in AbstractSchedulerPlanFollower | Major | . | lujie | lujie | -| [HDFS-12828](https://issues.apache.org/jira/browse/HDFS-12828) | OIV ReverseXML Processor fails with escaped characters | Critical | hdfs | Erik Krogen | Erik Krogen | -| [HADOOP-15180](https://issues.apache.org/jira/browse/HADOOP-15180) | branch-2 : daemon processes' sysout overwrites 'ulimit -a' in daemon's out file | Minor | scripts | Ranith Sardar | Ranith Sardar | -| [HADOOP-15396](https://issues.apache.org/jira/browse/HADOOP-15396) | Some java source files are executable | Minor | . | Akira Ajisaka | Shashikant Banerjee | -| [YARN-6827](https://issues.apache.org/jira/browse/YARN-6827) | [ATS1/1.5] NPE exception while publishing recovering applications into ATS during RM restart. | Major | resourcemanager | Rohith Sharma K S | Rohith Sharma K S | -| [YARN-7786](https://issues.apache.org/jira/browse/YARN-7786) | NullPointerException while launching ApplicationMaster | Major | . | lujie | lujie | -| [HDFS-13408](https://issues.apache.org/jira/browse/HDFS-13408) | MiniDFSCluster to support being built on randomized base directory | Major | test | Xiao Liang | Xiao Liang | -| [HADOOP-15390](https://issues.apache.org/jira/browse/HADOOP-15390) | Yarn RM logs flooded by DelegationTokenRenewer trying to renew KMS tokens | Critical | . | Xiao Chen | Xiao Chen | -| [HDFS-13336](https://issues.apache.org/jira/browse/HDFS-13336) | Test cases of TestWriteToReplica failed in windows | Major | . | Xiao Liang | Xiao Liang | -| [YARN-7598](https://issues.apache.org/jira/browse/YARN-7598) | Document how to use classpath isolation for aux-services in YARN | Major | . | Xuan Gong | Xuan Gong | -| [YARN-8183](https://issues.apache.org/jira/browse/YARN-8183) | Fix ConcurrentModificationException inside RMAppAttemptMetrics#convertAtomicLongMaptoLongMap | Critical | yarn | Sumana Sathish | Suma Shivaprasad | -| [HADOOP-15385](https://issues.apache.org/jira/browse/HADOOP-15385) | Many tests are failing in hadoop-distcp project in branch-2 | Critical | tools/distcp | Rushabh Shah | Jason Darrell Lowe | -| [MAPREDUCE-7042](https://issues.apache.org/jira/browse/MAPREDUCE-7042) | Killed MR job data does not move to mapreduce.jobhistory.done-dir when ATS v2 is enabled | Major | . | Yesha Vora | Xuan Gong | -| [YARN-8205](https://issues.apache.org/jira/browse/YARN-8205) | Application State is not updated to ATS if AM launching is delayed. | Critical | . | Sumana Sathish | Rohith Sharma K S | -| [MAPREDUCE-7072](https://issues.apache.org/jira/browse/MAPREDUCE-7072) | mapred job -history prints duplicate counter in human output | Major | client | Wilfred Spiegelenburg | Wilfred Spiegelenburg | -| [YARN-8221](https://issues.apache.org/jira/browse/YARN-8221) | RMWebServices also need to honor yarn.resourcemanager.display.per-user-apps | Major | webapp | Sunil G | Sunil G | -| [HDFS-13509](https://issues.apache.org/jira/browse/HDFS-13509) | Bug fix for breakHardlinks() of ReplicaInfo/LocalReplica, and fix TestFileAppend failures on Windows | Major | . | Xiao Liang | Xiao Liang | -| [MAPREDUCE-7073](https://issues.apache.org/jira/browse/MAPREDUCE-7073) | Optimize TokenCache#obtainTokensForNamenodesInternal | Major | . | Bibin Chundatt | Bibin Chundatt | -| [YARN-8025](https://issues.apache.org/jira/browse/YARN-8025) | UsersManangers#getComputedResourceLimitForActiveUsers throws NPE due to preComputedActiveUserLimit is empty | Major | yarn | Jiandan Yang | Tao Yang | -| [YARN-8232](https://issues.apache.org/jira/browse/YARN-8232) | RMContainer lost queue name when RM HA happens | Major | resourcemanager | Hu Ziqian | Hu Ziqian | -| [HDFS-13537](https://issues.apache.org/jira/browse/HDFS-13537) | TestHdfsHelper does not generate jceks path properly for relative path in Windows | Major | . | Xiao Liang | Xiao Liang | -| [HADOOP-15446](https://issues.apache.org/jira/browse/HADOOP-15446) | WASB: PageBlobInputStream.skip breaks HBASE replication | Major | fs/azure | Thomas Marqardt | Thomas Marqardt | -| [YARN-8244](https://issues.apache.org/jira/browse/YARN-8244) | TestContainerSchedulerQueuing.testStartMultipleContainers failed | Major | . | Miklos Szegedi | Jim Brennan | -| [HDFS-13586](https://issues.apache.org/jira/browse/HDFS-13586) | Fsync fails on directories on Windows | Critical | datanode, hdfs | Lukas Majercak | Lukas Majercak | -| [HDFS-13590](https://issues.apache.org/jira/browse/HDFS-13590) | Backport HDFS-12378 to branch-2 | Major | datanode, hdfs, test | Lukas Majercak | Lukas Majercak | -| [HADOOP-15478](https://issues.apache.org/jira/browse/HADOOP-15478) | WASB: hflush() and hsync() regression | Major | fs/azure | Thomas Marqardt | Thomas Marqardt | -| [HADOOP-15450](https://issues.apache.org/jira/browse/HADOOP-15450) | Avoid fsync storm triggered by DiskChecker and handle disk full situation | Blocker | . | Kihwal Lee | Arpit Agarwal | -| [HDFS-13601](https://issues.apache.org/jira/browse/HDFS-13601) | Optimize ByteString conversions in PBHelper | Major | . | Andrew Wang | Andrew Wang | -| [HDFS-13588](https://issues.apache.org/jira/browse/HDFS-13588) | Fix TestFsDatasetImpl test failures on Windows | Major | . | Xiao Liang | Xiao Liang | -| [YARN-8310](https://issues.apache.org/jira/browse/YARN-8310) | Handle old NMTokenIdentifier, AMRMTokenIdentifier, and ContainerTokenIdentifier formats | Major | . | Robert Kanter | Robert Kanter | -| [YARN-8344](https://issues.apache.org/jira/browse/YARN-8344) | Missing nm.stop() in TestNodeManagerResync to fix testKillContainersOnResync | Major | . | Giovanni Matteo Fumarola | Giovanni Matteo Fumarola | -| [YARN-8327](https://issues.apache.org/jira/browse/YARN-8327) | Fix TestAggregatedLogFormat#testReadAcontainerLogs1 on Windows | Major | log-aggregation | Giovanni Matteo Fumarola | Giovanni Matteo Fumarola | -| [YARN-8346](https://issues.apache.org/jira/browse/YARN-8346) | Upgrading to 3.1 kills running containers with error "Opportunistic container queue is full" | Blocker | . | Rohith Sharma K S | Jason Darrell Lowe | -| [HDFS-13611](https://issues.apache.org/jira/browse/HDFS-13611) | Unsafe use of Text as a ConcurrentHashMap key in PBHelperClient | Major | . | Andrew Wang | Andrew Wang | -| [HDFS-13618](https://issues.apache.org/jira/browse/HDFS-13618) | Fix TestDataNodeFaultInjector test failures on Windows | Major | test | Xiao Liang | Xiao Liang | -| [HADOOP-15473](https://issues.apache.org/jira/browse/HADOOP-15473) | Configure serialFilter in KeyProvider to avoid UnrecoverableKeyException caused by JDK-8189997 | Critical | kms | Gabor Bota | Gabor Bota | -| [HDFS-13626](https://issues.apache.org/jira/browse/HDFS-13626) | Fix incorrect username when deny the setOwner operation | Minor | namenode | luhuachao | Zsolt Venczel | -| [MAPREDUCE-7103](https://issues.apache.org/jira/browse/MAPREDUCE-7103) | Fix TestHistoryViewerPrinter on windows due to a mismatch line separator | Minor | . | Giovanni Matteo Fumarola | Giovanni Matteo Fumarola | -| [YARN-8359](https://issues.apache.org/jira/browse/YARN-8359) | Exclude containermanager.linux test classes on Windows | Major | . | Giovanni Matteo Fumarola | Jason Darrell Lowe | -| [HDFS-13664](https://issues.apache.org/jira/browse/HDFS-13664) | Refactor ConfiguredFailoverProxyProvider to make inheritance easier | Minor | hdfs-client | Chao Sun | Chao Sun | -| [YARN-8405](https://issues.apache.org/jira/browse/YARN-8405) | RM zk-state-store.parent-path ACLs has been changed since HADOOP-14773 | Major | . | Rohith Sharma K S | Íñigo Goiri | -| [MAPREDUCE-7108](https://issues.apache.org/jira/browse/MAPREDUCE-7108) | TestFileOutputCommitter fails on Windows | Minor | test | Zuoming Zhang | Zuoming Zhang | -| [MAPREDUCE-7101](https://issues.apache.org/jira/browse/MAPREDUCE-7101) | Add config parameter to allow JHS to alway scan user dir irrespective of modTime | Critical | . | Wangda Tan | Thomas Marqardt | -| [YARN-8404](https://issues.apache.org/jira/browse/YARN-8404) | Timeline event publish need to be async to avoid Dispatcher thread leak in case ATS is down | Blocker | . | Rohith Sharma K S | Rohith Sharma K S | -| [HDFS-13675](https://issues.apache.org/jira/browse/HDFS-13675) | Speed up TestDFSAdminWithHA | Major | hdfs, namenode | Lukas Majercak | Lukas Majercak | -| [HDFS-13673](https://issues.apache.org/jira/browse/HDFS-13673) | TestNameNodeMetrics fails on Windows | Minor | test | Zuoming Zhang | Zuoming Zhang | -| [HDFS-13676](https://issues.apache.org/jira/browse/HDFS-13676) | TestEditLogRace fails on Windows | Minor | test | Zuoming Zhang | Zuoming Zhang | -| [HADOOP-15523](https://issues.apache.org/jira/browse/HADOOP-15523) | Shell command timeout given is in seconds whereas it is taken as millisec while scheduling | Major | . | Bilwa S T | Bilwa S T | -| [YARN-8444](https://issues.apache.org/jira/browse/YARN-8444) | NodeResourceMonitor crashes on bad swapFree value | Major | . | Jim Brennan | Jim Brennan | -| [YARN-8457](https://issues.apache.org/jira/browse/YARN-8457) | Compilation is broken with -Pyarn-ui | Major | webapp | Sunil G | Sunil G | -| [YARN-8401](https://issues.apache.org/jira/browse/YARN-8401) | [UI2] new ui is not accessible with out internet connection | Blocker | . | Bibin Chundatt | Bibin Chundatt | -| [YARN-8451](https://issues.apache.org/jira/browse/YARN-8451) | Multiple NM heartbeat thread created when a slow NM resync with RM | Major | nodemanager | Botong Huang | Botong Huang | -| [HADOOP-15548](https://issues.apache.org/jira/browse/HADOOP-15548) | Randomize local dirs | Minor | . | Jim Brennan | Jim Brennan | -| [YARN-8473](https://issues.apache.org/jira/browse/YARN-8473) | Containers being launched as app tears down can leave containers in NEW state | Major | nodemanager | Jason Darrell Lowe | Jason Darrell Lowe | -| [HDFS-13729](https://issues.apache.org/jira/browse/HDFS-13729) | Fix broken links to RBF documentation | Minor | documentation | jwhitter | Gabor Bota | -| [YARN-8515](https://issues.apache.org/jira/browse/YARN-8515) | container-executor can crash with SIGPIPE after nodemanager restart | Major | . | Jim Brennan | Jim Brennan | -| [YARN-8421](https://issues.apache.org/jira/browse/YARN-8421) | when moving app, activeUsers is increased, even though app does not have outstanding request | Major | . | kyungwan nam | | -| [HADOOP-15614](https://issues.apache.org/jira/browse/HADOOP-15614) | TestGroupsCaching.testExceptionOnBackgroundRefreshHandled reliably fails | Major | . | Kihwal Lee | Weiwei Yang | -| [YARN-4606](https://issues.apache.org/jira/browse/YARN-4606) | CapacityScheduler: applications could get starved because computation of #activeUsers considers pending apps | Critical | capacity scheduler, capacityscheduler | Karam Singh | Manikandan R | -| [HADOOP-15637](https://issues.apache.org/jira/browse/HADOOP-15637) | LocalFs#listLocatedStatus does not filter out hidden .crc files | Minor | fs | Erik Krogen | Erik Krogen | -| [HADOOP-15644](https://issues.apache.org/jira/browse/HADOOP-15644) | Hadoop Docker Image Pip Install Fails on branch-2 | Critical | build | Haibo Chen | Haibo Chen | -| [YARN-6966](https://issues.apache.org/jira/browse/YARN-6966) | NodeManager metrics may return wrong negative values when NM restart | Major | . | Yang Wang | Szilard Nemeth | -| [YARN-8331](https://issues.apache.org/jira/browse/YARN-8331) | Race condition in NM container launched after done | Major | . | Yang Wang | Pradeep Ambati | -| [HDFS-13758](https://issues.apache.org/jira/browse/HDFS-13758) | DatanodeManager should throw exception if it has BlockRecoveryCommand but the block is not under construction | Major | namenode | Wei-Chiu Chuang | chencan | -| [YARN-8612](https://issues.apache.org/jira/browse/YARN-8612) | Fix NM Collector Service Port issue in YarnConfiguration | Major | ATSv2 | Prabha Manepalli | Prabha Manepalli | -| [HADOOP-15674](https://issues.apache.org/jira/browse/HADOOP-15674) | Test failure TestSSLHttpServer.testExcludedCiphers with TLS\_ECDHE\_RSA\_WITH\_AES\_128\_CBC\_SHA256 cipher suite | Major | common | Gabor Bota | Szilard Nemeth | -| [YARN-8640](https://issues.apache.org/jira/browse/YARN-8640) | Restore previous state in container-executor after failure | Major | . | Jim Brennan | Jim Brennan | -| [YARN-8679](https://issues.apache.org/jira/browse/YARN-8679) | [ATSv2] If HBase cluster is down for long time, high chances that NM ContainerManager dispatcher get blocked | Major | . | Rohith Sharma K S | Wangda Tan | -| [HADOOP-14314](https://issues.apache.org/jira/browse/HADOOP-14314) | The OpenSolaris taxonomy link is dead in InterfaceClassification.md | Major | documentation | Daniel Templeton | Rui Gao | -| [YARN-8649](https://issues.apache.org/jira/browse/YARN-8649) | NPE in localizer hearbeat processing if a container is killed while localizing | Major | . | lujie | lujie | -| [HADOOP-10219](https://issues.apache.org/jira/browse/HADOOP-10219) | ipc.Client.setupIOstreams() needs to check for ClientCache.stopClient requested shutdowns | Major | ipc | Steve Loughran | Kihwal Lee | -| [MAPREDUCE-7131](https://issues.apache.org/jira/browse/MAPREDUCE-7131) | Job History Server has race condition where it moves files from intermediate to finished but thinks file is in intermediate | Major | . | Anthony Hsu | Anthony Hsu | -| [HDFS-13836](https://issues.apache.org/jira/browse/HDFS-13836) | RBF: Handle mount table znode with null value | Major | federation, hdfs | yanghuafeng | yanghuafeng | -| [HDFS-12716](https://issues.apache.org/jira/browse/HDFS-12716) | 'dfs.datanode.failed.volumes.tolerated' to support minimum number of volumes to be available | Major | datanode | usharani | Ranith Sardar | -| [YARN-8709](https://issues.apache.org/jira/browse/YARN-8709) | CS preemption monitor always fails since one under-served queue was deleted | Major | capacityscheduler, scheduler preemption | Tao Yang | Tao Yang | -| [HDFS-13051](https://issues.apache.org/jira/browse/HDFS-13051) | Fix dead lock during async editlog rolling if edit queue is full | Major | namenode | zhangwei | Daryn Sharp | -| [HDFS-13914](https://issues.apache.org/jira/browse/HDFS-13914) | Fix DN UI logs link broken when https is enabled after HDFS-13902 | Minor | datanode | Jianfei Jiang | Jianfei Jiang | -| [MAPREDUCE-7133](https://issues.apache.org/jira/browse/MAPREDUCE-7133) | History Server task attempts REST API returns invalid data | Major | jobhistoryserver | Oleksandr Shevchenko | Oleksandr Shevchenko | -| [YARN-8720](https://issues.apache.org/jira/browse/YARN-8720) | CapacityScheduler does not enforce max resource allocation check at queue level | Major | capacity scheduler, capacityscheduler, resourcemanager | Tarun Parimi | Tarun Parimi | -| [HDFS-13844](https://issues.apache.org/jira/browse/HDFS-13844) | Fix the fmt\_bytes function in the dfs-dust.js | Minor | hdfs, ui | yanghuafeng | yanghuafeng | -| [HADOOP-15755](https://issues.apache.org/jira/browse/HADOOP-15755) | StringUtils#createStartupShutdownMessage throws NPE when args is null | Major | . | Lokesh Jain | Dinesh Chitlangia | -| [MAPREDUCE-3801](https://issues.apache.org/jira/browse/MAPREDUCE-3801) | org.apache.hadoop.mapreduce.v2.app.TestRuntimeEstimators.testExponentialEstimator fails intermittently | Major | mrv2 | Robert Joseph Evans | Jason Darrell Lowe | -| [MAPREDUCE-7137](https://issues.apache.org/jira/browse/MAPREDUCE-7137) | MRAppBenchmark.benchmark1() fails with NullPointerException | Minor | test | Oleksandr Shevchenko | Oleksandr Shevchenko | -| [MAPREDUCE-7138](https://issues.apache.org/jira/browse/MAPREDUCE-7138) | ThrottledContainerAllocator in MRAppBenchmark should implement RMHeartbeatHandler | Minor | test | Oleksandr Shevchenko | Oleksandr Shevchenko | -| [HDFS-13908](https://issues.apache.org/jira/browse/HDFS-13908) | TestDataNodeMultipleRegistrations is flaky | Major | . | Íñigo Goiri | Ayush Saxena | -| [HADOOP-15772](https://issues.apache.org/jira/browse/HADOOP-15772) | Remove the 'Path ... should be specified as a URI' warnings on startup | Major | conf | Arpit Agarwal | Ayush Saxena | -| [YARN-8804](https://issues.apache.org/jira/browse/YARN-8804) | resourceLimits may be wrongly calculated when leaf-queue is blocked in cluster with 3+ level queues | Critical | capacityscheduler | Tao Yang | Tao Yang | -| [YARN-8774](https://issues.apache.org/jira/browse/YARN-8774) | Memory leak when CapacityScheduler allocates from reserved container with non-default label | Critical | capacityscheduler | Tao Yang | Tao Yang | -| [YARN-8840](https://issues.apache.org/jira/browse/YARN-8840) | Add missing cleanupSSLConfig() call for TestTimelineClient test | Minor | test, timelineclient | Aki Tanaka | Aki Tanaka | -| [HADOOP-15817](https://issues.apache.org/jira/browse/HADOOP-15817) | Reuse Object Mapper in KMSJSONReader | Major | kms | Jonathan Turner Eagles | Jonathan Turner Eagles | -| [HADOOP-15820](https://issues.apache.org/jira/browse/HADOOP-15820) | ZStandardDecompressor native code sets an integer field as a long | Blocker | . | Jason Darrell Lowe | Jason Darrell Lowe | -| [HDFS-13964](https://issues.apache.org/jira/browse/HDFS-13964) | RBF: TestRouterWebHDFSContractAppend fails with No Active Namenode under nameservice | Major | . | Ayush Saxena | Ayush Saxena | -| [HDFS-13768](https://issues.apache.org/jira/browse/HDFS-13768) | Adding replicas to volume map makes DataNode start slowly | Major | . | Yiqun Lin | Surendra Singh Lilhore | -| [HADOOP-15818](https://issues.apache.org/jira/browse/HADOOP-15818) | Fix deprecated maven-surefire-plugin configuration in hadoop-kms module | Minor | kms | Akira Ajisaka | Vidura Bhathiya Mudalige | -| [HDFS-13802](https://issues.apache.org/jira/browse/HDFS-13802) | RBF: Remove FSCK from Router Web UI | Major | . | Fei Hui | Fei Hui | -| [HADOOP-15859](https://issues.apache.org/jira/browse/HADOOP-15859) | ZStandardDecompressor.c mistakes a class for an instance | Blocker | . | Ben Lau | Jason Darrell Lowe | -| [HADOOP-15850](https://issues.apache.org/jira/browse/HADOOP-15850) | CopyCommitter#concatFileChunks should check that the blocks per chunk is not 0 | Critical | tools/distcp | Ted Yu | Ted Yu | -| [YARN-7502](https://issues.apache.org/jira/browse/YARN-7502) | Nodemanager restart docs should describe nodemanager supervised property | Major | documentation | Jason Darrell Lowe | Suma Shivaprasad | -| [HADOOP-15866](https://issues.apache.org/jira/browse/HADOOP-15866) | Renamed HADOOP\_SECURITY\_GROUP\_SHELL\_COMMAND\_TIMEOUT keys break compatibility | Blocker | . | Wei-Chiu Chuang | Wei-Chiu Chuang | -| [YARN-8826](https://issues.apache.org/jira/browse/YARN-8826) | Fix lingering timeline collector after serviceStop in TimelineCollectorManager | Trivial | ATSv2 | Prabha Manepalli | Prabha Manepalli | -| [HADOOP-15822](https://issues.apache.org/jira/browse/HADOOP-15822) | zstd compressor can fail with a small output buffer | Major | . | Jason Darrell Lowe | Jason Darrell Lowe | -| [HDFS-13959](https://issues.apache.org/jira/browse/HDFS-13959) | TestUpgradeDomainBlockPlacementPolicy is flaky | Major | . | Ayush Saxena | Ayush Saxena | -| [HADOOP-15899](https://issues.apache.org/jira/browse/HADOOP-15899) | Update AWS Java SDK versions in NOTICE.txt | Major | . | Akira Ajisaka | Akira Ajisaka | -| [HADOOP-15900](https://issues.apache.org/jira/browse/HADOOP-15900) | Update JSch versions in LICENSE.txt | Major | . | Akira Ajisaka | Akira Ajisaka | -| [HDFS-14043](https://issues.apache.org/jira/browse/HDFS-14043) | Tolerate corrupted seen\_txid file | Major | hdfs, namenode | Lukas Majercak | Lukas Majercak | -| [YARN-8858](https://issues.apache.org/jira/browse/YARN-8858) | CapacityScheduler should respect maximum node resource when per-queue maximum-allocation is being used. | Major | . | Sumana Sathish | Wangda Tan | -| [HDFS-14048](https://issues.apache.org/jira/browse/HDFS-14048) | DFSOutputStream close() throws exception on subsequent call after DataNode restart | Major | hdfs-client | Erik Krogen | Erik Krogen | -| [MAPREDUCE-7156](https://issues.apache.org/jira/browse/MAPREDUCE-7156) | NullPointerException when reaching max shuffle connections | Major | mrv2 | Peter Bacsko | Peter Bacsko | -| [YARN-8233](https://issues.apache.org/jira/browse/YARN-8233) | NPE in CapacityScheduler#tryCommit when handling allocate/reserve proposal whose allocatedOrReservedContainer is null | Critical | capacityscheduler | Tao Yang | Tao Yang | -| [HADOOP-15923](https://issues.apache.org/jira/browse/HADOOP-15923) | create-release script should set max-cache-ttl as well as default-cache-ttl for gpg-agent | Blocker | build | Akira Ajisaka | Akira Ajisaka | -| [HADOOP-15930](https://issues.apache.org/jira/browse/HADOOP-15930) | Exclude MD5 checksum files from release artifact | Critical | build | Akira Ajisaka | Akira Ajisaka | -| [HADOOP-15925](https://issues.apache.org/jira/browse/HADOOP-15925) | The config and log of gpg-agent are removed in create-release script | Major | build | Akira Ajisaka | Dinesh Chitlangia | -| [HDFS-14056](https://issues.apache.org/jira/browse/HDFS-14056) | Fix error messages in HDFS-12716 | Minor | hdfs | Adam Antal | Ayush Saxena | -| [YARN-7794](https://issues.apache.org/jira/browse/YARN-7794) | SLSRunner is not loading timeline service jars causing failure | Blocker | scheduler-load-simulator | Sunil G | Yufei Gu | -| [HADOOP-16008](https://issues.apache.org/jira/browse/HADOOP-16008) | Fix typo in CommandsManual.md | Minor | . | Akira Ajisaka | Shweta | -| [HADOOP-15973](https://issues.apache.org/jira/browse/HADOOP-15973) | Configuration: Included properties are not cached if resource is a stream | Critical | . | Eric Payne | Eric Payne | -| [YARN-9175](https://issues.apache.org/jira/browse/YARN-9175) | Null resources check in ResourceInfo for branch-3.0 | Major | . | Jonathan Hung | Jonathan Hung | -| [HADOOP-15992](https://issues.apache.org/jira/browse/HADOOP-15992) | JSON License is included in the transitive dependency of aliyun-sdk-oss 3.0.0 | Blocker | . | Akira Ajisaka | Akira Ajisaka | -| [HADOOP-16030](https://issues.apache.org/jira/browse/HADOOP-16030) | AliyunOSS: bring fixes back from HADOOP-15671 | Blocker | fs/oss | wujinhu | wujinhu | -| [YARN-9162](https://issues.apache.org/jira/browse/YARN-9162) | Fix TestRMAdminCLI#testHelp | Major | resourcemanager, test | Ayush Saxena | Ayush Saxena | -| [YARN-8833](https://issues.apache.org/jira/browse/YARN-8833) | Avoid potential integer overflow when computing fair shares | Major | fairscheduler | liyakun | liyakun | -| [HADOOP-16016](https://issues.apache.org/jira/browse/HADOOP-16016) | TestSSLFactory#testServerWeakCiphers sporadically fails in precommit builds | Major | security, test | Jason Darrell Lowe | Akira Ajisaka | -| [HADOOP-16013](https://issues.apache.org/jira/browse/HADOOP-16013) | DecayRpcScheduler decay thread should run as a daemon | Major | ipc | Erik Krogen | Erik Krogen | -| [YARN-8747](https://issues.apache.org/jira/browse/YARN-8747) | [UI2] YARN UI2 page loading failed due to js error under some time zone configuration | Critical | webapp | collinma | collinma | -| [YARN-9210](https://issues.apache.org/jira/browse/YARN-9210) | RM nodes web page can not display node info | Blocker | yarn | Jiandan Yang | Jiandan Yang | -| [YARN-7088](https://issues.apache.org/jira/browse/YARN-7088) | Add application launch time to Resource Manager REST API | Major | . | Abdullah Yousufi | Kanwaljeet Sachdev | -| [YARN-9222](https://issues.apache.org/jira/browse/YARN-9222) | Print launchTime in ApplicationSummary | Major | . | Jonathan Hung | Jonathan Hung | -| [YARN-6616](https://issues.apache.org/jira/browse/YARN-6616) | YARN AHS shows submitTime for jobs same as startTime | Minor | . | Prabhu Joseph | Prabhu Joseph | -| [MAPREDUCE-7177](https://issues.apache.org/jira/browse/MAPREDUCE-7177) | Disable speculative execution in TestDFSIO | Major | . | Kihwal Lee | Zhaohui Xin | -| [YARN-9206](https://issues.apache.org/jira/browse/YARN-9206) | RMServerUtils does not count SHUTDOWN as an accepted state | Major | . | Kuhu Shukla | Kuhu Shukla | -| [YARN-9283](https://issues.apache.org/jira/browse/YARN-9283) | Javadoc of LinuxContainerExecutor#addSchedPriorityCommand has a wrong property name as reference | Minor | documentation | Szilard Nemeth | Adam Antal | -| [HADOOP-15813](https://issues.apache.org/jira/browse/HADOOP-15813) | Enable more reliable SSL connection reuse | Major | common | Daryn Sharp | Daryn Sharp | -| [HDFS-14314](https://issues.apache.org/jira/browse/HDFS-14314) | fullBlockReportLeaseId should be reset after registering to NN | Critical | datanode | star | star | -| [YARN-5714](https://issues.apache.org/jira/browse/YARN-5714) | ContainerExecutor does not order environment map | Trivial | nodemanager | Remi Catherinot | Remi Catherinot | -| [HADOOP-16192](https://issues.apache.org/jira/browse/HADOOP-16192) | CallQueue backoff bug fixes: doesn't perform backoff when add() is used, and doesn't update backoff when refreshed | Major | ipc | Erik Krogen | Erik Krogen | -| [HDFS-14399](https://issues.apache.org/jira/browse/HDFS-14399) | Backport HDFS-10536 to branch-2 | Critical | . | Chao Sun | Chao Sun | -| [HDFS-14407](https://issues.apache.org/jira/browse/HDFS-14407) | Fix misuse of SLF4j logging API in DatasetVolumeChecker#checkAllVolumes | Minor | . | Wanqiang Ji | Wanqiang Ji | -| [HDFS-14414](https://issues.apache.org/jira/browse/HDFS-14414) | Clean up findbugs warning in branch-2 | Major | . | Wei-Chiu Chuang | Dinesh Chitlangia | -| [HADOOP-14544](https://issues.apache.org/jira/browse/HADOOP-14544) | DistCp documentation for command line options is misaligned. | Minor | documentation | Chris Nauroth | Masatake Iwasaki | -| [HDFS-10477](https://issues.apache.org/jira/browse/HDFS-10477) | Stop decommission a rack of DataNodes caused NameNode fail over to standby | Major | namenode | yunjiong zhao | yunjiong zhao | -| [HDFS-13677](https://issues.apache.org/jira/browse/HDFS-13677) | Dynamic refresh Disk configuration results in overwriting VolumeMap | Blocker | . | xuzq | xuzq | -| [YARN-9285](https://issues.apache.org/jira/browse/YARN-9285) | RM UI progress column is of wrong type | Minor | yarn | Ahmed Hussein | Ahmed Hussein | -| [HDFS-14500](https://issues.apache.org/jira/browse/HDFS-14500) | NameNode StartupProgress continues to report edit log segments after the LOADING\_EDITS phase is finished | Major | namenode | Erik Krogen | Erik Krogen | -| [HADOOP-16331](https://issues.apache.org/jira/browse/HADOOP-16331) | Fix ASF License check in pom.xml | Major | . | Wanqiang Ji | Akira Ajisaka | -| [HDFS-14514](https://issues.apache.org/jira/browse/HDFS-14514) | Actual read size of open file in encryption zone still larger than listing size even after enabling HDFS-11402 in Hadoop 2 | Major | encryption, hdfs, snapshots | Siyao Meng | Siyao Meng | -| [HDFS-14512](https://issues.apache.org/jira/browse/HDFS-14512) | ONE\_SSD policy will be violated while write data with DistributedFileSystem.create(....favoredNodes) | Major | . | Shen Yinjie | Ayush Saxena | -| [HDFS-14521](https://issues.apache.org/jira/browse/HDFS-14521) | Suppress setReplication logging. | Major | . | Kihwal Lee | Kihwal Lee | -| [YARN-8625](https://issues.apache.org/jira/browse/YARN-8625) | Aggregate Resource Allocation for each job is not present in ATS | Major | ATSv2 | Prabhu Joseph | Prabhu Joseph | -| [HADOOP-16345](https://issues.apache.org/jira/browse/HADOOP-16345) | Potential NPE when instantiating FairCallQueue metrics | Major | ipc | Erik Krogen | Erik Krogen | -| [HDFS-14494](https://issues.apache.org/jira/browse/HDFS-14494) | Move Server logging of StatedId inside receiveRequestState() | Major | . | Konstantin Shvachko | Shweta | -| [HDFS-14535](https://issues.apache.org/jira/browse/HDFS-14535) | The default 8KB buffer in requestFileDescriptors#BufferedOutputStream is causing lots of heap allocation in HBase when using short-circut read | Major | hdfs-client | Zheng Hu | Zheng Hu | -| [HDFS-13730](https://issues.apache.org/jira/browse/HDFS-13730) | BlockReaderRemote.sendReadResult throws NPE | Major | hdfs-client | Wei-Chiu Chuang | Yuanbo Liu | -| [HDFS-13770](https://issues.apache.org/jira/browse/HDFS-13770) | dfsadmin -report does not always decrease "missing blocks (with replication factor 1)" metrics when file is deleted | Major | hdfs | Kitti Nanasi | Kitti Nanasi | -| [HDFS-14101](https://issues.apache.org/jira/browse/HDFS-14101) | Random failure of testListCorruptFilesCorruptedBlock | Major | test | Kihwal Lee | Zsolt Venczel | -| [HDFS-14465](https://issues.apache.org/jira/browse/HDFS-14465) | When the Block expected replications is larger than the number of DataNodes, entering maintenance will never exit. | Major | . | Yicong Cai | Yicong Cai | -| [HDFS-14541](https://issues.apache.org/jira/browse/HDFS-14541) | When evictableMmapped or evictable size is zero, do not throw NoSuchElementException | Major | hdfs-client, performance | Zheng Hu | Lisheng Sun | -| [HDFS-14629](https://issues.apache.org/jira/browse/HDFS-14629) | Property value Hard Coded in DNConf.java | Trivial | . | hemanthboyina | hemanthboyina | -| [HDFS-12703](https://issues.apache.org/jira/browse/HDFS-12703) | Exceptions are fatal to decommissioning monitor | Critical | namenode | Daryn Sharp | Xiaoqiao He | -| [HDFS-12748](https://issues.apache.org/jira/browse/HDFS-12748) | NameNode memory leak when accessing webhdfs GETHOMEDIRECTORY | Major | hdfs | Jiandan Yang | Weiwei Yang | -| [HADOOP-16386](https://issues.apache.org/jira/browse/HADOOP-16386) | FindBugs warning in branch-2: GlobalStorageStatistics defines non-transient non-serializable instance field map | Major | fs | Wei-Chiu Chuang | Masatake Iwasaki | -| [MAPREDUCE-6521](https://issues.apache.org/jira/browse/MAPREDUCE-6521) | MiniMRYarnCluster should not create /tmp/hadoop-yarn/staging on local filesystem in unit test | Major | test | Masatake Iwasaki | Masatake Iwasaki | -| [MAPREDUCE-7076](https://issues.apache.org/jira/browse/MAPREDUCE-7076) | TestNNBench#testNNBenchCreateReadAndDelete failing in our internal build | Minor | test | Rushabh Shah | kevin su | -| [YARN-9668](https://issues.apache.org/jira/browse/YARN-9668) | UGI conf doesn't read user overridden configurations on RM and NM startup | Major | . | Jonathan Hung | Jonathan Hung | -| [HADOOP-9844](https://issues.apache.org/jira/browse/HADOOP-9844) | NPE when trying to create an error message response of SASL RPC | Major | ipc | Steve Loughran | Steve Loughran | -| [HADOOP-16245](https://issues.apache.org/jira/browse/HADOOP-16245) | Enabling SSL within LdapGroupsMapping can break system SSL configs | Major | common, security | Erik Krogen | Erik Krogen | -| [HDFS-14660](https://issues.apache.org/jira/browse/HDFS-14660) | [SBN Read] ObserverNameNode should throw StandbyException for requests not from ObserverProxyProvider | Major | . | Chao Sun | Chao Sun | -| [HDFS-14429](https://issues.apache.org/jira/browse/HDFS-14429) | Block remain in COMMITTED but not COMPLETE caused by Decommission | Major | . | Yicong Cai | Yicong Cai | -| [HDFS-14672](https://issues.apache.org/jira/browse/HDFS-14672) | Backport HDFS-12703 to branch-2 | Major | namenode | Xiaoqiao He | Xiaoqiao He | -| [HADOOP-16435](https://issues.apache.org/jira/browse/HADOOP-16435) | RpcMetrics should not be retained forever | Critical | rpc-server | Zoltan Haindrich | Zoltan Haindrich | -| [HDFS-14464](https://issues.apache.org/jira/browse/HDFS-14464) | Remove unnecessary log message from DFSInputStream | Trivial | . | Kihwal Lee | Chao Sun | -| [HDFS-14569](https://issues.apache.org/jira/browse/HDFS-14569) | Result of crypto -listZones is not formatted properly | Major | . | hemanthboyina | hemanthboyina | -| [YARN-9596](https://issues.apache.org/jira/browse/YARN-9596) | QueueMetrics has incorrect metrics when labelled partitions are involved | Major | capacity scheduler | Muhammad Samir Khan | Muhammad Samir Khan | -| [HADOOP-15237](https://issues.apache.org/jira/browse/HADOOP-15237) | In KMS docs there should be one space between KMS\_LOG and NOTE | Minor | kms | Snigdhanjali Mishra | Snigdhanjali Mishra | -| [HDFS-14462](https://issues.apache.org/jira/browse/HDFS-14462) | WebHDFS throws "Error writing request body to server" instead of DSQuotaExceededException | Major | webhdfs | Erik Krogen | Simbarashe Dzinamarira | -| [HDFS-14631](https://issues.apache.org/jira/browse/HDFS-14631) | The DirectoryScanner doesn't fix the wrongly placed replica. | Major | . | Jinglun | Jinglun | -| [HDFS-14724](https://issues.apache.org/jira/browse/HDFS-14724) | Fix JDK7 compatibility in branch-2 | Blocker | . | Wei-Chiu Chuang | Chen Liang | -| [HDFS-14423](https://issues.apache.org/jira/browse/HDFS-14423) | Percent (%) and plus (+) characters no longer work in WebHDFS | Major | webhdfs | Jing Wang | Masatake Iwasaki | -| [HDFS-13101](https://issues.apache.org/jira/browse/HDFS-13101) | Yet another fsimage corruption related to snapshot | Major | snapshots | Yongjun Zhang | Shashikant Banerjee | -| [HDFS-14311](https://issues.apache.org/jira/browse/HDFS-14311) | Multi-threading conflict at layoutVersion when loading block pool storage | Major | rolling upgrades | Yicong Cai | Yicong Cai | -| [HADOOP-16494](https://issues.apache.org/jira/browse/HADOOP-16494) | Add SHA-256 or SHA-512 checksum to release artifacts to comply with the release distribution policy | Blocker | build | Akira Ajisaka | Akira Ajisaka | -| [HDFS-13977](https://issues.apache.org/jira/browse/HDFS-13977) | NameNode can kill itself if it tries to send too many txns to a QJM simultaneously | Major | namenode, qjm | Erik Krogen | Erik Krogen | -| [YARN-9438](https://issues.apache.org/jira/browse/YARN-9438) | launchTime not written to state store for running applications | Major | . | Jonathan Hung | Jonathan Hung | -| [YARN-7585](https://issues.apache.org/jira/browse/YARN-7585) | NodeManager should go unhealthy when state store throws DBException | Major | nodemanager | Wilfred Spiegelenburg | Wilfred Spiegelenburg | -| [HDFS-14726](https://issues.apache.org/jira/browse/HDFS-14726) | Fix JN incompatibility issue in branch-2 due to backport of HDFS-10519 | Blocker | journal-node | Chen Liang | Chen Liang | -| [YARN-9806](https://issues.apache.org/jira/browse/YARN-9806) | TestNMSimulator#testNMSimulator fails in branch-2 | Major | . | Jonathan Hung | Jonathan Hung | -| [YARN-9820](https://issues.apache.org/jira/browse/YARN-9820) | RM logs InvalidStateTransitionException when app is submitted | Critical | . | Rohith Sharma K S | Prabhu Joseph | -| [HDFS-14303](https://issues.apache.org/jira/browse/HDFS-14303) | check block directory logic not correct when there is only meta file, print no meaning warn log | Minor | datanode, hdfs | qiang Liu | qiang Liu | -| [HADOOP-16582](https://issues.apache.org/jira/browse/HADOOP-16582) | LocalFileSystem's mkdirs() does not work as expected under viewfs. | Major | . | Kihwal Lee | Kihwal Lee | -| [HADOOP-16581](https://issues.apache.org/jira/browse/HADOOP-16581) | ValueQueue does not trigger an async refill when number of values falls below watermark | Major | common, kms | Yuval Degani | Yuval Degani | -| [HDFS-14853](https://issues.apache.org/jira/browse/HDFS-14853) | NPE in DFSNetworkTopology#chooseRandomWithStorageType() when the excludedNode is not present | Major | . | Ranith Sardar | Ranith Sardar | -| [YARN-9858](https://issues.apache.org/jira/browse/YARN-9858) | Optimize RMContext getExclusiveEnforcedPartitions | Major | . | Jonathan Hung | Jonathan Hung | -| [HDFS-14655](https://issues.apache.org/jira/browse/HDFS-14655) | [SBN Read] Namenode crashes if one of The JN is down | Critical | . | Harshakiran Reddy | Ayush Saxena | -| [HDFS-14245](https://issues.apache.org/jira/browse/HDFS-14245) | Class cast error in GetGroups with ObserverReadProxyProvider | Major | . | Shen Yinjie | Erik Krogen | -| [HDFS-14509](https://issues.apache.org/jira/browse/HDFS-14509) | DN throws InvalidToken due to inequality of password when upgrade NN 2.x to 3.x | Blocker | . | Yuxuan Wang | Yuxuan Wang | -| [HADOOP-16655](https://issues.apache.org/jira/browse/HADOOP-16655) | Change cipher suite when fetching tomcat tarball for branch-2 | Major | . | Jonathan Hung | Jonathan Hung | - - -### TESTS: - -| JIRA | Summary | Priority | Component | Reporter | Contributor | -|:---- |:---- | :--- |:---- |:---- |:---- | -| [HDFS-13337](https://issues.apache.org/jira/browse/HDFS-13337) | Backport HDFS-4275 to branch-2.9 | Minor | . | Íñigo Goiri | Xiao Liang | -| [HDFS-13503](https://issues.apache.org/jira/browse/HDFS-13503) | Fix TestFsck test failures on Windows | Major | hdfs | Xiao Liang | Xiao Liang | -| [HDFS-13542](https://issues.apache.org/jira/browse/HDFS-13542) | TestBlockManager#testNeededReplicationWhileAppending fails due to improper cluster shutdown in TestBlockManager#testBlockManagerMachinesArray on Windows | Minor | . | Anbang Hu | Anbang Hu | -| [HDFS-13551](https://issues.apache.org/jira/browse/HDFS-13551) | TestMiniDFSCluster#testClusterSetStorageCapacity does not shut down cluster | Minor | . | Anbang Hu | Anbang Hu | -| [HDFS-11700](https://issues.apache.org/jira/browse/HDFS-11700) | TestHDFSServerPorts#testBackupNodePorts doesn't pass on Windows | Minor | . | Anbang Hu | Anbang Hu | -| [HDFS-13548](https://issues.apache.org/jira/browse/HDFS-13548) | TestResolveHdfsSymlink#testFcResolveAfs fails on Windows | Minor | . | Anbang Hu | Anbang Hu | -| [HDFS-13567](https://issues.apache.org/jira/browse/HDFS-13567) | TestNameNodeMetrics#testGenerateEDEKTime,TestNameNodeMetrics#testResourceCheck should use a different cluster basedir | Minor | . | Anbang Hu | Anbang Hu | -| [HDFS-13557](https://issues.apache.org/jira/browse/HDFS-13557) | TestDFSAdmin#testListOpenFiles fails on Windows | Minor | . | Anbang Hu | Anbang Hu | -| [HDFS-13550](https://issues.apache.org/jira/browse/HDFS-13550) | TestDebugAdmin#testComputeMetaCommand fails on Windows | Minor | . | Anbang Hu | Anbang Hu | -| [HDFS-13559](https://issues.apache.org/jira/browse/HDFS-13559) | TestBlockScanner does not close TestContext properly | Minor | . | Anbang Hu | Anbang Hu | -| [HDFS-13570](https://issues.apache.org/jira/browse/HDFS-13570) | TestQuotaByStorageType,TestQuota,TestDFSOutputStream fail on Windows | Minor | . | Anbang Hu | Anbang Hu | -| [HDFS-13558](https://issues.apache.org/jira/browse/HDFS-13558) | TestDatanodeHttpXFrame does not shut down cluster | Minor | . | Anbang Hu | Anbang Hu | -| [HDFS-13554](https://issues.apache.org/jira/browse/HDFS-13554) | TestDatanodeRegistration#testForcedRegistration does not shut down cluster | Minor | . | Anbang Hu | Anbang Hu | -| [HDFS-13556](https://issues.apache.org/jira/browse/HDFS-13556) | TestNestedEncryptionZones does not shut down cluster | Minor | . | Anbang Hu | Anbang Hu | -| [HDFS-13560](https://issues.apache.org/jira/browse/HDFS-13560) | Insufficient system resources exist to complete the requested service for some tests on Windows | Minor | . | Anbang Hu | Anbang Hu | -| [HDFS-13592](https://issues.apache.org/jira/browse/HDFS-13592) | TestNameNodePrunesMissingStorages#testNameNodePrunesUnreportedStorages does not shut down cluster properly | Minor | . | Anbang Hu | Anbang Hu | -| [HDFS-13593](https://issues.apache.org/jira/browse/HDFS-13593) | TestBlockReaderLocalLegacy#testBlockReaderLocalLegacyWithAppend fails on Windows | Minor | test | Anbang Hu | Anbang Hu | -| [HDFS-13587](https://issues.apache.org/jira/browse/HDFS-13587) | TestQuorumJournalManager fails on Windows | Minor | . | Anbang Hu | Anbang Hu | -| [HDFS-13620](https://issues.apache.org/jira/browse/HDFS-13620) | Randomize the test directory path for TestHDFSFileSystemContract | Minor | . | Anbang Hu | Anbang Hu | -| [HDFS-13591](https://issues.apache.org/jira/browse/HDFS-13591) | TestDFSShell#testSetrepLow fails on Windows | Minor | . | Anbang Hu | Anbang Hu | -| [HDFS-13632](https://issues.apache.org/jira/browse/HDFS-13632) | Randomize baseDir for MiniJournalCluster in MiniQJMHACluster for TestDFSAdminWithHA | Minor | . | Anbang Hu | Anbang Hu | -| [MAPREDUCE-7102](https://issues.apache.org/jira/browse/MAPREDUCE-7102) | Fix TestJavaSerialization for Windows due a mismatch line separator | Minor | . | Giovanni Matteo Fumarola | Giovanni Matteo Fumarola | -| [HDFS-13652](https://issues.apache.org/jira/browse/HDFS-13652) | Randomize baseDir for MiniDFSCluster in TestBlockScanner | Minor | . | Anbang Hu | Anbang Hu | -| [YARN-8370](https://issues.apache.org/jira/browse/YARN-8370) | Some Node Manager tests fail on Windows due to improper path/file separator | Minor | . | Anbang Hu | Anbang Hu | -| [YARN-8422](https://issues.apache.org/jira/browse/YARN-8422) | TestAMSimulator failing with NPE | Minor | . | Giovanni Matteo Fumarola | Giovanni Matteo Fumarola | -| [HADOOP-15532](https://issues.apache.org/jira/browse/HADOOP-15532) | TestBasicDiskValidator fails with NoSuchFileException | Minor | . | Íñigo Goiri | Giovanni Matteo Fumarola | -| [HDFS-13563](https://issues.apache.org/jira/browse/HDFS-13563) | TestDFSAdminWithHA times out on Windows | Minor | . | Anbang Hu | Lukas Majercak | -| [HDFS-13681](https://issues.apache.org/jira/browse/HDFS-13681) | Fix TestStartup.testNNFailToStartOnReadOnlyNNDir test failure on Windows | Major | test | Xiao Liang | Xiao Liang | -| [YARN-8944](https://issues.apache.org/jira/browse/YARN-8944) | TestContainerAllocation.testUserLimitAllocationMultipleContainers failure after YARN-8896 | Minor | capacity scheduler | Wilfred Spiegelenburg | Wilfred Spiegelenburg | -| [HDFS-11950](https://issues.apache.org/jira/browse/HDFS-11950) | Disable libhdfs zerocopy test on Mac | Minor | libhdfs | John Zhuge | Akira Ajisaka | - - -### SUB-TASKS: - -| JIRA | Summary | Priority | Component | Reporter | Contributor | -|:---- |:---- | :--- |:---- |:---- |:---- | -| [YARN-4081](https://issues.apache.org/jira/browse/YARN-4081) | Add support for multiple resource types in the Resource class | Major | resourcemanager | Varun Vasudev | Varun Vasudev | -| [YARN-4172](https://issues.apache.org/jira/browse/YARN-4172) | Extend DominantResourceCalculator to account for all resources | Major | resourcemanager | Varun Vasudev | Varun Vasudev | -| [YARN-4715](https://issues.apache.org/jira/browse/YARN-4715) | Add support to read resource types from a config file | Major | nodemanager, resourcemanager | Varun Vasudev | Varun Vasudev | -| [YARN-4829](https://issues.apache.org/jira/browse/YARN-4829) | Add support for binary units | Major | nodemanager, resourcemanager | Varun Vasudev | Varun Vasudev | -| [YARN-4830](https://issues.apache.org/jira/browse/YARN-4830) | Add support for resource types in the nodemanager | Major | nodemanager | Varun Vasudev | Varun Vasudev | -| [YARN-5242](https://issues.apache.org/jira/browse/YARN-5242) | Update DominantResourceCalculator to consider all resource types in calculations | Major | resourcemanager | Varun Vasudev | Varun Vasudev | -| [YARN-5586](https://issues.apache.org/jira/browse/YARN-5586) | Update the Resources class to consider all resource types | Major | nodemanager, resourcemanager | Varun Vasudev | Varun Vasudev | -| [YARN-6761](https://issues.apache.org/jira/browse/YARN-6761) | Fix build for YARN-3926 branch | Major | nodemanager, resourcemanager | Varun Vasudev | Varun Vasudev | -| [YARN-6786](https://issues.apache.org/jira/browse/YARN-6786) | ResourcePBImpl imports cleanup | Trivial | resourcemanager | Daniel Templeton | Yeliang Cang | -| [YARN-6788](https://issues.apache.org/jira/browse/YARN-6788) | Improve performance of resource profile branch | Blocker | nodemanager, resourcemanager | Sunil G | Sunil G | -| [YARN-6994](https://issues.apache.org/jira/browse/YARN-6994) | Remove last uses of Long from resource types code | Minor | resourcemanager | Daniel Templeton | Daniel Templeton | -| [YARN-6892](https://issues.apache.org/jira/browse/YARN-6892) | Improve API implementation in Resources and DominantResourceCalculator class | Major | nodemanager, resourcemanager | Sunil G | Sunil G | -| [YARN-6610](https://issues.apache.org/jira/browse/YARN-6610) | DominantResourceCalculator#getResourceAsValue dominant param is updated to handle multiple resources | Critical | resourcemanager | Daniel Templeton | Daniel Templeton | -| [YARN-7030](https://issues.apache.org/jira/browse/YARN-7030) | Performance optimizations in Resource and ResourceUtils class | Critical | nodemanager, resourcemanager | Wangda Tan | Wangda Tan | -| [YARN-7042](https://issues.apache.org/jira/browse/YARN-7042) | Clean up unit tests after YARN-6610 | Major | test | Daniel Templeton | Daniel Templeton | -| [YARN-6789](https://issues.apache.org/jira/browse/YARN-6789) | Add Client API to get all supported resource types from RM | Major | nodemanager, resourcemanager | Sunil G | Sunil G | -| [YARN-6781](https://issues.apache.org/jira/browse/YARN-6781) | ResourceUtils#initializeResourcesMap takes an unnecessary Map parameter | Minor | resourcemanager | Daniel Templeton | Yu-Tang Lin | -| [YARN-7067](https://issues.apache.org/jira/browse/YARN-7067) | Optimize ResourceType information display in UI | Critical | nodemanager, resourcemanager | Wangda Tan | Wangda Tan | -| [YARN-7039](https://issues.apache.org/jira/browse/YARN-7039) | Fix javac and javadoc errors in YARN-3926 branch | Major | nodemanager, resourcemanager | Sunil G | Sunil G | -| [YARN-7093](https://issues.apache.org/jira/browse/YARN-7093) | Improve log message in ResourceUtils | Trivial | nodemanager, resourcemanager | Sunil G | Sunil G | -| [YARN-6933](https://issues.apache.org/jira/browse/YARN-6933) | ResourceUtils.DISALLOWED\_NAMES check is duplicated | Major | resourcemanager | Daniel Templeton | Manikandan R | -| [YARN-7137](https://issues.apache.org/jira/browse/YARN-7137) | Move newly added APIs to unstable in YARN-3926 branch | Blocker | nodemanager, resourcemanager | Wangda Tan | Wangda Tan | -| [HADOOP-14799](https://issues.apache.org/jira/browse/HADOOP-14799) | Update nimbus-jose-jwt to 4.41.1 | Major | . | Ray Chiang | Ray Chiang | -| [YARN-7345](https://issues.apache.org/jira/browse/YARN-7345) | GPU Isolation: Incorrect minor device numbers written to devices.deny file | Major | . | Jonathan Hung | Jonathan Hung | -| [HADOOP-14997](https://issues.apache.org/jira/browse/HADOOP-14997) | Add hadoop-aliyun as dependency of hadoop-cloud-storage | Minor | fs/oss | Genmao Yu | Genmao Yu | -| [YARN-7143](https://issues.apache.org/jira/browse/YARN-7143) | FileNotFound handling in ResourceUtils is inconsistent | Major | resourcemanager | Daniel Templeton | Daniel Templeton | -| [YARN-6909](https://issues.apache.org/jira/browse/YARN-6909) | Use LightWeightedResource when number of resource types more than two | Critical | resourcemanager | Daniel Templeton | Sunil G | -| [HDFS-12801](https://issues.apache.org/jira/browse/HDFS-12801) | RBF: Set MountTableResolver as default file resolver | Minor | . | Íñigo Goiri | Íñigo Goiri | -| [YARN-7430](https://issues.apache.org/jira/browse/YARN-7430) | Enable user re-mapping for Docker containers by default | Blocker | security, yarn | Eric Yang | Eric Yang | -| [HADOOP-15024](https://issues.apache.org/jira/browse/HADOOP-15024) | AliyunOSS: support user agent configuration and include that & Hadoop version information to oss server | Major | fs, fs/oss | Sammi Chen | Sammi Chen | -| [HDFS-12858](https://issues.apache.org/jira/browse/HDFS-12858) | RBF: Add router admin commands usage in HDFS commands reference doc | Minor | documentation | Yiqun Lin | Yiqun Lin | -| [HDFS-12835](https://issues.apache.org/jira/browse/HDFS-12835) | RBF: Fix Javadoc parameter errors | Minor | . | Wei Yan | Wei Yan | -| [YARN-7573](https://issues.apache.org/jira/browse/YARN-7573) | Gpu Information page could be empty for nodes without GPU | Major | webapp, yarn-ui-v2 | Sunil G | Sunil G | -| [HDFS-12396](https://issues.apache.org/jira/browse/HDFS-12396) | Webhdfs file system should get delegation token from kms provider. | Major | encryption, kms, webhdfs | Rushabh Shah | Rushabh Shah | -| [YARN-6704](https://issues.apache.org/jira/browse/YARN-6704) | Add support for work preserving NM restart when FederationInterceptor is enabled in AMRMProxyService | Major | . | Botong Huang | Botong Huang | -| [HDFS-12875](https://issues.apache.org/jira/browse/HDFS-12875) | RBF: Complete logic for -readonly option of dfsrouteradmin add command | Major | . | Yiqun Lin | Íñigo Goiri | -| [YARN-7383](https://issues.apache.org/jira/browse/YARN-7383) | Node resource is not parsed correctly for resource names containing dot | Major | nodemanager, resourcemanager | Jonathan Hung | Gergely Novák | -| [YARN-7630](https://issues.apache.org/jira/browse/YARN-7630) | Fix AMRMToken rollover handling in AMRMProxy | Minor | . | Botong Huang | Botong Huang | -| [HDFS-12937](https://issues.apache.org/jira/browse/HDFS-12937) | RBF: Add more unit tests for router admin commands | Major | test | Yiqun Lin | Yiqun Lin | -| [YARN-7032](https://issues.apache.org/jira/browse/YARN-7032) | [ATSv2] NPE while starting hbase co-processor when HBase authorization is enabled. | Critical | . | Rohith Sharma K S | Rohith Sharma K S | -| [HDFS-12988](https://issues.apache.org/jira/browse/HDFS-12988) | RBF: Mount table entries not properly updated in the local cache | Major | . | Íñigo Goiri | Íñigo Goiri | -| [HDFS-12802](https://issues.apache.org/jira/browse/HDFS-12802) | RBF: Control MountTableResolver cache size | Major | . | Íñigo Goiri | Íñigo Goiri | -| [HDFS-12934](https://issues.apache.org/jira/browse/HDFS-12934) | RBF: Federation supports global quota | Major | . | Yiqun Lin | Yiqun Lin | -| [HDFS-12972](https://issues.apache.org/jira/browse/HDFS-12972) | RBF: Display mount table quota info in Web UI and admin command | Major | . | Yiqun Lin | Yiqun Lin | -| [YARN-6736](https://issues.apache.org/jira/browse/YARN-6736) | Consider writing to both ats v1 & v2 from RM for smoother upgrades | Major | timelineserver | Vrushali C | Aaron Gresch | -| [HADOOP-15027](https://issues.apache.org/jira/browse/HADOOP-15027) | AliyunOSS: Support multi-thread pre-read to improve sequential read from Hadoop to Aliyun OSS performance | Major | fs/oss | wujinhu | wujinhu | -| [HDFS-12973](https://issues.apache.org/jira/browse/HDFS-12973) | RBF: Document global quota supporting in federation | Major | . | Yiqun Lin | Yiqun Lin | -| [HDFS-13028](https://issues.apache.org/jira/browse/HDFS-13028) | RBF: Fix spurious TestRouterRpc#testProxyGetStats | Minor | . | Íñigo Goiri | Íñigo Goiri | -| [HDFS-12772](https://issues.apache.org/jira/browse/HDFS-12772) | RBF: Federation Router State State Store internal API | Major | . | Íñigo Goiri | Íñigo Goiri | -| [HDFS-13042](https://issues.apache.org/jira/browse/HDFS-13042) | RBF: Heartbeat Router State | Major | . | Íñigo Goiri | Íñigo Goiri | -| [HDFS-13049](https://issues.apache.org/jira/browse/HDFS-13049) | RBF: Inconsistent Router OPTS config in branch-2 and branch-3 | Minor | . | Wei Yan | Wei Yan | -| [YARN-7817](https://issues.apache.org/jira/browse/YARN-7817) | Add Resource reference to RM's NodeInfo object so REST API can get non memory/vcore resource usages. | Major | . | Sumana Sathish | Sunil G | -| [HDFS-12574](https://issues.apache.org/jira/browse/HDFS-12574) | Add CryptoInputStream to WebHdfsFileSystem read call. | Major | encryption, kms, webhdfs | Rushabh Shah | Rushabh Shah | -| [HDFS-13044](https://issues.apache.org/jira/browse/HDFS-13044) | RBF: Add a safe mode for the Router | Major | . | Íñigo Goiri | Íñigo Goiri | -| [HDFS-13043](https://issues.apache.org/jira/browse/HDFS-13043) | RBF: Expose the state of the Routers in the federation | Major | . | Íñigo Goiri | Íñigo Goiri | -| [HDFS-13068](https://issues.apache.org/jira/browse/HDFS-13068) | RBF: Add router admin option to manage safe mode | Major | . | Íñigo Goiri | Yiqun Lin | -| [YARN-7860](https://issues.apache.org/jira/browse/YARN-7860) | Fix UT failure TestRMWebServiceAppsNodelabel#testAppsRunning | Major | . | Weiwei Yang | Sunil G | -| [HDFS-13119](https://issues.apache.org/jira/browse/HDFS-13119) | RBF: Manage unavailable clusters | Major | . | Íñigo Goiri | Yiqun Lin | -| [YARN-7223](https://issues.apache.org/jira/browse/YARN-7223) | Document GPU isolation feature | Blocker | . | Wangda Tan | Wangda Tan | -| [HDFS-13187](https://issues.apache.org/jira/browse/HDFS-13187) | RBF: Fix Routers information shown in the web UI | Minor | . | Wei Yan | Wei Yan | -| [HDFS-13184](https://issues.apache.org/jira/browse/HDFS-13184) | RBF: Improve the unit test TestRouterRPCClientRetries | Minor | test | Yiqun Lin | Yiqun Lin | -| [HDFS-13199](https://issues.apache.org/jira/browse/HDFS-13199) | RBF: Fix the hdfs router page missing label icon issue | Major | federation, hdfs | maobaolong | maobaolong | -| [HADOOP-15090](https://issues.apache.org/jira/browse/HADOOP-15090) | Add ADL troubleshooting doc | Major | documentation, fs/adl | Steve Loughran | Steve Loughran | -| [YARN-7919](https://issues.apache.org/jira/browse/YARN-7919) | Refactor timelineservice-hbase module into submodules | Major | timelineservice | Haibo Chen | Haibo Chen | -| [YARN-8003](https://issues.apache.org/jira/browse/YARN-8003) | Backport the code structure changes in YARN-7346 to branch-2 | Major | . | Haibo Chen | Haibo Chen | -| [HDFS-13214](https://issues.apache.org/jira/browse/HDFS-13214) | RBF: Complete document of Router configuration | Major | . | Tao Jie | Yiqun Lin | -| [HADOOP-15267](https://issues.apache.org/jira/browse/HADOOP-15267) | S3A multipart upload fails when SSE-C encryption is enabled | Critical | fs/s3 | Anis Elleuch | Anis Elleuch | -| [HDFS-13230](https://issues.apache.org/jira/browse/HDFS-13230) | RBF: ConnectionManager's cleanup task will compare each pool's own active conns with its total conns | Minor | . | Wei Yan | Chao Sun | -| [HDFS-13233](https://issues.apache.org/jira/browse/HDFS-13233) | RBF: MountTableResolver doesn't return the correct mount point of the given path | Major | hdfs | wangzhiyuan | wangzhiyuan | -| [HDFS-13212](https://issues.apache.org/jira/browse/HDFS-13212) | RBF: Fix router location cache issue | Major | federation, hdfs | Wu Weiwei | Wu Weiwei | -| [HDFS-13232](https://issues.apache.org/jira/browse/HDFS-13232) | RBF: ConnectionPool should return first usable connection | Minor | . | Wei Yan | Ekanth Sethuramalingam | -| [HDFS-13240](https://issues.apache.org/jira/browse/HDFS-13240) | RBF: Update some inaccurate document descriptions | Minor | . | Yiqun Lin | Yiqun Lin | -| [HDFS-11399](https://issues.apache.org/jira/browse/HDFS-11399) | Many tests fails in Windows due to injecting disk failures | Major | . | Yiqun Lin | Yiqun Lin | -| [HDFS-13241](https://issues.apache.org/jira/browse/HDFS-13241) | RBF: TestRouterSafemode failed if the port 8888 is in use | Major | hdfs, test | maobaolong | maobaolong | -| [HDFS-13253](https://issues.apache.org/jira/browse/HDFS-13253) | RBF: Quota management incorrect parent-child relationship judgement | Major | . | Yiqun Lin | Yiqun Lin | -| [HDFS-13226](https://issues.apache.org/jira/browse/HDFS-13226) | RBF: Throw the exception if mount table entry validated failed | Major | hdfs | maobaolong | maobaolong | -| [HADOOP-15308](https://issues.apache.org/jira/browse/HADOOP-15308) | TestConfiguration fails on Windows because of paths | Major | test | Íñigo Goiri | Xiao Liang | -| [HDFS-12773](https://issues.apache.org/jira/browse/HDFS-12773) | RBF: Improve State Store FS implementation | Major | . | Íñigo Goiri | Íñigo Goiri | -| [HDFS-13198](https://issues.apache.org/jira/browse/HDFS-13198) | RBF: RouterHeartbeatService throws out CachedStateStore related exceptions when starting router | Minor | . | Wei Yan | Wei Yan | -| [HDFS-13224](https://issues.apache.org/jira/browse/HDFS-13224) | RBF: Resolvers to support mount points across multiple subclusters | Major | . | Íñigo Goiri | Íñigo Goiri | -| [HDFS-13299](https://issues.apache.org/jira/browse/HDFS-13299) | RBF : Fix compilation error in branch-2 (TestMultipleDestinationResolver) | Blocker | . | Brahma Reddy Battula | Brahma Reddy Battula | -| [HADOOP-15262](https://issues.apache.org/jira/browse/HADOOP-15262) | AliyunOSS: move files under a directory in parallel when rename a directory | Major | fs/oss | wujinhu | wujinhu | -| [HDFS-13215](https://issues.apache.org/jira/browse/HDFS-13215) | RBF: Move Router to its own module | Major | . | Íñigo Goiri | Wei Yan | -| [HDFS-13307](https://issues.apache.org/jira/browse/HDFS-13307) | RBF: Improve the use of setQuota command | Major | . | liuhongtong | liuhongtong | -| [HDFS-13250](https://issues.apache.org/jira/browse/HDFS-13250) | RBF: Router to manage requests across multiple subclusters | Major | . | Íñigo Goiri | Íñigo Goiri | -| [HDFS-13318](https://issues.apache.org/jira/browse/HDFS-13318) | RBF: Fix FindBugs in hadoop-hdfs-rbf | Minor | . | Íñigo Goiri | Ekanth Sethuramalingam | -| [HDFS-12792](https://issues.apache.org/jira/browse/HDFS-12792) | RBF: Test Router-based federation using HDFSContract | Major | . | Íñigo Goiri | Íñigo Goiri | -| [YARN-7581](https://issues.apache.org/jira/browse/YARN-7581) | HBase filters are not constructed correctly in ATSv2 | Major | ATSv2 | Haibo Chen | Haibo Chen | -| [YARN-7986](https://issues.apache.org/jira/browse/YARN-7986) | ATSv2 REST API queries do not return results for uppercase application tags | Critical | . | Charan Hebri | Charan Hebri | -| [HDFS-12512](https://issues.apache.org/jira/browse/HDFS-12512) | RBF: Add WebHDFS | Major | fs | Íñigo Goiri | Wei Yan | -| [HDFS-13291](https://issues.apache.org/jira/browse/HDFS-13291) | RBF: Implement available space based OrderResolver | Major | . | Yiqun Lin | Yiqun Lin | -| [HDFS-13204](https://issues.apache.org/jira/browse/HDFS-13204) | RBF: Optimize name service safe mode icon | Minor | . | liuhongtong | liuhongtong | -| [HDFS-13352](https://issues.apache.org/jira/browse/HDFS-13352) | RBF: Add xsl stylesheet for hdfs-rbf-default.xml | Major | documentation | Takanobu Asanuma | Takanobu Asanuma | -| [YARN-8010](https://issues.apache.org/jira/browse/YARN-8010) | Add config in FederationRMFailoverProxy to not bypass facade cache when failing over | Minor | . | Botong Huang | Botong Huang | -| [HDFS-13347](https://issues.apache.org/jira/browse/HDFS-13347) | RBF: Cache datanode reports | Minor | . | Íñigo Goiri | Íñigo Goiri | -| [HDFS-13289](https://issues.apache.org/jira/browse/HDFS-13289) | RBF: TestConnectionManager#testCleanup() test case need correction | Minor | . | Dibyendu Karmakar | Dibyendu Karmakar | -| [HDFS-13364](https://issues.apache.org/jira/browse/HDFS-13364) | RBF: Support NamenodeProtocol in the Router | Major | . | Íñigo Goiri | Íñigo Goiri | -| [HADOOP-14651](https://issues.apache.org/jira/browse/HADOOP-14651) | Update okhttp version to 2.7.5 | Major | fs/adl | Ray Chiang | Ray Chiang | -| [YARN-6936](https://issues.apache.org/jira/browse/YARN-6936) | [Atsv2] Retrospect storing entities into sub application table from client perspective | Major | . | Rohith Sharma K S | Rohith Sharma K S | -| [HDFS-13353](https://issues.apache.org/jira/browse/HDFS-13353) | RBF: TestRouterWebHDFSContractCreate failed | Major | test | Takanobu Asanuma | Takanobu Asanuma | -| [YARN-8107](https://issues.apache.org/jira/browse/YARN-8107) | Give an informative message when incorrect format is used in ATSv2 filter attributes | Major | ATSv2 | Charan Hebri | Rohith Sharma K S | -| [YARN-8110](https://issues.apache.org/jira/browse/YARN-8110) | AMRMProxy recover should catch for all throwable to avoid premature exit | Major | . | Botong Huang | Botong Huang | -| [HDFS-13402](https://issues.apache.org/jira/browse/HDFS-13402) | RBF: Fix java doc for StateStoreFileSystemImpl | Minor | hdfs | Yiran Wu | Yiran Wu | -| [HDFS-13380](https://issues.apache.org/jira/browse/HDFS-13380) | RBF: mv/rm fail after the directory exceeded the quota limit | Major | . | Wu Weiwei | Yiqun Lin | -| [HDFS-13410](https://issues.apache.org/jira/browse/HDFS-13410) | RBF: Support federation with no subclusters | Minor | . | Íñigo Goiri | Íñigo Goiri | -| [HDFS-13384](https://issues.apache.org/jira/browse/HDFS-13384) | RBF: Improve timeout RPC call mechanism | Minor | . | Íñigo Goiri | Íñigo Goiri | -| [HDFS-13045](https://issues.apache.org/jira/browse/HDFS-13045) | RBF: Improve error message returned from subcluster | Minor | . | Wei Yan | Íñigo Goiri | -| [HDFS-13428](https://issues.apache.org/jira/browse/HDFS-13428) | RBF: Remove LinkedList From StateStoreFileImpl.java | Trivial | federation | David Mollitor | David Mollitor | -| [HADOOP-14999](https://issues.apache.org/jira/browse/HADOOP-14999) | AliyunOSS: provide one asynchronous multi-part based uploading mechanism | Major | fs/oss | Genmao Yu | Genmao Yu | -| [YARN-7810](https://issues.apache.org/jira/browse/YARN-7810) | TestDockerContainerRuntime test failures due to UID lookup of a non-existent user | Major | . | Shane Kumpf | Shane Kumpf | -| [HDFS-13435](https://issues.apache.org/jira/browse/HDFS-13435) | RBF: Improve the error loggings for printing the stack trace | Major | . | Yiqun Lin | Yiqun Lin | -| [YARN-7189](https://issues.apache.org/jira/browse/YARN-7189) | Container-executor doesn't remove Docker containers that error out early | Major | yarn | Eric Badger | Eric Badger | -| [HDFS-13466](https://issues.apache.org/jira/browse/HDFS-13466) | RBF: Add more router-related information to the UI | Minor | . | Wei Yan | Wei Yan | -| [HDFS-13453](https://issues.apache.org/jira/browse/HDFS-13453) | RBF: getMountPointDates should fetch latest subdir time/date when parent dir is not present but /parent/child dirs are present in mount table | Major | . | Dibyendu Karmakar | Dibyendu Karmakar | -| [HDFS-13478](https://issues.apache.org/jira/browse/HDFS-13478) | RBF: Disabled Nameservice store API | Major | . | Íñigo Goiri | Íñigo Goiri | -| [HDFS-13490](https://issues.apache.org/jira/browse/HDFS-13490) | RBF: Fix setSafeMode in the Router | Major | . | Íñigo Goiri | Íñigo Goiri | -| [HDFS-13484](https://issues.apache.org/jira/browse/HDFS-13484) | RBF: Disable Nameservices from the federation | Major | . | Íñigo Goiri | Íñigo Goiri | -| [HDFS-13326](https://issues.apache.org/jira/browse/HDFS-13326) | RBF: Improve the interfaces to modify and view mount tables | Minor | . | Wei Yan | Gang Li | -| [HDFS-13499](https://issues.apache.org/jira/browse/HDFS-13499) | RBF: Show disabled name services in the UI | Minor | . | Íñigo Goiri | Íñigo Goiri | -| [YARN-8215](https://issues.apache.org/jira/browse/YARN-8215) | ATS v2 returns invalid YARN\_CONTAINER\_ALLOCATED\_HOST\_HTTP\_ADDRESS from NM | Critical | ATSv2 | Yesha Vora | Rohith Sharma K S | -| [HDFS-13508](https://issues.apache.org/jira/browse/HDFS-13508) | RBF: Normalize paths (automatically) when adding, updating, removing or listing mount table entries | Minor | . | Ekanth Sethuramalingam | Ekanth Sethuramalingam | -| [HDFS-13434](https://issues.apache.org/jira/browse/HDFS-13434) | RBF: Fix dead links in RBF document | Major | documentation | Akira Ajisaka | Chetna Chaudhari | -| [HDFS-13488](https://issues.apache.org/jira/browse/HDFS-13488) | RBF: Reject requests when a Router is overloaded | Major | . | Íñigo Goiri | Íñigo Goiri | -| [HDFS-13525](https://issues.apache.org/jira/browse/HDFS-13525) | RBF: Add unit test TestStateStoreDisabledNameservice | Major | . | Yiqun Lin | Yiqun Lin | -| [YARN-8253](https://issues.apache.org/jira/browse/YARN-8253) | HTTPS Ats v2 api call fails with "bad HTTP parsed" | Critical | ATSv2 | Yesha Vora | Charan Hebri | -| [HADOOP-15454](https://issues.apache.org/jira/browse/HADOOP-15454) | TestRollingFileSystemSinkWithLocal fails on Windows | Major | test | Xiao Liang | Xiao Liang | -| [HDFS-13346](https://issues.apache.org/jira/browse/HDFS-13346) | RBF: Fix synchronization of router quota and nameservice quota | Major | . | liuhongtong | Yiqun Lin | -| [YARN-8247](https://issues.apache.org/jira/browse/YARN-8247) | Incorrect HTTP status code returned by ATSv2 for non-whitelisted users | Critical | ATSv2 | Charan Hebri | Rohith Sharma K S | -| [YARN-8130](https://issues.apache.org/jira/browse/YARN-8130) | Race condition when container events are published for KILLED applications | Major | ATSv2 | Charan Hebri | Rohith Sharma K S | -| [YARN-7900](https://issues.apache.org/jira/browse/YARN-7900) | [AMRMProxy] AMRMClientRelayer for stateful FederationInterceptor | Major | . | Botong Huang | Botong Huang | -| [HADOOP-15498](https://issues.apache.org/jira/browse/HADOOP-15498) | TestHadoopArchiveLogs (#testGenerateScript, #testPrepareWorkingDir) fails on Windows | Minor | . | Anbang Hu | Anbang Hu | -| [HADOOP-15497](https://issues.apache.org/jira/browse/HADOOP-15497) | TestTrash should use proper test path to avoid failing on Windows | Minor | . | Anbang Hu | Anbang Hu | -| [HDFS-13637](https://issues.apache.org/jira/browse/HDFS-13637) | RBF: Router fails when threadIndex (in ConnectionPool) wraps around Integer.MIN\_VALUE | Critical | federation | CR Hota | CR Hota | -| [YARN-4781](https://issues.apache.org/jira/browse/YARN-4781) | Support intra-queue preemption for fairness ordering policy. | Major | scheduler | Wangda Tan | Eric Payne | -| [HADOOP-15506](https://issues.apache.org/jira/browse/HADOOP-15506) | Upgrade Azure Storage Sdk version to 7.0.0 and update corresponding code blocks | Minor | fs/azure | Esfandiar Manii | Esfandiar Manii | -| [HADOOP-15529](https://issues.apache.org/jira/browse/HADOOP-15529) | ContainerLaunch#testInvalidEnvVariableSubstitutionType is not supported in Windows | Minor | . | Giovanni Matteo Fumarola | Giovanni Matteo Fumarola | -| [HADOOP-15533](https://issues.apache.org/jira/browse/HADOOP-15533) | Make WASB listStatus messages consistent | Trivial | fs/azure | Esfandiar Manii | Esfandiar Manii | -| [HADOOP-15458](https://issues.apache.org/jira/browse/HADOOP-15458) | TestLocalFileSystem#testFSOutputStreamBuilder fails on Windows | Minor | test | Xiao Liang | Xiao Liang | -| [YARN-8481](https://issues.apache.org/jira/browse/YARN-8481) | AMRMProxyPolicies should accept heartbeat response from new/unknown subclusters | Minor | amrmproxy, federation | Botong Huang | Botong Huang | -| [HDFS-13528](https://issues.apache.org/jira/browse/HDFS-13528) | RBF: If a directory exceeds quota limit then quota usage is not refreshed for other mount entries | Major | . | Dibyendu Karmakar | Dibyendu Karmakar | -| [HDFS-13710](https://issues.apache.org/jira/browse/HDFS-13710) | RBF: setQuota and getQuotaUsage should check the dfs.federation.router.quota.enable | Major | federation, hdfs | yanghuafeng | yanghuafeng | -| [HDFS-13726](https://issues.apache.org/jira/browse/HDFS-13726) | RBF: Fix RBF configuration links | Minor | documentation | Takanobu Asanuma | Takanobu Asanuma | -| [HDFS-13475](https://issues.apache.org/jira/browse/HDFS-13475) | RBF: Admin cannot enforce Router enter SafeMode | Major | . | Wei Yan | Chao Sun | -| [HDFS-13733](https://issues.apache.org/jira/browse/HDFS-13733) | RBF: Add Web UI configurations and descriptions to RBF document | Minor | documentation | Takanobu Asanuma | Takanobu Asanuma | -| [HDFS-13743](https://issues.apache.org/jira/browse/HDFS-13743) | RBF: Router throws NullPointerException due to the invalid initialization of MountTableResolver | Major | . | Takanobu Asanuma | Takanobu Asanuma | -| [HDFS-13583](https://issues.apache.org/jira/browse/HDFS-13583) | RBF: Router admin clrQuota is not synchronized with nameservice | Major | . | Dibyendu Karmakar | Dibyendu Karmakar | -| [HDFS-13750](https://issues.apache.org/jira/browse/HDFS-13750) | RBF: Router ID in RouterRpcClient is always null | Major | . | Takanobu Asanuma | Takanobu Asanuma | -| [YARN-8129](https://issues.apache.org/jira/browse/YARN-8129) | Improve error message for invalid value in fields attribute | Minor | ATSv2 | Charan Hebri | Abhishek Modi | -| [YARN-8581](https://issues.apache.org/jira/browse/YARN-8581) | [AMRMProxy] Add sub-cluster timeout in LocalityMulticastAMRMProxyPolicy | Major | amrmproxy, federation | Botong Huang | Botong Huang | -| [YARN-8673](https://issues.apache.org/jira/browse/YARN-8673) | [AMRMProxy] More robust responseId resync after an YarnRM master slave switch | Major | amrmproxy | Botong Huang | Botong Huang | -| [HDFS-13848](https://issues.apache.org/jira/browse/HDFS-13848) | Refactor NameNode failover proxy providers | Major | ha, hdfs-client | Konstantin Shvachko | Konstantin Shvachko | -| [HDFS-13634](https://issues.apache.org/jira/browse/HDFS-13634) | RBF: Configurable value in xml for async connection request queue size. | Major | federation | CR Hota | CR Hota | -| [HADOOP-15731](https://issues.apache.org/jira/browse/HADOOP-15731) | TestDistributedShell fails on Windows | Major | . | Botong Huang | Botong Huang | -| [HADOOP-15759](https://issues.apache.org/jira/browse/HADOOP-15759) | AliyunOSS: update oss-sdk version to 3.0.0 | Major | fs/oss | wujinhu | wujinhu | -| [HADOOP-15748](https://issues.apache.org/jira/browse/HADOOP-15748) | S3 listing inconsistency can raise NPE in globber | Major | fs | Steve Loughran | Steve Loughran | -| [YARN-8696](https://issues.apache.org/jira/browse/YARN-8696) | [AMRMProxy] FederationInterceptor upgrade: home sub-cluster heartbeat async | Major | nodemanager | Botong Huang | Botong Huang | -| [HADOOP-15671](https://issues.apache.org/jira/browse/HADOOP-15671) | AliyunOSS: Support Assume Roles in AliyunOSS | Major | fs/oss | wujinhu | wujinhu | -| [HDFS-13790](https://issues.apache.org/jira/browse/HDFS-13790) | RBF: Move ClientProtocol APIs to its own module | Major | . | Íñigo Goiri | Chao Sun | -| [YARN-7652](https://issues.apache.org/jira/browse/YARN-7652) | Handle AM register requests asynchronously in FederationInterceptor | Major | amrmproxy, federation | Subramaniam Krishnan | Botong Huang | -| [YARN-6989](https://issues.apache.org/jira/browse/YARN-6989) | Ensure timeline service v2 codebase gets UGI from HttpServletRequest in a consistent way | Major | timelineserver | Vrushali C | Abhishek Modi | -| [YARN-3879](https://issues.apache.org/jira/browse/YARN-3879) | [Storage implementation] Create HDFS backing storage implementation for ATS reads | Major | timelineserver | Tsuyoshi Ozawa | Abhishek Modi | -| [HADOOP-15837](https://issues.apache.org/jira/browse/HADOOP-15837) | DynamoDB table Update can fail S3A FS init | Major | fs/s3 | Steve Loughran | Steve Loughran | -| [HADOOP-15607](https://issues.apache.org/jira/browse/HADOOP-15607) | AliyunOSS: fix duplicated partNumber issue in AliyunOSSBlockOutputStream | Critical | . | wujinhu | wujinhu | -| [HADOOP-15868](https://issues.apache.org/jira/browse/HADOOP-15868) | AliyunOSS: update document for properties of multiple part download, multiple part upload and directory copy | Major | fs/oss | wujinhu | wujinhu | -| [YARN-8893](https://issues.apache.org/jira/browse/YARN-8893) | [AMRMProxy] Fix thread leak in AMRMClientRelayer and UAM client | Major | amrmproxy, federation | Botong Huang | Botong Huang | -| [YARN-8905](https://issues.apache.org/jira/browse/YARN-8905) | [Router] Add JvmMetricsInfo and pause monitor | Minor | . | Bibin Chundatt | Bilwa S T | -| [HADOOP-15917](https://issues.apache.org/jira/browse/HADOOP-15917) | AliyunOSS: fix incorrect ReadOps and WriteOps in statistics | Major | fs/oss | wujinhu | wujinhu | -| [HADOOP-16009](https://issues.apache.org/jira/browse/HADOOP-16009) | Replace the url of the repository in Apache Hadoop source code | Major | documentation | Akira Ajisaka | Akira Ajisaka | -| [HADOOP-15323](https://issues.apache.org/jira/browse/HADOOP-15323) | AliyunOSS: Improve copy file performance for AliyunOSSFileSystemStore | Major | fs/oss | wujinhu | wujinhu | -| [YARN-9182](https://issues.apache.org/jira/browse/YARN-9182) | Backport YARN-6445 resource profile performance improvements to branch-2 | Major | . | Jonathan Hung | Jonathan Hung | -| [YARN-9181](https://issues.apache.org/jira/browse/YARN-9181) | Backport YARN-6232 for generic resource type usage to branch-2 | Major | . | Jonathan Hung | Jonathan Hung | -| [YARN-9177](https://issues.apache.org/jira/browse/YARN-9177) | Use resource map for app metrics in TestCombinedSystemMetricsPublisher for branch-2 | Major | . | Jonathan Hung | Jonathan Hung | -| [YARN-9188](https://issues.apache.org/jira/browse/YARN-9188) | Port YARN-7136 to branch-2 | Major | . | Jonathan Hung | Jonathan Hung | -| [YARN-9187](https://issues.apache.org/jira/browse/YARN-9187) | Backport YARN-6852 for GPU-specific native changes to branch-2 | Major | . | Jonathan Hung | Jonathan Hung | -| [YARN-9180](https://issues.apache.org/jira/browse/YARN-9180) | Port YARN-7033 NM recovery of assigned resources to branch-3.0/branch-2 | Major | . | Jonathan Hung | Jonathan Hung | -| [YARN-9280](https://issues.apache.org/jira/browse/YARN-9280) | Backport YARN-6620 to YARN-8200/branch-2 | Major | . | Jonathan Hung | Jonathan Hung | -| [YARN-9174](https://issues.apache.org/jira/browse/YARN-9174) | Backport YARN-7224 for refactoring of GpuDevice class | Major | . | Jonathan Hung | Jonathan Hung | -| [YARN-9289](https://issues.apache.org/jira/browse/YARN-9289) | Backport YARN-7330 for GPU in UI to branch-2 | Major | . | Jonathan Hung | Jonathan Hung | -| [HDFS-14262](https://issues.apache.org/jira/browse/HDFS-14262) | [SBN read] Unclear Log.WARN message in GlobalStateIdContext | Major | hdfs | Shweta | Shweta | -| [YARN-8549](https://issues.apache.org/jira/browse/YARN-8549) | Adding a NoOp timeline writer and reader plugin classes for ATSv2 | Minor | ATSv2, timelineclient, timelineserver | Prabha Manepalli | Prabha Manepalli | -| [HADOOP-16109](https://issues.apache.org/jira/browse/HADOOP-16109) | Parquet reading S3AFileSystem causes EOF | Blocker | fs/s3 | Dave Christianson | Steve Loughran | -| [YARN-9397](https://issues.apache.org/jira/browse/YARN-9397) | Fix empty NMResourceInfo object test failures in branch-2 | Major | . | Jonathan Hung | Jonathan Hung | -| [HADOOP-16191](https://issues.apache.org/jira/browse/HADOOP-16191) | AliyunOSS: improvements for copyFile/copyDirectory and logging | Major | fs/oss | wujinhu | wujinhu | -| [YARN-9271](https://issues.apache.org/jira/browse/YARN-9271) | Backport YARN-6927 for resource type support in MapReduce | Major | . | Jonathan Hung | Jonathan Hung | -| [YARN-9291](https://issues.apache.org/jira/browse/YARN-9291) | Backport YARN-7637 to branch-2 | Major | . | Jonathan Hung | Jonathan Hung | -| [YARN-9409](https://issues.apache.org/jira/browse/YARN-9409) | Port resource type changes from YARN-7237 to branch-3.0/branch-2 | Major | . | Jonathan Hung | Jonathan Hung | -| [YARN-9272](https://issues.apache.org/jira/browse/YARN-9272) | Backport YARN-7738 for refreshing max allocation for multiple resource types | Major | . | Jonathan Hung | Jonathan Hung | -| [HADOOP-16205](https://issues.apache.org/jira/browse/HADOOP-16205) | Backporting ABFS driver from trunk to branch 2.0 | Major | fs/azure | Esfandiar Manii | Yuan Gao | -| [HADOOP-16269](https://issues.apache.org/jira/browse/HADOOP-16269) | ABFS: add listFileStatus with StartFrom | Major | fs/azure | Da Zhou | Da Zhou | -| [HADOOP-16306](https://issues.apache.org/jira/browse/HADOOP-16306) | AliyunOSS: Remove temporary files when upload small files to OSS | Major | fs/oss | wujinhu | wujinhu | -| [HDFS-14034](https://issues.apache.org/jira/browse/HDFS-14034) | Support getQuotaUsage API in WebHDFS | Major | fs, webhdfs | Erik Krogen | Chao Sun | -| [YARN-9775](https://issues.apache.org/jira/browse/YARN-9775) | RMWebServices /scheduler-conf GET returns all hadoop configurations for ZKConfigurationStore | Major | restapi | Prabhu Joseph | Prabhu Joseph | -| [HDFS-14771](https://issues.apache.org/jira/browse/HDFS-14771) | Backport HDFS-14617 to branch-2 (Improve fsimage load time by writing sub-sections to the fsimage index) | Major | namenode | Xiaoqiao He | Xiaoqiao He | -| [HDFS-14822](https://issues.apache.org/jira/browse/HDFS-14822) | [SBN read] Revisit GlobalStateIdContext locking when getting server state id | Major | hdfs | Chen Liang | Chen Liang | -| [HDFS-14785](https://issues.apache.org/jira/browse/HDFS-14785) | [SBN read] Change client logging to be less aggressive | Major | hdfs | Chen Liang | Chen Liang | -| [HDFS-14858](https://issues.apache.org/jira/browse/HDFS-14858) | [SBN read] Allow configurably enable/disable AlignmentContext on NameNode | Major | hdfs | Chen Liang | Chen Liang | -| [HDFS-12979](https://issues.apache.org/jira/browse/HDFS-12979) | StandbyNode should upload FsImage to ObserverNode after checkpointing. | Major | hdfs | Konstantin Shvachko | Chen Liang | -| [HADOOP-16630](https://issues.apache.org/jira/browse/HADOOP-16630) | Backport HADOOP-16548 - "ABFS: Config to enable/disable flush operation" to branch-2 | Minor | fs/azure | Sneha Vijayarajan | Sneha Vijayarajan | -| [HADOOP-16631](https://issues.apache.org/jira/browse/HADOOP-16631) | Backport HADOOP-16578 - "ABFS: fileSystemExists() should not call container level apis" to Branch-2 | Major | fs/azure | Sneha Vijayarajan | Sneha Vijayarajan | -| [HDFS-14162](https://issues.apache.org/jira/browse/HDFS-14162) | Balancer should work with ObserverNode | Major | . | Konstantin Shvachko | Erik Krogen | - - -### OTHER: - -| JIRA | Summary | Priority | Component | Reporter | Contributor | -|:---- |:---- | :--- |:---- |:---- |:---- | -| [HADOOP-15149](https://issues.apache.org/jira/browse/HADOOP-15149) | CryptoOutputStream should implement StreamCapabilities | Major | fs | Mike Drob | Xiao Chen | -| [HADOOP-15177](https://issues.apache.org/jira/browse/HADOOP-15177) | Update the release year to 2018 | Blocker | build | Akira Ajisaka | Bharat Viswanadham | -| [YARN-8412](https://issues.apache.org/jira/browse/YARN-8412) | Move ResourceRequest.clone logic everywhere into a proper API | Minor | . | Botong Huang | Botong Huang | -| [HDFS-13870](https://issues.apache.org/jira/browse/HDFS-13870) | WebHDFS: Document ALLOWSNAPSHOT and DISALLOWSNAPSHOT API doc | Minor | documentation, webhdfs | Siyao Meng | Siyao Meng | -| [HDFS-12729](https://issues.apache.org/jira/browse/HDFS-12729) | Document special paths in HDFS | Major | documentation | Christopher Douglas | Masatake Iwasaki | -| [HADOOP-15711](https://issues.apache.org/jira/browse/HADOOP-15711) | Move branch-2 precommit/nightly test builds to java 8 | Critical | . | Jonathan Hung | Jonathan Hung | -| [HDFS-14510](https://issues.apache.org/jira/browse/HDFS-14510) | Backport HDFS-13087 to branch-2 (Snapshotted encryption zone information should be immutable) | Major | encryption, snapshots | Wei-Chiu Chuang | Siyao Meng | -| [HDFS-14585](https://issues.apache.org/jira/browse/HDFS-14585) | Backport HDFS-8901 Use ByteBuffer in DFSInputStream#read to branch2.9 | Major | . | Lisheng Sun | Lisheng Sun | -| [HDFS-14483](https://issues.apache.org/jira/browse/HDFS-14483) | Backport HDFS-14111,HDFS-3246 ByteBuffer pread interface to branch-2.9 | Major | . | Zheng Hu | Lisheng Sun | -| [YARN-9559](https://issues.apache.org/jira/browse/YARN-9559) | Create AbstractContainersLauncher for pluggable ContainersLauncher logic | Major | . | Jonathan Hung | Jonathan Hung | -| [HDFS-14725](https://issues.apache.org/jira/browse/HDFS-14725) | Backport HDFS-12914 to branch-2 (Block report leases cause missing blocks until next report) | Major | namenode | Wei-Chiu Chuang | Xiaoqiao He | -| [YARN-8200](https://issues.apache.org/jira/browse/YARN-8200) | Backport resource types/GPU features to branch-3.0/branch-2 | Major | . | Jonathan Hung | Jonathan Hung | -| [HADOOP-16555](https://issues.apache.org/jira/browse/HADOOP-16555) | Update commons-compress to 1.19 | Major | . | Wei-Chiu Chuang | YiSheng Lien | -| [YARN-9730](https://issues.apache.org/jira/browse/YARN-9730) | Support forcing configured partitions to be exclusive based on app node label | Major | . | Jonathan Hung | Jonathan Hung | -| [HADOOP-16544](https://issues.apache.org/jira/browse/HADOOP-16544) | update io.netty in branch-2 | Major | . | Wei-Chiu Chuang | Masatake Iwasaki | -| [HADOOP-16588](https://issues.apache.org/jira/browse/HADOOP-16588) | Update commons-beanutils version to 1.9.4 in branch-2 | Critical | . | Wei-Chiu Chuang | Wei-Chiu Chuang | - - diff --git a/hadoop-common-project/hadoop-common/src/site/markdown/release/2.10.1/CHANGELOG.2.10.1.md b/hadoop-common-project/hadoop-common/src/site/markdown/release/2.10.1/CHANGELOG.2.10.1.md new file mode 100644 index 0000000000000..584bced722366 --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/site/markdown/release/2.10.1/CHANGELOG.2.10.1.md @@ -0,0 +1,288 @@ + + +# "Apache Hadoop" Changelog + +## Release 2.10.1 - 2020-09-14 + +### INCOMPATIBLE CHANGES: + +| JIRA | Summary | Priority | Component | Reporter | Contributor | +|:---- |:---- | :--- |:---- |:---- |:---- | + + +### IMPORTANT ISSUES: + +| JIRA | Summary | Priority | Component | Reporter | Contributor | +|:---- |:---- | :--- |:---- |:---- |:---- | + + +### NEW FEATURES: + +| JIRA | Summary | Priority | Component | Reporter | Contributor | +|:---- |:---- | :--- |:---- |:---- |:---- | + + +### IMPROVEMENTS: + +| JIRA | Summary | Priority | Component | Reporter | Contributor | +|:---- |:---- | :--- |:---- |:---- |:---- | +| [HADOOP-14597](https://issues.apache.org/jira/browse/HADOOP-14597) | Native compilation broken with OpenSSL-1.1.0 because EVP\_CIPHER\_CTX has been made opaque | Major | . | Ravi Prakash | Ravi Prakash | +| [MAPREDUCE-7069](https://issues.apache.org/jira/browse/MAPREDUCE-7069) | Add ability to specify user environment variables individually | Major | . | Jim Brennan | Jim Brennan | +| [HDFS-13511](https://issues.apache.org/jira/browse/HDFS-13511) | Provide specialized exception when block length cannot be obtained | Major | . | Ted Yu | Gabor Bota | +| [HDFS-10659](https://issues.apache.org/jira/browse/HDFS-10659) | Namenode crashes after Journalnode re-installation in an HA cluster due to missing paxos directory | Major | ha, journal-node | Amit Anand | star | +| [MAPREDUCE-7208](https://issues.apache.org/jira/browse/MAPREDUCE-7208) | Tuning TaskRuntimeEstimator | Minor | . | Ahmed Hussein | Ahmed Hussein | +| [HDFS-14979](https://issues.apache.org/jira/browse/HDFS-14979) | [Observer Node] Balancer should submit getBlocks to Observer Node when possible | Major | balancer & mover, hdfs | Erik Krogen | Erik Krogen | +| [HDFS-14991](https://issues.apache.org/jira/browse/HDFS-14991) | Backport better time precision of configuration#getTimeDuration to branch-2 to support SBN read | Major | hdfs | Chen Liang | Chen Liang | +| [HDFS-14952](https://issues.apache.org/jira/browse/HDFS-14952) | Skip safemode if blockTotal is 0 in new NN | Trivial | namenode | Rajesh Balamohan | Xiaoqiao He | +| [YARN-8842](https://issues.apache.org/jira/browse/YARN-8842) | Expose metrics for custom resource types in QueueMetrics | Major | . | Szilard Nemeth | Szilard Nemeth | +| [HADOOP-16735](https://issues.apache.org/jira/browse/HADOOP-16735) | Make it clearer in config default that EnvironmentVariableCredentialsProvider supports AWS\_SESSION\_TOKEN | Minor | documentation, fs/s3 | Mingliang Liu | Mingliang Liu | +| [YARN-10012](https://issues.apache.org/jira/browse/YARN-10012) | Guaranteed and max capacity queue metrics for custom resources | Major | . | Jonathan Hung | Manikandan R | +| [HDFS-15050](https://issues.apache.org/jira/browse/HDFS-15050) | Optimize log information when DFSInputStream meet CannotObtainBlockLengthException | Major | dfsclient | Xiaoqiao He | Xiaoqiao He | +| [YARN-10039](https://issues.apache.org/jira/browse/YARN-10039) | Allow disabling app submission from REST endpoints | Major | . | Jonathan Hung | Jonathan Hung | +| [YARN-9894](https://issues.apache.org/jira/browse/YARN-9894) | CapacitySchedulerPerf test for measuring hundreds of apps in a large number of queues. | Major | capacity scheduler, test | Eric Payne | Eric Payne | +| [YARN-10009](https://issues.apache.org/jira/browse/YARN-10009) | In Capacity Scheduler, DRC can treat minimum user limit percent as a max when custom resource is defined | Critical | capacity scheduler | Eric Payne | Eric Payne | +| [HADOOP-16753](https://issues.apache.org/jira/browse/HADOOP-16753) | Refactor HAAdmin | Major | ha | Akira Ajisaka | Xieming Li | +| [HDFS-14968](https://issues.apache.org/jira/browse/HDFS-14968) | Add ability to know datanode staleness | Minor | datanode, logging, namenode | Ahmed Hussein | Ahmed Hussein | +| [MAPREDUCE-7262](https://issues.apache.org/jira/browse/MAPREDUCE-7262) | MRApp helpers block for long intervals (500ms) | Minor | mr-am | Ahmed Hussein | Ahmed Hussein | +| [YARN-10084](https://issues.apache.org/jira/browse/YARN-10084) | Allow inheritance of max app lifetime / default app lifetime | Major | capacity scheduler | Eric Payne | Eric Payne | +| [HDFS-15046](https://issues.apache.org/jira/browse/HDFS-15046) | Backport HDFS-7060 to branch-2.10 | Major | datanode | Wei-Chiu Chuang | Lisheng Sun | +| [HDFS-12491](https://issues.apache.org/jira/browse/HDFS-12491) | Support wildcard in CLASSPATH for libhdfs | Major | libhdfs | John Zhuge | Muhammad Samir Khan | +| [YARN-10116](https://issues.apache.org/jira/browse/YARN-10116) | Expose diagnostics in RMAppManager summary | Major | . | Jonathan Hung | Jonathan Hung | +| [HADOOP-16739](https://issues.apache.org/jira/browse/HADOOP-16739) | Fix native build failure of hadoop-pipes on CentOS 8 | Major | tools/pipes | Masatake Iwasaki | Masatake Iwasaki | +| [HDFS-14758](https://issues.apache.org/jira/browse/HDFS-14758) | Decrease lease hard limit | Minor | . | Eric Payne | Hemanth Boyina | +| [YARN-9018](https://issues.apache.org/jira/browse/YARN-9018) | Add functionality to AuxiliaryLocalPathHandler to return all locations to read for a given path | Major | . | Kuhu Shukla | Kuhu Shukla | +| [HADOOP-16882](https://issues.apache.org/jira/browse/HADOOP-16882) | Update jackson-databind to 2.9.10.2 in branch-3.1, branch-2.10 | Blocker | . | Wei-Chiu Chuang | Lisheng Sun | +| [HADOOP-16776](https://issues.apache.org/jira/browse/HADOOP-16776) | backport HADOOP-16775: distcp copies to s3 are randomly corrupted | Blocker | tools/distcp | Amir Shenavandeh | Amir Shenavandeh | +| [HDFS-15197](https://issues.apache.org/jira/browse/HDFS-15197) | [SBN read] Change ObserverRetryOnActiveException log to debug | Minor | hdfs | Chen Liang | Chen Liang | +| [YARN-10200](https://issues.apache.org/jira/browse/YARN-10200) | Add number of containers to RMAppManager summary | Major | . | Jonathan Hung | Jonathan Hung | +| [YARN-8213](https://issues.apache.org/jira/browse/YARN-8213) | Add Capacity Scheduler performance metrics | Critical | capacityscheduler, metrics | Weiwei Yang | Weiwei Yang | +| [HADOOP-16952](https://issues.apache.org/jira/browse/HADOOP-16952) | Add .diff to gitignore | Minor | . | Ayush Saxena | Ayush Saxena | +| [YARN-10212](https://issues.apache.org/jira/browse/YARN-10212) | Create separate configuration for max global AM attempts | Major | . | Jonathan Hung | Bilwa S T | +| [HDFS-14476](https://issues.apache.org/jira/browse/HDFS-14476) | lock too long when fix inconsistent blocks between disk and in-memory | Major | datanode | Sean Chow | Sean Chow | +| [YARN-8680](https://issues.apache.org/jira/browse/YARN-8680) | YARN NM: Implement Iterable Abstraction for LocalResourceTracker state | Critical | yarn | Pradeep Ambati | Pradeep Ambati | +| [YARN-9954](https://issues.apache.org/jira/browse/YARN-9954) | Configurable max application tags and max tag length | Major | . | Jonathan Hung | Bilwa S T | +| [YARN-10260](https://issues.apache.org/jira/browse/YARN-10260) | Allow transitioning queue from DRAINING to RUNNING state | Major | . | Jonathan Hung | Bilwa S T | +| [HDFS-15264](https://issues.apache.org/jira/browse/HDFS-15264) | Backport Datanode detection to branch-2.10 | Major | . | Lisheng Sun | Lisheng Sun | +| [YARN-6492](https://issues.apache.org/jira/browse/YARN-6492) | Generate queue metrics for each partition | Major | capacity scheduler | Jonathan Hung | Manikandan R | +| [HADOOP-17047](https://issues.apache.org/jira/browse/HADOOP-17047) | TODO comments exist in trunk while the related issues are already fixed. | Trivial | . | Rungroj Maipradit | Rungroj Maipradit | +| [HADOOP-17090](https://issues.apache.org/jira/browse/HADOOP-17090) | Increase precommit job timeout from 5 hours to 20 hours | Major | build | Akira Ajisaka | Akira Ajisaka | +| [YARN-10297](https://issues.apache.org/jira/browse/YARN-10297) | TestContinuousScheduling#testFairSchedulerContinuousSchedulingInitTime fails intermittently | Major | . | Jonathan Hung | Jim Brennan | +| [HADOOP-17127](https://issues.apache.org/jira/browse/HADOOP-17127) | Use RpcMetrics.TIMEUNIT to initialize rpc queueTime and processingTime | Minor | common | Jim Brennan | Jim Brennan | +| [HDFS-15404](https://issues.apache.org/jira/browse/HDFS-15404) | ShellCommandFencer should expose info about source | Major | . | Chen Liang | Chen Liang | +| [YARN-10343](https://issues.apache.org/jira/browse/YARN-10343) | Legacy RM UI should include labeled metrics for allocated, total, and reserved resources. | Major | . | Eric Payne | Eric Payne | +| [YARN-1529](https://issues.apache.org/jira/browse/YARN-1529) | Add Localization overhead metrics to NM | Major | nodemanager | Gera Shegalov | Jim Brennan | +| [YARN-10251](https://issues.apache.org/jira/browse/YARN-10251) | Show extended resources on legacy RM UI. | Major | . | Eric Payne | Eric Payne | +| [HADOOP-17159](https://issues.apache.org/jira/browse/HADOOP-17159) | Make UGI support forceful relogin from keytab ignoring the last login time | Major | security | Sandeep Guggilam | Sandeep Guggilam | +| [HADOOP-17251](https://issues.apache.org/jira/browse/HADOOP-17251) | Upgrade netty-all to 4.1.50.Final on branch-2.10 | Major | . | Masatake Iwasaki | Masatake Iwasaki | +| [HADOOP-17249](https://issues.apache.org/jira/browse/HADOOP-17249) | Upgrade jackson-databind to 2.9.10.6 on branch-2.10 | Major | . | Masatake Iwasaki | Masatake Iwasaki | +| [HADOOP-17253](https://issues.apache.org/jira/browse/HADOOP-17253) | Upgrade zookeeper to 3.4.14 on branch-2.10 | Major | . | Masatake Iwasaki | Masatake Iwasaki | +| [HADOOP-17254](https://issues.apache.org/jira/browse/HADOOP-17254) | Upgrade hbase to 1.4.13 on branch-2.10 | Major | . | Masatake Iwasaki | Masatake Iwasaki | +| [HADOOP-16918](https://issues.apache.org/jira/browse/HADOOP-16918) | Dependency update for Hadoop 2.10 | Major | . | Wei-Chiu Chuang | | + + +### BUG FIXES: + +| JIRA | Summary | Priority | Component | Reporter | Contributor | +|:---- |:---- | :--- |:---- |:---- |:---- | +| [HDFS-9376](https://issues.apache.org/jira/browse/HDFS-9376) | TestSeveralNameNodes fails occasionally | Major | test | Kihwal Lee | Masatake Iwasaki | +| [MAPREDUCE-6881](https://issues.apache.org/jira/browse/MAPREDUCE-6881) | Fix warnings from Spotbugs in hadoop-mapreduce | Major | . | Weiwei Yang | Weiwei Yang | +| [HADOOP-14354](https://issues.apache.org/jira/browse/HADOOP-14354) | SysInfoWindows is not thread safe | Major | . | Íñigo Goiri | Íñigo Goiri | +| [HADOOP-14400](https://issues.apache.org/jira/browse/HADOOP-14400) | Fix warnings from spotbugs in hadoop-tools | Major | tools | Weiwei Yang | Weiwei Yang | +| [YARN-6515](https://issues.apache.org/jira/browse/YARN-6515) | Fix warnings from Spotbugs in hadoop-yarn-server-nodemanager | Major | nodemanager | Naganarasimha G R | Naganarasimha G R | +| [HDFS-11696](https://issues.apache.org/jira/browse/HDFS-11696) | Fix warnings from Spotbugs in hadoop-hdfs | Major | . | Yiqun Lin | Yiqun Lin | +| [HADOOP-14910](https://issues.apache.org/jira/browse/HADOOP-14910) | Upgrade netty-all jar to latest 4.0.x.Final | Critical | . | Vinayakumar B | Vinayakumar B | +| [YARN-7589](https://issues.apache.org/jira/browse/YARN-7589) | TestPBImplRecords fails with NullPointerException | Major | . | Jason Darrell Lowe | Daniel Templeton | +| [YARN-7739](https://issues.apache.org/jira/browse/YARN-7739) | DefaultAMSProcessor should properly check customized resource types against minimum/maximum allocation | Blocker | . | Wangda Tan | Wangda Tan | +| [YARN-8011](https://issues.apache.org/jira/browse/YARN-8011) | TestOpportunisticContainerAllocatorAMService#testContainerPromoteAndDemoteBeforeContainerStart fails sometimes in trunk | Minor | . | Tao Yang | Tao Yang | +| [HADOOP-15062](https://issues.apache.org/jira/browse/HADOOP-15062) | TestCryptoStreamsWithOpensslAesCtrCryptoCodec fails on Debian 9 | Major | . | Miklos Szegedi | Miklos Szegedi | +| [YARN-8004](https://issues.apache.org/jira/browse/YARN-8004) | Add unit tests for inter queue preemption for dominant resource calculator | Critical | yarn | Sumana Sathish | Zian Chen | +| [YARN-8210](https://issues.apache.org/jira/browse/YARN-8210) | AMRMClient logging on every heartbeat to track updation of AM RM token causes too many log lines to be generated in AM logs | Major | yarn | Suma Shivaprasad | Suma Shivaprasad | +| [YARN-8202](https://issues.apache.org/jira/browse/YARN-8202) | DefaultAMSProcessor should properly check units of requested custom resource types against minimum/maximum allocation | Blocker | . | Szilard Nemeth | Szilard Nemeth | +| [YARN-8179](https://issues.apache.org/jira/browse/YARN-8179) | Preemption does not happen due to natural\_termination\_factor when DRF is used | Major | . | kyungwan nam | kyungwan nam | +| [YARN-8369](https://issues.apache.org/jira/browse/YARN-8369) | Javadoc build failed due to "bad use of '\>'" | Critical | build, docs | Takanobu Asanuma | Takanobu Asanuma | +| [YARN-8382](https://issues.apache.org/jira/browse/YARN-8382) | cgroup file leak in NM | Major | nodemanager | Hu Ziqian | Hu Ziqian | +| [HDFS-12459](https://issues.apache.org/jira/browse/HDFS-12459) | Fix revert: Add new op GETFILEBLOCKLOCATIONS to WebHDFS REST API | Major | webhdfs | Weiwei Yang | Weiwei Yang | +| [HDFS-14054](https://issues.apache.org/jira/browse/HDFS-14054) | TestLeaseRecovery2: testHardLeaseRecoveryAfterNameNodeRestart2 and testHardLeaseRecoveryWithRenameAfterNameNodeRestart are flaky | Major | . | Zsolt Venczel | Zsolt Venczel | +| [YARN-9205](https://issues.apache.org/jira/browse/YARN-9205) | When using custom resource type, application will fail to run due to the CapacityScheduler throws InvalidResourceRequestException(GREATER\_THEN\_MAX\_ALLOCATION) | Critical | . | Zhankun Tang | Zhankun Tang | +| [YARN-4901](https://issues.apache.org/jira/browse/YARN-4901) | QueueMetrics needs to be cleared before MockRM is initialized | Major | scheduler | Daniel Templeton | Peter Bacsko | +| [HDFS-14647](https://issues.apache.org/jira/browse/HDFS-14647) | NPE during secure namenode startup | Major | hdfs | Fengnan Li | Fengnan Li | +| [HADOOP-16255](https://issues.apache.org/jira/browse/HADOOP-16255) | ChecksumFS.Make FileSystem.rename(path, path, options) doesn't rename checksum | Major | fs | Steve Loughran | Jungtaek Lim | +| [HADOOP-16580](https://issues.apache.org/jira/browse/HADOOP-16580) | Disable retry of FailoverOnNetworkExceptionRetry in case of AccessControlException | Major | common | Adam Antal | Adam Antal | +| [HADOOP-16662](https://issues.apache.org/jira/browse/HADOOP-16662) | Remove unnecessary InnerNode check in NetworkTopology#add() | Minor | . | Lisheng Sun | Lisheng Sun | +| [YARN-9915](https://issues.apache.org/jira/browse/YARN-9915) | Fix FindBug issue in QueueMetrics | Minor | . | Prabhu Joseph | Prabhu Joseph | +| [HDFS-12749](https://issues.apache.org/jira/browse/HDFS-12749) | DN may not send block report to NN after NN restart | Major | datanode | TanYuxin | Xiaoqiao He | +| [HDFS-13901](https://issues.apache.org/jira/browse/HDFS-13901) | INode access time is ignored because of race between open and rename | Major | . | Jinglun | Jinglun | +| [HDFS-14931](https://issues.apache.org/jira/browse/HDFS-14931) | hdfs crypto commands limit column width | Major | . | Eric Badger | Eric Badger | +| [YARN-9945](https://issues.apache.org/jira/browse/YARN-9945) | Fix javadoc in FederationProxyProviderUtil in branch-2 | Major | . | Jonathan Hung | Jonathan Hung | +| [HDFS-14806](https://issues.apache.org/jira/browse/HDFS-14806) | Bootstrap standby may fail if used in-progress tailing | Major | namenode | Chen Liang | Chen Liang | +| [HDFS-14941](https://issues.apache.org/jira/browse/HDFS-14941) | Potential editlog race condition can cause corrupted file | Major | namenode | Chen Liang | Chen Liang | +| [HDFS-14958](https://issues.apache.org/jira/browse/HDFS-14958) | TestBalancerWithNodeGroup is not using NetworkTopologyWithNodeGroup | Minor | hdfs | Jim Brennan | Jim Brennan | +| [HDFS-14884](https://issues.apache.org/jira/browse/HDFS-14884) | Add sanity check that zone key equals feinfo key while setting Xattrs | Major | encryption, hdfs | Mukul Kumar Singh | Mukul Kumar Singh | +| [HADOOP-15097](https://issues.apache.org/jira/browse/HADOOP-15097) | AbstractContractDeleteTest::testDeleteNonEmptyDirRecursive with misleading path | Minor | fs, test | zhoutai.zt | Xieming Li | +| [HADOOP-16700](https://issues.apache.org/jira/browse/HADOOP-16700) | RpcQueueTime may be negative when the response has to be sent later | Minor | . | xuzq | xuzq | +| [YARN-9838](https://issues.apache.org/jira/browse/YARN-9838) | Fix resource inconsistency for queues when moving app with reserved container to another queue | Critical | capacity scheduler | jiulongzhu | jiulongzhu | +| [HDFS-14973](https://issues.apache.org/jira/browse/HDFS-14973) | Balancer getBlocks RPC dispersal does not function properly | Major | balancer & mover | Erik Krogen | Erik Krogen | +| [MAPREDUCE-7240](https://issues.apache.org/jira/browse/MAPREDUCE-7240) | Exception ' Invalid event: TA\_TOO\_MANY\_FETCH\_FAILURE at SUCCESS\_FINISHING\_CONTAINER' cause job error | Critical | . | luhuachao | luhuachao | +| [HDFS-14986](https://issues.apache.org/jira/browse/HDFS-14986) | ReplicaCachingGetSpaceUsed throws ConcurrentModificationException | Major | datanode, performance | Ryan Wu | Aiphago | +| [MAPREDUCE-7249](https://issues.apache.org/jira/browse/MAPREDUCE-7249) | Invalid event TA\_TOO\_MANY\_FETCH\_FAILURE at SUCCESS\_CONTAINER\_CLEANUP causes job failure | Critical | applicationmaster, mrv2 | Wilfred Spiegelenburg | Wilfred Spiegelenburg | +| [HDFS-15005](https://issues.apache.org/jira/browse/HDFS-15005) | Backport HDFS-12300 to branch-2 | Major | hdfs | Chao Sun | Chao Sun | +| [HDFS-15017](https://issues.apache.org/jira/browse/HDFS-15017) | Remove redundant import of AtomicBoolean in NameNodeConnector. | Major | balancer & mover, hdfs | Konstantin Shvachko | Chao Sun | +| [HADOOP-16754](https://issues.apache.org/jira/browse/HADOOP-16754) | Fix docker failed to build yetus/hadoop | Blocker | build | Kevin Su | Kevin Su | +| [HDFS-15032](https://issues.apache.org/jira/browse/HDFS-15032) | Balancer crashes when it fails to contact an unavailable NN via ObserverReadProxyProvider | Major | balancer & mover | Erik Krogen | Erik Krogen | +| [HDFS-15036](https://issues.apache.org/jira/browse/HDFS-15036) | Active NameNode should not silently fail the image transfer | Major | namenode | Konstantin Shvachko | Chen Liang | +| [HDFS-15048](https://issues.apache.org/jira/browse/HDFS-15048) | Fix findbug in DirectoryScanner | Major | . | Takanobu Asanuma | Masatake Iwasaki | +| [HDFS-14519](https://issues.apache.org/jira/browse/HDFS-14519) | NameQuota is not update after concat operation, so namequota is wrong | Major | . | Ranith Sardar | Ranith Sardar | +| [HDFS-15076](https://issues.apache.org/jira/browse/HDFS-15076) | Fix tests that hold FSDirectory lock, without holding FSNamesystem lock. | Major | test | Konstantin Shvachko | Konstantin Shvachko | +| [HDFS-14934](https://issues.apache.org/jira/browse/HDFS-14934) | [SBN Read] Standby NN throws many InterruptedExceptions when dfs.ha.tail-edits.period is 0 | Major | . | Takanobu Asanuma | Ayush Saxena | +| [HADOOP-16789](https://issues.apache.org/jira/browse/HADOOP-16789) | In TestZKFailoverController, restore changes from HADOOP-11149 that were dropped by HDFS-6440 | Minor | common | Jim Brennan | Jim Brennan | +| [HDFS-15077](https://issues.apache.org/jira/browse/HDFS-15077) | Fix intermittent failure of TestDFSClientRetries#testLeaseRenewSocketTimeout | Minor | test | Masatake Iwasaki | Masatake Iwasaki | +| [YARN-7387](https://issues.apache.org/jira/browse/YARN-7387) | org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.TestIncreaseAllocationExpirer fails intermittently | Major | . | Miklos Szegedi | Jim Brennan | +| [YARN-8672](https://issues.apache.org/jira/browse/YARN-8672) | TestContainerManager#testLocalingResourceWhileContainerRunning occasionally times out | Major | nodemanager | Jason Darrell Lowe | Chandni Singh | +| [MAPREDUCE-7252](https://issues.apache.org/jira/browse/MAPREDUCE-7252) | Handling 0 progress in SimpleExponential task runtime estimator | Minor | . | Ahmed Hussein | Ahmed Hussein | +| [HADOOP-16749](https://issues.apache.org/jira/browse/HADOOP-16749) | Configuration parsing of CDATA values are blank | Major | conf | Jonathan Turner Eagles | Daryn Sharp | +| [HDFS-15099](https://issues.apache.org/jira/browse/HDFS-15099) | [SBN Read] checkOperation(WRITE) should throw ObserverRetryOnActiveException on ObserverNode | Major | namenode | Konstantin Shvachko | Chen Liang | +| [HADOOP-16683](https://issues.apache.org/jira/browse/HADOOP-16683) | Disable retry of FailoverOnNetworkExceptionRetry in case of wrapped AccessControlException | Major | common | Adam Antal | Adam Antal | +| [HDFS-13339](https://issues.apache.org/jira/browse/HDFS-13339) | Volume reference can't be released and may lead to deadlock when DataXceiver does a check volume | Critical | datanode | liaoyuxiangqin | Zsolt Venczel | +| [MAPREDUCE-7256](https://issues.apache.org/jira/browse/MAPREDUCE-7256) | Fix javadoc error in SimpleExponentialSmoothing | Minor | documentation | Masatake Iwasaki | Masatake Iwasaki | +| [MAPREDUCE-7247](https://issues.apache.org/jira/browse/MAPREDUCE-7247) | Modify HistoryServerRest.html content,change The job attempt id‘s datatype from string to int | Major | documentation | zhaoshengjie | zhaoshengjie | +| [HADOOP-16808](https://issues.apache.org/jira/browse/HADOOP-16808) | Use forkCount and reuseForks parameters instead of forkMode in the config of maven surefire plugin | Minor | build | Akira Ajisaka | Xieming Li | +| [HADOOP-16793](https://issues.apache.org/jira/browse/HADOOP-16793) | Remove WARN log when ipc connection interrupted in Client#handleSaslConnectionFailure() | Minor | . | Lisheng Sun | Lisheng Sun | +| [YARN-9790](https://issues.apache.org/jira/browse/YARN-9790) | Failed to set default-application-lifetime if maximum-application-lifetime is less than or equal to zero | Major | . | kyungwan nam | kyungwan nam | +| [HDFS-13179](https://issues.apache.org/jira/browse/HDFS-13179) | TestLazyPersistReplicaRecovery#testDnRestartWithSavedReplicas fails intermittently | Critical | fs | Gabor Bota | Ahmed Hussein | +| [MAPREDUCE-7259](https://issues.apache.org/jira/browse/MAPREDUCE-7259) | testSpeculateSuccessfulWithUpdateEvents fails Intermittently | Minor | . | Ahmed Hussein | Ahmed Hussein | +| [HDFS-15146](https://issues.apache.org/jira/browse/HDFS-15146) | TestBalancerRPCDelay.testBalancerRPCDelay fails intermittently | Minor | . | Ahmed Hussein | Ahmed Hussein | +| [MAPREDUCE-7079](https://issues.apache.org/jira/browse/MAPREDUCE-7079) | JobHistory#ServiceStop implementation is incorrect | Major | . | Jason Darrell Lowe | Ahmed Hussein | +| [HDFS-15118](https://issues.apache.org/jira/browse/HDFS-15118) | [SBN Read] Slow clients when Observer reads are enabled but there are no Observers on the cluster. | Major | hdfs-client | Konstantin Shvachko | Chen Liang | +| [HDFS-15148](https://issues.apache.org/jira/browse/HDFS-15148) | dfs.namenode.send.qop.enabled should not apply to primary NN port | Major | . | Chen Liang | Chen Liang | +| [YARN-8292](https://issues.apache.org/jira/browse/YARN-8292) | Fix the dominant resource preemption cannot happen when some of the resource vector becomes negative | Critical | yarn | Sumana Sathish | Wangda Tan | +| [HDFS-15115](https://issues.apache.org/jira/browse/HDFS-15115) | Namenode crash caused by NPE in BlockPlacementPolicyDefault when dynamically change logger to debug | Major | . | wangzhixiang | wangzhixiang | +| [HADOOP-16849](https://issues.apache.org/jira/browse/HADOOP-16849) | start-build-env.sh behaves incorrectly when username is numeric only | Minor | build | Jihyun Cho | Jihyun Cho | +| [HDFS-15161](https://issues.apache.org/jira/browse/HDFS-15161) | When evictableMmapped or evictable size is zero, do not throw NoSuchElementException in ShortCircuitCache#close() | Major | . | Lisheng Sun | Lisheng Sun | +| [HDFS-15164](https://issues.apache.org/jira/browse/HDFS-15164) | Fix TestDelegationTokensWithHA | Major | . | Ayush Saxena | Ayush Saxena | +| [HADOOP-16868](https://issues.apache.org/jira/browse/HADOOP-16868) | ipc.Server readAndProcess threw NullPointerException | Major | rpc-server | Tsz-wo Sze | Tsz-wo Sze | +| [HADOOP-16869](https://issues.apache.org/jira/browse/HADOOP-16869) | Upgrade findbugs-maven-plugin to 3.0.5 to fix mvn findbugs:findbugs failure | Major | build | Akira Ajisaka | Akira Ajisaka | +| [HDFS-15052](https://issues.apache.org/jira/browse/HDFS-15052) | WebHDFS getTrashRoot leads to OOM due to FileSystem object creation | Major | webhdfs | Wei-Chiu Chuang | Masatake Iwasaki | +| [HDFS-15185](https://issues.apache.org/jira/browse/HDFS-15185) | StartupProgress reports edits segments until the entire startup completes | Major | namenode | Konstantin Shvachko | Konstantin Shvachko | +| [HDFS-15166](https://issues.apache.org/jira/browse/HDFS-15166) | Remove redundant field fStream in ByteStringLog | Major | . | Konstantin Shvachko | Xieming Li | +| [YARN-10140](https://issues.apache.org/jira/browse/YARN-10140) | TestTimelineAuthFilterForV2 fails due to login failures in branch-2.10 | Major | timelineservice | Ahmed Hussein | Ahmed Hussein | +| [YARN-10156](https://issues.apache.org/jira/browse/YARN-10156) | Fix typo 'complaint' which means quite different in Federation.md | Minor | documentation, federation | Sungpeo Kook | Sungpeo Kook | +| [HDFS-15147](https://issues.apache.org/jira/browse/HDFS-15147) | LazyPersistTestCase wait logic is error-prone | Minor | . | Ahmed Hussein | Ahmed Hussein | +| [HADOOP-16891](https://issues.apache.org/jira/browse/HADOOP-16891) | Upgrade jackson-databind to 2.9.10.3 | Blocker | . | Siyao Meng | Siyao Meng | +| [HDFS-15204](https://issues.apache.org/jira/browse/HDFS-15204) | TestRetryCacheWithHA testRemoveCacheDescriptor fails intermittently | Major | hdfs | Ahmed Hussein | Ahmed Hussein | +| [HADOOP-16840](https://issues.apache.org/jira/browse/HADOOP-16840) | AliyunOSS: getFileStatus throws FileNotFoundException in versioning bucket | Major | fs/oss | wujinhu | wujinhu | +| [YARN-9427](https://issues.apache.org/jira/browse/YARN-9427) | TestContainerSchedulerQueuing.testKillOnlyRequiredOpportunisticContainers fails sporadically | Major | scheduler, test | Prabhu Joseph | Ahmed Hussein | +| [HDFS-11396](https://issues.apache.org/jira/browse/HDFS-11396) | TestNameNodeMetadataConsistency#testGenerationStampInFuture timed out | Minor | namenode, test | John Zhuge | Ayush Saxena | +| [HADOOP-16949](https://issues.apache.org/jira/browse/HADOOP-16949) | pylint fails in the build environment | Critical | build | Akira Ajisaka | Akira Ajisaka | +| [YARN-10227](https://issues.apache.org/jira/browse/YARN-10227) | Pull YARN-8242 back to branch-2.10 | Major | yarn | Jim Brennan | Jim Brennan | +| [YARN-2710](https://issues.apache.org/jira/browse/YARN-2710) | RM HA tests failed intermittently on trunk | Major | client | Wangda Tan | Ahmed Hussein | +| [MAPREDUCE-7272](https://issues.apache.org/jira/browse/MAPREDUCE-7272) | TaskAttemptListenerImpl excessive log messages | Major | . | Ahmed Hussein | Ahmed Hussein | +| [HDFS-15283](https://issues.apache.org/jira/browse/HDFS-15283) | Cache pool MAXTTL is not persisted and restored on cluster restart | Major | namenode | Stephen O'Donnell | Stephen O'Donnell | +| [HADOOP-16944](https://issues.apache.org/jira/browse/HADOOP-16944) | Use Yetus 0.12.0 in GitHub PR | Major | build | Akira Ajisaka | Akira Ajisaka | +| [HDFS-15276](https://issues.apache.org/jira/browse/HDFS-15276) | Concat on INodeRefernce fails with illegal state exception | Critical | . | Hemanth Boyina | Hemanth Boyina | +| [HADOOP-16361](https://issues.apache.org/jira/browse/HADOOP-16361) | TestSecureLogins#testValidKerberosName fails on branch-2 | Major | security | Jim Brennan | Jim Brennan | +| [HDFS-15281](https://issues.apache.org/jira/browse/HDFS-15281) | ZKFC ignores dfs.namenode.rpc-bind-host and uses dfs.namenode.rpc-address to bind to host address | Major | ha, namenode | Dhiraj Hegde | Dhiraj Hegde | +| [HDFS-15297](https://issues.apache.org/jira/browse/HDFS-15297) | TestNNHandlesBlockReportPerStorage::blockReport\_02 fails intermittently in trunk | Major | datanode, test | Mingliang Liu | Ayush Saxena | +| [YARN-8193](https://issues.apache.org/jira/browse/YARN-8193) | YARN RM hangs abruptly (stops allocating resources) when running successive applications. | Blocker | yarn | Zian Chen | Zian Chen | +| [YARN-10255](https://issues.apache.org/jira/browse/YARN-10255) | fix intermittent failure TestContainerSchedulerQueuing.testContainerUpdateExecTypeGuaranteedToOpportunistic in branch-2.10 | Major | . | Ahmed Hussein | Ahmed Hussein | +| [HDFS-15302](https://issues.apache.org/jira/browse/HDFS-15302) | Backport HDFS-15286 to branch-2.x | Blocker | . | Akira Ajisaka | Hemanth Boyina | +| [YARN-8959](https://issues.apache.org/jira/browse/YARN-8959) | TestContainerResizing fails randomly | Minor | . | Bibin Chundatt | Ahmed Hussein | +| [HDFS-15323](https://issues.apache.org/jira/browse/HDFS-15323) | StandbyNode fails transition to active due to insufficient transaction tailing | Major | namenode, qjm | Konstantin Shvachko | Konstantin Shvachko | +| [HADOOP-15565](https://issues.apache.org/jira/browse/HADOOP-15565) | ViewFileSystem.close doesn't close child filesystems and causes FileSystem objects leak. | Major | . | Jinglun | Jinglun | +| [YARN-9444](https://issues.apache.org/jira/browse/YARN-9444) | YARN API ResourceUtils's getRequestedResourcesFromConfig doesn't recognize yarn.io/gpu as a valid resource | Minor | api | Gergely Pollak | Gergely Pollak | +| [HDFS-15038](https://issues.apache.org/jira/browse/HDFS-15038) | TestFsck testFsckListCorruptSnapshotFiles is failing in trunk | Major | . | Hemanth Boyina | Hemanth Boyina | +| [HADOOP-17045](https://issues.apache.org/jira/browse/HADOOP-17045) | `Configuration` javadoc describes supporting environment variables, but the feature is not available | Minor | . | Nick Dimiduk | Masatake Iwasaki | +| [HDFS-15293](https://issues.apache.org/jira/browse/HDFS-15293) | Relax the condition for accepting a fsimage when receiving a checkpoint | Critical | namenode | Chen Liang | Chen Liang | +| [HADOOP-17052](https://issues.apache.org/jira/browse/HADOOP-17052) | NetUtils.connect() throws unchecked exception (UnresolvedAddressException) causing clients to abort | Major | net | Dhiraj Hegde | Dhiraj Hegde | +| [HADOOP-17062](https://issues.apache.org/jira/browse/HADOOP-17062) | Fix shelldocs path in Jenkinsfile | Major | build | Akira Ajisaka | Akira Ajisaka | +| [HDFS-15386](https://issues.apache.org/jira/browse/HDFS-15386) | ReplicaNotFoundException keeps happening in DN after removing multiple DN's data directories | Major | . | Toshihiro Suzuki | Toshihiro Suzuki | +| [YARN-10300](https://issues.apache.org/jira/browse/YARN-10300) | appMasterHost not set in RM ApplicationSummary when AM fails before first heartbeat | Major | . | Eric Badger | Eric Badger | +| [HADOOP-17059](https://issues.apache.org/jira/browse/HADOOP-17059) | ArrayIndexOfboundsException in ViewFileSystem#listStatus | Major | viewfs | Hemanth Boyina | Hemanth Boyina | +| [YARN-10312](https://issues.apache.org/jira/browse/YARN-10312) | Add support for yarn logs -logFile to retain backward compatibility | Major | client | Jim Brennan | Jim Brennan | +| [HDFS-12453](https://issues.apache.org/jira/browse/HDFS-12453) | TestDataNodeHotSwapVolumes fails in trunk Jenkins runs | Critical | test | Arpit Agarwal | Lei (Eddy) Xu | +| [HADOOP-17089](https://issues.apache.org/jira/browse/HADOOP-17089) | WASB: Update azure-storage-java SDK | Critical | fs/azure | Thomas Marqardt | Thomas Marqardt | +| [HDFS-15421](https://issues.apache.org/jira/browse/HDFS-15421) | IBR leak causes standby NN to be stuck in safe mode | Blocker | namenode | Kihwal Lee | Akira Ajisaka | +| [YARN-9903](https://issues.apache.org/jira/browse/YARN-9903) | Support reservations continue looking for Node Labels | Major | . | Tarun Parimi | Jim Brennan | +| [HADOOP-17094](https://issues.apache.org/jira/browse/HADOOP-17094) | vulnerabilities reported in jackson and jackson-databind in branch-2.10 | Major | . | Ahmed Hussein | Ahmed Hussein | +| [HADOOP-17120](https://issues.apache.org/jira/browse/HADOOP-17120) | Fix failure of docker image creation due to pip2 install error | Major | . | Masatake Iwasaki | Masatake Iwasaki | +| [YARN-10348](https://issues.apache.org/jira/browse/YARN-10348) | Allow RM to always cancel tokens after app completes | Major | yarn | Jim Brennan | Jim Brennan | +| [HDFS-14498](https://issues.apache.org/jira/browse/HDFS-14498) | LeaseManager can loop forever on the file for which create has failed | Major | namenode | Sergey Shelukhin | Stephen O'Donnell | +| [YARN-4771](https://issues.apache.org/jira/browse/YARN-4771) | Some containers can be skipped during log aggregation after NM restart | Major | nodemanager | Jason Darrell Lowe | Jim Brennan | +| [HDFS-15313](https://issues.apache.org/jira/browse/HDFS-15313) | Ensure inodes in active filesystem are not deleted during snapshot delete | Major | snapshots | Shashikant Banerjee | Shashikant Banerjee | +| [YARN-10363](https://issues.apache.org/jira/browse/YARN-10363) | TestRMAdminCLI.testHelp is failing in branch-2.10 | Major | yarn | Jim Brennan | Bilwa S T | +| [HADOOP-17184](https://issues.apache.org/jira/browse/HADOOP-17184) | Add --mvn-custom-repos parameter to yetus calls | Major | build | Mingliang Liu | Mingliang Liu | +| [HADOOP-17185](https://issues.apache.org/jira/browse/HADOOP-17185) | Set MAVEN\_OPTS in Jenkinsfile | Major | build | Akira Ajisaka | Akira Ajisaka | +| [HDFS-15499](https://issues.apache.org/jira/browse/HDFS-15499) | Clean up httpfs/pom.xml to remove aws-java-sdk-s3 exclusion | Major | httpfs | Mingliang Liu | Mingliang Liu | +| [HADOOP-17164](https://issues.apache.org/jira/browse/HADOOP-17164) | UGI loginUserFromKeytab doesn't set the last login time | Major | security | Sandeep Guggilam | Sandeep Guggilam | +| [YARN-4575](https://issues.apache.org/jira/browse/YARN-4575) | ApplicationResourceUsageReport should return ALL reserved resource | Major | . | Bibin Chundatt | Bibin Chundatt | +| [YARN-7677](https://issues.apache.org/jira/browse/YARN-7677) | Docker image cannot set HADOOP\_CONF\_DIR | Major | . | Eric Badger | Jim Brennan | +| [YARN-8459](https://issues.apache.org/jira/browse/YARN-8459) | Improve Capacity Scheduler logs to debug invalid states | Major | capacity scheduler | Wangda Tan | Wangda Tan | +| [MAPREDUCE-7286](https://issues.apache.org/jira/browse/MAPREDUCE-7286) | Fix findbugs warnings in hadoop-mapreduce-project on branch-2.10 | Minor | . | Masatake Iwasaki | Masatake Iwasaki | +| [HADOOP-17204](https://issues.apache.org/jira/browse/HADOOP-17204) | Fix typo in Hadoop KMS document | Trivial | documentation, kms | Akira Ajisaka | Xieming Li | +| [HADOOP-17202](https://issues.apache.org/jira/browse/HADOOP-17202) | Fix findbugs warnings in hadoop-tools on branch-2.10 | Minor | . | Masatake Iwasaki | Masatake Iwasaki | +| [HDFS-15290](https://issues.apache.org/jira/browse/HDFS-15290) | NPE in HttpServer during NameNode startup | Major | namenode | Konstantin Shvachko | Simbarashe Dzinamarira | +| [YARN-10358](https://issues.apache.org/jira/browse/YARN-10358) | Fix findbugs warnings in hadoop-yarn-project on branch-2.10 | Minor | . | Masatake Iwasaki | Masatake Iwasaki | +| [YARN-10177](https://issues.apache.org/jira/browse/YARN-10177) | Backport YARN-7307 to branch-2.10 Allow client/AM update supported resource types via YARN APIs | Blocker | . | Wei-Chiu Chuang | Wei-Chiu Chuang | + + +### TESTS: + +| JIRA | Summary | Priority | Component | Reporter | Contributor | +|:---- |:---- | :--- |:---- |:---- |:---- | +| [YARN-9404](https://issues.apache.org/jira/browse/YARN-9404) | TestApplicationLifetimeMonitor#testApplicationLifetimeMonitor fails intermittent | Major | resourcemanager | Prabhu Joseph | Prabhu Joseph | +| [YARN-10072](https://issues.apache.org/jira/browse/YARN-10072) | TestCSAllocateCustomResource failures | Major | yarn | Jim Brennan | Jim Brennan | +| [HDFS-15125](https://issues.apache.org/jira/browse/HDFS-15125) | Pull back HDFS-11353, HDFS-13993, HDFS-13945, and HDFS-14324 to branch-2.10 | Minor | hdfs | Jim Brennan | Jim Brennan | +| [YARN-10161](https://issues.apache.org/jira/browse/YARN-10161) | TestRouterWebServicesREST is corrupting STDOUT | Minor | yarn | Jim Brennan | Jim Brennan | +| [HADOOP-14206](https://issues.apache.org/jira/browse/HADOOP-14206) | TestSFTPFileSystem#testFileExists failure: Invalid encoding for signature | Major | fs, test | John Zhuge | Jim Brennan | +| [HADOOP-17205](https://issues.apache.org/jira/browse/HADOOP-17205) | Move personality file from Yetus to Hadoop repository | Major | test, yetus | Chao Sun | Chao Sun | + + +### SUB-TASKS: + +| JIRA | Summary | Priority | Component | Reporter | Contributor | +|:---- |:---- | :--- |:---- |:---- |:---- | +| [HADOOP-14056](https://issues.apache.org/jira/browse/HADOOP-14056) | Update maven-javadoc-plugin to 2.10.4 | Major | build | Akira Ajisaka | Akira Ajisaka | +| [HADOOP-12940](https://issues.apache.org/jira/browse/HADOOP-12940) | Fix warnings from Spotbugs in hadoop-common | Major | . | Akira Ajisaka | Akira Ajisaka | +| [YARN-7411](https://issues.apache.org/jira/browse/YARN-7411) | Inter-Queue preemption's computeFixpointAllocation need to handle absolute resources while computing normalizedGuarantee | Major | resourcemanager | Sunil G | Sunil G | +| [YARN-7541](https://issues.apache.org/jira/browse/YARN-7541) | Node updates don't update the maximum cluster capability for resources other than CPU and memory | Critical | resourcemanager | Daniel Templeton | Daniel Templeton | +| [HADOOP-15787](https://issues.apache.org/jira/browse/HADOOP-15787) | [JDK11] TestIPC.testRTEDuringConnectionSetup fails | Major | . | Akira Ajisaka | Zsolt Venczel | +| [HDFS-13404](https://issues.apache.org/jira/browse/HDFS-13404) | RBF: TestRouterWebHDFSContractAppend.testRenameFileBeingAppended fails | Major | test | Takanobu Asanuma | Takanobu Asanuma | +| [HDFS-14590](https://issues.apache.org/jira/browse/HDFS-14590) | [SBN Read] Add the document link to the top page | Major | documentation | Takanobu Asanuma | Takanobu Asanuma | +| [HADOOP-16652](https://issues.apache.org/jira/browse/HADOOP-16652) | Backport HADOOP-16587 - "Make AAD endpoint configurable on all Auth flows" to branch-2 | Minor | fs/azure | Bilahari T H | Bilahari T H | +| [YARN-9773](https://issues.apache.org/jira/browse/YARN-9773) | Add QueueMetrics for Custom Resources | Major | . | Manikandan R | Manikandan R | +| [HADOOP-16598](https://issues.apache.org/jira/browse/HADOOP-16598) | Backport "HADOOP-16558 [COMMON+HDFS] use protobuf-maven-plugin to generate protobuf classes" to all active branches | Major | common | Duo Zhang | Duo Zhang | +| [HDFS-14792](https://issues.apache.org/jira/browse/HDFS-14792) | [SBN read] StanbyNode does not come out of safemode while adding new blocks. | Major | namenode | Konstantin Shvachko | | +| [HADOOP-16610](https://issues.apache.org/jira/browse/HADOOP-16610) | Upgrade to yetus 0.11.1 and use emoji vote on github pre commit | Major | build | Duo Zhang | Duo Zhang | +| [HADOOP-16758](https://issues.apache.org/jira/browse/HADOOP-16758) | Refine testing.md to tell user better how to use auth-keys.xml | Minor | fs/s3 | Mingliang Liu | Mingliang Liu | +| [HADOOP-16609](https://issues.apache.org/jira/browse/HADOOP-16609) | Add Jenkinsfile for all active branches | Major | build | Duo Zhang | Akira Ajisaka | +| [HADOOP-16734](https://issues.apache.org/jira/browse/HADOOP-16734) | Backport HADOOP-16455- "ABFS: Implement FileSystem.access() method" to branch-2 | Minor | fs/azure | Bilahari T H | Bilahari T H | +| [HADOOP-16778](https://issues.apache.org/jira/browse/HADOOP-16778) | ABFS: Backport HADOOP-16660 ABFS: Make RetryCount in ExponentialRetryPolicy Configurable to Branch-2 | Major | fs/azure | Sneha Vijayarajan | Sneha Vijayarajan | +| [HADOOP-14918](https://issues.apache.org/jira/browse/HADOOP-14918) | Remove the Local Dynamo DB test option | Major | fs/s3 | Steve Loughran | Gabor Bota | +| [HADOOP-16933](https://issues.apache.org/jira/browse/HADOOP-16933) | Backport HADOOP-16890- "ABFS: Change in expiry calculation for MSI token provider" & HADOOP-16825 "ITestAzureBlobFileSystemCheckAccess failing" to branch-2 | Minor | fs/azure | Bilahari T H | Bilahari T H | +| [HDFS-10499](https://issues.apache.org/jira/browse/HDFS-10499) | TestNameNodeMetadataConsistency#testGenerationStampInFuture Fails Intermittently | Major | namenode, test | Hanisha Koneru | Yiqun Lin | +| [HADOOP-17199](https://issues.apache.org/jira/browse/HADOOP-17199) | Backport HADOOP-13230 list/getFileStatus changes for preserved directory markers | Major | fs/s3 | Steve Loughran | Steve Loughran | + + +### OTHER: + +| JIRA | Summary | Priority | Component | Reporter | Contributor | +|:---- |:---- | :--- |:---- |:---- |:---- | +| [HADOOP-16784](https://issues.apache.org/jira/browse/HADOOP-16784) | Update the year to 2020 | Major | . | Ayush Saxena | Ayush Saxena | +| [HADOOP-16647](https://issues.apache.org/jira/browse/HADOOP-16647) | Support OpenSSL 1.1.1 LTS | Critical | security | Wei-Chiu Chuang | Rakesh Radhakrishnan | + + diff --git a/hadoop-common-project/hadoop-common/src/site/markdown/release/2.10.1/RELEASENOTES.2.10.1.md b/hadoop-common-project/hadoop-common/src/site/markdown/release/2.10.1/RELEASENOTES.2.10.1.md new file mode 100644 index 0000000000000..73f8f1ea2f424 --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/site/markdown/release/2.10.1/RELEASENOTES.2.10.1.md @@ -0,0 +1,64 @@ + + +# "Apache Hadoop" 2.10.1 Release Notes + +These release notes cover new developer and user-facing incompatibilities, important issues, features, and major improvements. + + +--- + +* [MAPREDUCE-7069](https://issues.apache.org/jira/browse/MAPREDUCE-7069) | *Major* | **Add ability to specify user environment variables individually** + +Environment variables for MapReduce tasks can now be specified as separate properties, e.g.: +mapreduce.map.env.VARNAME=value +mapreduce.reduce.env.VARNAME=value +yarn.app.mapreduce.am.env.VARNAME=value +yarn.app.mapreduce.am.admin.user.env.VARNAME=value +This form of specifying environment variables is useful when the value of an environment variable contains commas. + + +--- + +* [HDFS-15281](https://issues.apache.org/jira/browse/HDFS-15281) | *Major* | **ZKFC ignores dfs.namenode.rpc-bind-host and uses dfs.namenode.rpc-address to bind to host address** + +ZKFC binds host address to "dfs.namenode.servicerpc-bind-host", if configured. Otherwise, it binds to "dfs.namenode.rpc-bind-host". If neither of those is configured, ZKFC binds itself to NameNode RPC server address (effectively "dfs.namenode.rpc-address"). + + +--- + +* [HDFS-12453](https://issues.apache.org/jira/browse/HDFS-12453) | *Critical* | **TestDataNodeHotSwapVolumes fails in trunk Jenkins runs** + +Submitted version of patch for branch-2.10 + + +--- + +* [HADOOP-17089](https://issues.apache.org/jira/browse/HADOOP-17089) | *Critical* | **WASB: Update azure-storage-java SDK** + +Azure WASB bug fix that can cause list results to appear empty. + + +--- + +* [YARN-7677](https://issues.apache.org/jira/browse/YARN-7677) | *Major* | **Docker image cannot set HADOOP\_CONF\_DIR** + +The HADOOP\_CONF\_DIR environment variable is no longer unconditionally inherited by containers even if it does not appear in the nodemanager whitelist variables specified by the yarn.nodemanager.env-whitelist property. If the whitelist property has been modified from the default to not include HADOOP\_CONF\_DIR yet containers need it to be inherited from the nodemanager's environment then the whitelist settings need to be updated to include HADOOP\_CONF\_DIR. + + + diff --git a/hadoop-common-project/hadoop-common/src/site/markdown/release/3.1.2/CHANGES.3.1.2.md b/hadoop-common-project/hadoop-common/src/site/markdown/release/3.1.2/CHANGELOG.3.1.2.md similarity index 100% rename from hadoop-common-project/hadoop-common/src/site/markdown/release/3.1.2/CHANGES.3.1.2.md rename to hadoop-common-project/hadoop-common/src/site/markdown/release/3.1.2/CHANGELOG.3.1.2.md diff --git a/hadoop-common-project/hadoop-common/src/site/markdown/release/3.1.3/CHANGES.3.1.3.md b/hadoop-common-project/hadoop-common/src/site/markdown/release/3.1.3/CHANGELOG.3.1.3.md similarity index 100% rename from hadoop-common-project/hadoop-common/src/site/markdown/release/3.1.3/CHANGES.3.1.3.md rename to hadoop-common-project/hadoop-common/src/site/markdown/release/3.1.3/CHANGELOG.3.1.3.md diff --git a/hadoop-common-project/hadoop-common/src/site/markdown/release/3.1.4/CHANGELOG.3.1.4.md b/hadoop-common-project/hadoop-common/src/site/markdown/release/3.1.4/CHANGELOG.3.1.4.md new file mode 100644 index 0000000000000..ee3547ebfd2c8 --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/site/markdown/release/3.1.4/CHANGELOG.3.1.4.md @@ -0,0 +1,363 @@ + + +# Apache Hadoop Changelog + +## Release 3.1.4 - 2020-07-21 + + + +### NEW FEATURES: + +| JIRA | Summary | Priority | Component | Reporter | Contributor | +|:---- |:---- | :--- |:---- |:---- |:---- | +| [YARN-9761](https://issues.apache.org/jira/browse/YARN-9761) | Allow overriding application submissions based on server side configs | Major | . | Jonathan Hung | pralabhkumar | +| [YARN-9760](https://issues.apache.org/jira/browse/YARN-9760) | Support configuring application priorities on a workflow level | Major | . | Jonathan Hung | Varun Saxena | +| [HDFS-14745](https://issues.apache.org/jira/browse/HDFS-14745) | Backport HDFS persistent memory read cache support to branch-3.1 | Major | . | Feilong He | Feilong He | +| [HDFS-12943](https://issues.apache.org/jira/browse/HDFS-12943) | Consistent Reads from Standby Node | Major | hdfs | Konstantin Shvachko | Konstantin Shvachko | + + +### IMPROVEMENTS: + +| JIRA | Summary | Priority | Component | Reporter | Contributor | +|:---- |:---- | :--- |:---- |:---- |:---- | +| [YARN-8361](https://issues.apache.org/jira/browse/YARN-8361) | Change App Name Placement Rule to use App Name instead of App Id for configuration | Major | yarn | Zian Chen | Zian Chen | +| [YARN-8750](https://issues.apache.org/jira/browse/YARN-8750) | Refactor TestQueueMetrics | Minor | resourcemanager | Szilard Nemeth | Szilard Nemeth | +| [HADOOP-15849](https://issues.apache.org/jira/browse/HADOOP-15849) | Upgrade netty version to 3.10.6 | Major | . | Xiao Chen | Xiao Chen | +| [HDFS-14064](https://issues.apache.org/jira/browse/HDFS-14064) | WEBHDFS: Support Enable/Disable EC Policy | Major | erasure-coding, webhdfs | Ayush Saxena | Ayush Saxena | +| [HDFS-14113](https://issues.apache.org/jira/browse/HDFS-14113) | EC : Add Configuration to restrict UserDefined Policies | Major | erasure-coding | Ayush Saxena | Ayush Saxena | +| [HDFS-14124](https://issues.apache.org/jira/browse/HDFS-14124) | EC : Support EC Commands (set/get/unset EcPolicy) via WebHdfs | Major | erasure-coding, httpfs, webhdfs | Souryakanta Dwivedy | Ayush Saxena | +| [HDFS-14006](https://issues.apache.org/jira/browse/HDFS-14006) | Refactor name node to allow different token verification implementations | Major | . | CR Hota | CR Hota | +| [HADOOP-15909](https://issues.apache.org/jira/browse/HADOOP-15909) | KeyProvider class should implement Closeable | Major | kms | Kuhu Shukla | Kuhu Shukla | +| [HDFS-14187](https://issues.apache.org/jira/browse/HDFS-14187) | Make warning message more clear when there are not enough data nodes for EC write | Major | erasure-coding | Kitti Nanasi | Kitti Nanasi | +| [HDFS-14235](https://issues.apache.org/jira/browse/HDFS-14235) | Handle ArrayIndexOutOfBoundsException in DataNodeDiskMetrics#slowDiskDetectionDaemon | Major | . | Surendra Singh Lilhore | Ranith Sardar | +| [HADOOP-16126](https://issues.apache.org/jira/browse/HADOOP-16126) | ipc.Client.stop() may sleep too long to wait for all connections | Major | ipc | Tsz-wo Sze | Tsz-wo Sze | +| [HADOOP-16140](https://issues.apache.org/jira/browse/HADOOP-16140) | hadoop fs expunge to add -immediate option to purge trash immediately | Major | fs | Stephen O'Donnell | Stephen O'Donnell | +| [HADOOP-15014](https://issues.apache.org/jira/browse/HADOOP-15014) | KMS should log the IP address of the clients | Major | kms | Zsombor Gegesy | Zsombor Gegesy | +| [HDFS-14460](https://issues.apache.org/jira/browse/HDFS-14460) | DFSUtil#getNamenodeWebAddr should return HTTPS address based on policy configured | Major | . | CR Hota | CR Hota | +| [HDFS-14624](https://issues.apache.org/jira/browse/HDFS-14624) | When decommissioning a node, log remaining blocks to replicate periodically | Major | namenode | Stephen O'Donnell | Stephen O'Donnell | +| [HDFS-13693](https://issues.apache.org/jira/browse/HDFS-13693) | Remove unnecessary search in INodeDirectory.addChild during image loading | Major | namenode | zhouyingchao | Lisheng Sun | +| [HDFS-14313](https://issues.apache.org/jira/browse/HDFS-14313) | Get hdfs used space from FsDatasetImpl#volumeMap#ReplicaInfo in memory instead of df/du | Major | datanode, performance | Lisheng Sun | Lisheng Sun | +| [HDFS-14678](https://issues.apache.org/jira/browse/HDFS-14678) | Allow triggerBlockReport to a specific namenode | Major | datanode | Leon Gao | Leon Gao | +| [HDFS-14523](https://issues.apache.org/jira/browse/HDFS-14523) | Remove excess read lock for NetworkToplogy | Major | . | Wu Weiwei | Wu Weiwei | +| [HDFS-14497](https://issues.apache.org/jira/browse/HDFS-14497) | Write lock held by metasave impact following RPC processing | Major | namenode | Xiaoqiao He | Xiaoqiao He | +| [YARN-9756](https://issues.apache.org/jira/browse/YARN-9756) | Create metric that sums total memory/vcores preempted per round | Major | capacity scheduler | Eric Payne | Manikandan R | +| [YARN-9810](https://issues.apache.org/jira/browse/YARN-9810) | Add queue capacity/maxcapacity percentage metrics | Major | . | Jonathan Hung | Shubham Gupta | +| [HADOOP-16531](https://issues.apache.org/jira/browse/HADOOP-16531) | Log more detail for slow RPC | Major | . | Chen Zhang | Chen Zhang | +| [YARN-9763](https://issues.apache.org/jira/browse/YARN-9763) | Print application tags in application summary | Major | . | Jonathan Hung | Manoj Kumar | +| [YARN-9764](https://issues.apache.org/jira/browse/YARN-9764) | Print application submission context label in application summary | Major | . | Jonathan Hung | Manoj Kumar | +| [YARN-9824](https://issues.apache.org/jira/browse/YARN-9824) | Fall back to configured queue ordering policy class name | Major | . | Jonathan Hung | Jonathan Hung | +| [HADOOP-16069](https://issues.apache.org/jira/browse/HADOOP-16069) | Support configure ZK\_DTSM\_ZK\_KERBEROS\_PRINCIPAL in ZKDelegationTokenSecretManager using principal with Schema /\_HOST | Minor | common | luhuachao | luhuachao | +| [YARN-9762](https://issues.apache.org/jira/browse/YARN-9762) | Add submission context label to audit logs | Major | . | Jonathan Hung | Manoj Kumar | +| [HDFS-14850](https://issues.apache.org/jira/browse/HDFS-14850) | Optimize FileSystemAccessService#getFileSystemConfiguration | Major | httpfs, performance | Lisheng Sun | Lisheng Sun | +| [HDFS-14192](https://issues.apache.org/jira/browse/HDFS-14192) | Track missing DFS operations in Statistics and StorageStatistics | Major | . | Ayush Saxena | Ayush Saxena | +| [HADOOP-16625](https://issues.apache.org/jira/browse/HADOOP-16625) | Backport HADOOP-14624 to branch-3.1 | Major | . | Wei-Chiu Chuang | Wei-Chiu Chuang | +| [YARN-9356](https://issues.apache.org/jira/browse/YARN-9356) | Add more tests to ratio method in TestResourceCalculator | Major | . | Szilard Nemeth | Zoltan Siegl | +| [HDFS-14915](https://issues.apache.org/jira/browse/HDFS-14915) | Move Superuser Check Before Taking Lock For Encryption API | Major | . | Ayush Saxena | Ayush Saxena | +| [HDFS-14921](https://issues.apache.org/jira/browse/HDFS-14921) | Remove SuperUser Check in Setting Storage Policy in FileStatus During Listing | Major | . | Ayush Saxena | Ayush Saxena | +| [HDFS-14923](https://issues.apache.org/jira/browse/HDFS-14923) | Remove dead code from HealthMonitor | Minor | . | Fei Hui | Fei Hui | +| [YARN-9914](https://issues.apache.org/jira/browse/YARN-9914) | Use separate configs for free disk space checking for full and not-full disks | Minor | yarn | Jim Brennan | Jim Brennan | +| [MAPREDUCE-7208](https://issues.apache.org/jira/browse/MAPREDUCE-7208) | Tuning TaskRuntimeEstimator | Minor | . | Ahmed Hussein | Ahmed Hussein | +| [HDFS-14942](https://issues.apache.org/jira/browse/HDFS-14942) | Change Log Level to debug in JournalNodeSyncer#syncWithJournalAtIndex | Minor | . | Lisheng Sun | Lisheng Sun | +| [HDFS-14979](https://issues.apache.org/jira/browse/HDFS-14979) | [Observer Node] Balancer should submit getBlocks to Observer Node when possible | Major | balancer & mover, hdfs | Erik Krogen | Erik Krogen | +| [HADOOP-16705](https://issues.apache.org/jira/browse/HADOOP-16705) | MBeanInfoBuilder puts unnecessary memory pressure on the system with a debug log | Major | metrics | Lukas Majercak | Lukas Majercak | +| [HADOOP-16712](https://issues.apache.org/jira/browse/HADOOP-16712) | Config ha.failover-controller.active-standby-elector.zk.op.retries is not in core-default.xml | Trivial | . | Wei-Chiu Chuang | Xieming Li | +| [HADOOP-16703](https://issues.apache.org/jira/browse/HADOOP-16703) | Backport HADOOP-16152 to branch-3.1 | Major | . | Siyao Meng | Siyao Meng | +| [HDFS-14952](https://issues.apache.org/jira/browse/HDFS-14952) | Skip safemode if blockTotal is 0 in new NN | Trivial | namenode | Rajesh Balamohan | Xiaoqiao He | +| [YARN-8842](https://issues.apache.org/jira/browse/YARN-8842) | Expose metrics for custom resource types in QueueMetrics | Major | . | Szilard Nemeth | Szilard Nemeth | +| [YARN-9966](https://issues.apache.org/jira/browse/YARN-9966) | Code duplication in UserGroupMappingPlacementRule | Major | . | Szilard Nemeth | Kevin Su | +| [YARN-9937](https://issues.apache.org/jira/browse/YARN-9937) | Add missing queue configs in RMWebService#CapacitySchedulerQueueInfo | Major | capacity scheduler | Prabhu Joseph | Prabhu Joseph | +| [HADOOP-16718](https://issues.apache.org/jira/browse/HADOOP-16718) | Allow disabling Server Name Indication (SNI) for Jetty | Major | . | Siyao Meng | Aravindan Vijayan | +| [HADOOP-16735](https://issues.apache.org/jira/browse/HADOOP-16735) | Make it clearer in config default that EnvironmentVariableCredentialsProvider supports AWS\_SESSION\_TOKEN | Minor | documentation, fs/s3 | Mingliang Liu | Mingliang Liu | +| [YARN-10012](https://issues.apache.org/jira/browse/YARN-10012) | Guaranteed and max capacity queue metrics for custom resources | Major | . | Jonathan Hung | Manikandan R | +| [HDFS-15050](https://issues.apache.org/jira/browse/HDFS-15050) | Optimize log information when DFSInputStream meet CannotObtainBlockLengthException | Major | dfsclient | Xiaoqiao He | Xiaoqiao He | +| [YARN-10033](https://issues.apache.org/jira/browse/YARN-10033) | TestProportionalCapacityPreemptionPolicy not initializing vcores for effective max resources | Major | capacity scheduler, test | Eric Payne | Eric Payne | +| [YARN-10039](https://issues.apache.org/jira/browse/YARN-10039) | Allow disabling app submission from REST endpoints | Major | . | Jonathan Hung | Jonathan Hung | +| [YARN-9894](https://issues.apache.org/jira/browse/YARN-9894) | CapacitySchedulerPerf test for measuring hundreds of apps in a large number of queues. | Major | capacity scheduler, test | Eric Payne | Eric Payne | +| [HADOOP-16771](https://issues.apache.org/jira/browse/HADOOP-16771) | Update checkstyle to 8.26 and maven-checkstyle-plugin to 3.1.0 | Major | build | Andras Bokor | Andras Bokor | +| [YARN-10009](https://issues.apache.org/jira/browse/YARN-10009) | In Capacity Scheduler, DRC can treat minimum user limit percent as a max when custom resource is defined | Critical | capacity scheduler | Eric Payne | Eric Payne | +| [HDFS-12999](https://issues.apache.org/jira/browse/HDFS-12999) | When reach the end of the block group, it may not need to flush all the data packets(flushAllInternals) twice. | Major | erasure-coding, hdfs-client | lufei | lufei | +| [HDFS-15074](https://issues.apache.org/jira/browse/HDFS-15074) | DataNode.DataTransfer thread should catch all the expception and log it. | Major | datanode | Surendra Singh Lilhore | Hemanth Boyina | +| [HDFS-14740](https://issues.apache.org/jira/browse/HDFS-14740) | Recover data blocks from persistent memory read cache during datanode restarts | Major | caching, datanode | Feilong He | Feilong He | +| [HDFS-15097](https://issues.apache.org/jira/browse/HDFS-15097) | Purge log in KMS and HttpFS | Minor | httpfs, kms | Doris Gu | Doris Gu | +| [HDFS-14968](https://issues.apache.org/jira/browse/HDFS-14968) | Add ability to know datanode staleness | Minor | datanode, logging, namenode | Ahmed Hussein | Ahmed Hussein | +| [YARN-7913](https://issues.apache.org/jira/browse/YARN-7913) | Improve error handling when application recovery fails with exception | Major | resourcemanager | Gergo Repas | Wilfred Spiegelenburg | +| [HDFS-15119](https://issues.apache.org/jira/browse/HDFS-15119) | Allow expiration of cached locations in DFSInputStream | Minor | dfsclient | Ahmed Hussein | Ahmed Hussein | +| [MAPREDUCE-7262](https://issues.apache.org/jira/browse/MAPREDUCE-7262) | MRApp helpers block for long intervals (500ms) | Minor | mr-am | Ahmed Hussein | Ahmed Hussein | +| [YARN-10084](https://issues.apache.org/jira/browse/YARN-10084) | Allow inheritance of max app lifetime / default app lifetime | Major | capacity scheduler | Eric Payne | Eric Payne | +| [HDFS-12491](https://issues.apache.org/jira/browse/HDFS-12491) | Support wildcard in CLASSPATH for libhdfs | Major | libhdfs | John Zhuge | Muhammad Samir Khan | +| [YARN-10116](https://issues.apache.org/jira/browse/YARN-10116) | Expose diagnostics in RMAppManager summary | Major | . | Jonathan Hung | Jonathan Hung | +| [HDFS-14758](https://issues.apache.org/jira/browse/HDFS-14758) | Decrease lease hard limit | Minor | . | Eric Payne | Hemanth Boyina | +| [HDFS-15086](https://issues.apache.org/jira/browse/HDFS-15086) | Block scheduled counter never get decremet if the block got deleted before replication. | Major | 3.1.1 | Surendra Singh Lilhore | Hemanth Boyina | +| [HDFS-15174](https://issues.apache.org/jira/browse/HDFS-15174) | Optimize ReplicaCachingGetSpaceUsed by reducing unnecessary io operations | Major | . | Lisheng Sun | Lisheng Sun | +| [YARN-9018](https://issues.apache.org/jira/browse/YARN-9018) | Add functionality to AuxiliaryLocalPathHandler to return all locations to read for a given path | Major | . | Kuhu Shukla | Kuhu Shukla | +| [HDFS-14861](https://issues.apache.org/jira/browse/HDFS-14861) | Reset LowRedundancyBlocks Iterator periodically | Major | namenode | Stephen O'Donnell | Stephen O'Donnell | +| [HADOOP-16882](https://issues.apache.org/jira/browse/HADOOP-16882) | Update jackson-databind to 2.9.10.2 in branch-3.1, branch-2.10 | Blocker | . | Wei-Chiu Chuang | Lisheng Sun | +| [HADOOP-16776](https://issues.apache.org/jira/browse/HADOOP-16776) | backport HADOOP-16775: distcp copies to s3 are randomly corrupted | Blocker | tools/distcp | Amir Shenavandeh | Amir Shenavandeh | +| [HDFS-15197](https://issues.apache.org/jira/browse/HDFS-15197) | [SBN read] Change ObserverRetryOnActiveException log to debug | Minor | hdfs | Chen Liang | Chen Liang | +| [YARN-10200](https://issues.apache.org/jira/browse/YARN-10200) | Add number of containers to RMAppManager summary | Major | . | Jonathan Hung | Jonathan Hung | +| [HADOOP-16952](https://issues.apache.org/jira/browse/HADOOP-16952) | Add .diff to gitignore | Minor | . | Ayush Saxena | Ayush Saxena | +| [YARN-10212](https://issues.apache.org/jira/browse/YARN-10212) | Create separate configuration for max global AM attempts | Major | . | Jonathan Hung | Bilwa S T | +| [YARN-9954](https://issues.apache.org/jira/browse/YARN-9954) | Configurable max application tags and max tag length | Major | . | Jonathan Hung | Bilwa S T | +| [HADOOP-17001](https://issues.apache.org/jira/browse/HADOOP-17001) | The suffix name of the unified compression class | Major | io | bianqi | bianqi | +| [HDFS-15295](https://issues.apache.org/jira/browse/HDFS-15295) | AvailableSpaceBlockPlacementPolicy should use chooseRandomWithStorageTypeTwoTrial() for better performance. | Minor | . | Jinglun | Jinglun | +| [HADOOP-17127](https://issues.apache.org/jira/browse/HADOOP-17127) | Use RpcMetrics.TIMEUNIT to initialize rpc queueTime and processingTime | Minor | common | Jim Brennan | Jim Brennan | + + +### BUG FIXES: + +| JIRA | Summary | Priority | Component | Reporter | Contributor | +|:---- |:---- | :--- |:---- |:---- |:---- | +| [HDFS-13806](https://issues.apache.org/jira/browse/HDFS-13806) | EC: No error message for unsetting EC policy of the directory inherits the erasure coding policy from an ancestor directory | Minor | erasure-coding | Souryakanta Dwivedy | Ayush Saxena | +| [HDFS-12459](https://issues.apache.org/jira/browse/HDFS-12459) | Fix revert: Add new op GETFILEBLOCKLOCATIONS to WebHDFS REST API | Major | webhdfs | Weiwei Yang | Weiwei Yang | +| [HADOOP-15418](https://issues.apache.org/jira/browse/HADOOP-15418) | Hadoop KMSAuthenticationFilter needs to use getPropsByPrefix instead of iterator to avoid ConcurrentModificationException | Major | common | Suma Shivaprasad | Suma Shivaprasad | +| [HDFS-14004](https://issues.apache.org/jira/browse/HDFS-14004) | TestLeaseRecovery2#testCloseWhileRecoverLease fails intermittently in trunk | Major | . | Ayush Saxena | Ayush Saxena | +| [HDFS-13959](https://issues.apache.org/jira/browse/HDFS-13959) | TestUpgradeDomainBlockPlacementPolicy is flaky | Major | . | Ayush Saxena | Ayush Saxena | +| [YARN-8948](https://issues.apache.org/jira/browse/YARN-8948) | PlacementRule interface should be for all YarnSchedulers | Major | . | Bibin Chundatt | Bibin Chundatt | +| [HADOOP-16013](https://issues.apache.org/jira/browse/HADOOP-16013) | DecayRpcScheduler decay thread should run as a daemon | Major | ipc | Erik Krogen | Erik Krogen | +| [HDFS-14175](https://issues.apache.org/jira/browse/HDFS-14175) | EC: Native XOR decoder should reset the output buffer before using it. | Major | ec, hdfs | Surendra Singh Lilhore | Ayush Saxena | +| [HDFS-14202](https://issues.apache.org/jira/browse/HDFS-14202) | "dfs.disk.balancer.max.disk.throughputInMBperSec" property is not working as per set value. | Major | diskbalancer | Ranith Sardar | Ranith Sardar | +| [HADOOP-16032](https://issues.apache.org/jira/browse/HADOOP-16032) | Distcp It should clear sub directory ACL before applying new ACL on it. | Major | tools/distcp | Ranith Sardar | Ranith Sardar | +| [HADOOP-16127](https://issues.apache.org/jira/browse/HADOOP-16127) | In ipc.Client, put a new connection could happen after stop | Major | ipc | Tsz-wo Sze | Tsz-wo Sze | +| [YARN-4901](https://issues.apache.org/jira/browse/YARN-4901) | QueueMetrics needs to be cleared before MockRM is initialized | Major | scheduler | Daniel Templeton | Peter Bacsko | +| [HADOOP-16161](https://issues.apache.org/jira/browse/HADOOP-16161) | NetworkTopology#getWeightUsingNetworkLocation return unexpected result | Major | net | Xiaoqiao He | Xiaoqiao He | +| [HDFS-14434](https://issues.apache.org/jira/browse/HDFS-14434) | webhdfs that connect secure hdfs should not use user.name parameter | Minor | webhdfs | KWON BYUNGCHANG | KWON BYUNGCHANG | +| [HDFS-14527](https://issues.apache.org/jira/browse/HDFS-14527) | Stop all DataNodes may result in NN terminate | Major | namenode | Xiaoqiao He | Xiaoqiao He | +| [HDFS-14494](https://issues.apache.org/jira/browse/HDFS-14494) | Move Server logging of StatedId inside receiveRequestState() | Major | . | Konstantin Shvachko | Shweta | +| [HDFS-14618](https://issues.apache.org/jira/browse/HDFS-14618) | Incorrect synchronization of ArrayList field (ArrayList is thread-unsafe). | Critical | . | Paul Ward | Paul Ward | +| [HDFS-14610](https://issues.apache.org/jira/browse/HDFS-14610) | HashMap is not thread safe. Field storageMap is typically synchronized by storageMap. However, in one place, field storageMap is not protected with synchronized. | Critical | . | Paul Ward | Paul Ward | +| [HDFS-14499](https://issues.apache.org/jira/browse/HDFS-14499) | Misleading REM\_QUOTA value with snapshot and trash feature enabled for a directory | Major | snapshots | Shashikant Banerjee | Shashikant Banerjee | +| [HDFS-14647](https://issues.apache.org/jira/browse/HDFS-14647) | NPE during secure namenode startup | Major | hdfs | Fengnan Li | Fengnan Li | +| [HADOOP-16461](https://issues.apache.org/jira/browse/HADOOP-16461) | Regression: FileSystem cache lock parses XML within the lock | Major | fs | Gopal Vijayaraghavan | Gopal Vijayaraghavan | +| [HDFS-14660](https://issues.apache.org/jira/browse/HDFS-14660) | [SBN Read] ObserverNameNode should throw StandbyException for requests not from ObserverProxyProvider | Major | . | Chao Sun | Chao Sun | +| [HDFS-14569](https://issues.apache.org/jira/browse/HDFS-14569) | Result of crypto -listZones is not formatted properly | Major | . | Hemanth Boyina | Hemanth Boyina | +| [HADOOP-12282](https://issues.apache.org/jira/browse/HADOOP-12282) | Connection thread's name should be updated after address changing is detected | Major | ipc | zhouyingchao | Lisheng Sun | +| [HDFS-14686](https://issues.apache.org/jira/browse/HDFS-14686) | HttpFS: HttpFSFileSystem#getErasureCodingPolicy always returns null | Major | httpfs | Siyao Meng | Siyao Meng | +| [HADOOP-15865](https://issues.apache.org/jira/browse/HADOOP-15865) | ConcurrentModificationException in Configuration.overlay() method | Major | . | Oleksandr Shevchenko | Oleksandr Shevchenko | +| [HDFS-14759](https://issues.apache.org/jira/browse/HDFS-14759) | HDFS cat logs an info message | Major | . | Eric Badger | Eric Badger | +| [HDFS-2470](https://issues.apache.org/jira/browse/HDFS-2470) | NN should automatically set permissions on dfs.namenode.\*.dir | Major | namenode | Aaron Myers | Siddharth Wagle | +| [YARN-9718](https://issues.apache.org/jira/browse/YARN-9718) | Yarn REST API, services endpoint remote command ejection | Major | . | Eric Yang | Eric Yang | +| [YARN-9813](https://issues.apache.org/jira/browse/YARN-9813) | RM does not start on JDK11 when UIv2 is enabled | Critical | resourcemanager, yarn | Adam Antal | Adam Antal | +| [YARN-9820](https://issues.apache.org/jira/browse/YARN-9820) | RM logs InvalidStateTransitionException when app is submitted | Critical | . | Rohith Sharma K S | Prabhu Joseph | +| [HDFS-14838](https://issues.apache.org/jira/browse/HDFS-14838) | RBF: Display RPC (instead of HTTP) Port Number in RBF web UI | Minor | rbf, ui | Xieming Li | Xieming Li | +| [HDFS-14699](https://issues.apache.org/jira/browse/HDFS-14699) | Erasure Coding: Storage not considered in live replica when replication streams hard limit reached to threshold | Critical | ec | Zhao Yi Ming | Zhao Yi Ming | +| [YARN-9833](https://issues.apache.org/jira/browse/YARN-9833) | Race condition when DirectoryCollection.checkDirs() runs during container launch | Major | . | Peter Bacsko | Peter Bacsko | +| [YARN-9837](https://issues.apache.org/jira/browse/YARN-9837) | YARN Service fails to fetch status for Stopped apps with bigger spec files | Major | yarn-native-services | Tarun Parimi | Tarun Parimi | +| [YARN-2255](https://issues.apache.org/jira/browse/YARN-2255) | YARN Audit logging not added to log4j.properties | Major | . | Varun Saxena | Aihua Xu | +| [HDFS-14836](https://issues.apache.org/jira/browse/HDFS-14836) | FileIoProvider should not increase FileIoErrors metric in datanode volume metric | Minor | . | Aiphago | Aiphago | +| [HADOOP-16581](https://issues.apache.org/jira/browse/HADOOP-16581) | ValueQueue does not trigger an async refill when number of values falls below watermark | Major | common, kms | Yuval Degani | Yuval Degani | +| [HDFS-14853](https://issues.apache.org/jira/browse/HDFS-14853) | NPE in DFSNetworkTopology#chooseRandomWithStorageType() when the excludedNode is not present | Major | . | Ranith Sardar | Ranith Sardar | +| [HDFS-13660](https://issues.apache.org/jira/browse/HDFS-13660) | DistCp job fails when new data is appended in the file while the distCp copy job is running | Critical | distcp | Mukund Thakur | Mukund Thakur | +| [HDFS-14808](https://issues.apache.org/jira/browse/HDFS-14808) | EC: Improper size values for corrupt ec block in LOG | Major | ec | Harshakiran Reddy | Ayush Saxena | +| [HDFS-14849](https://issues.apache.org/jira/browse/HDFS-14849) | Erasure Coding: the internal block is replicated many times when datanode is decommissioning | Major | ec, erasure-coding | HuangTao | HuangTao | +| [YARN-9858](https://issues.apache.org/jira/browse/YARN-9858) | Optimize RMContext getExclusiveEnforcedPartitions | Major | . | Jonathan Hung | Jonathan Hung | +| [HDFS-14492](https://issues.apache.org/jira/browse/HDFS-14492) | Snapshot memory leak | Major | snapshots | Wei-Chiu Chuang | Wei-Chiu Chuang | +| [HDFS-14418](https://issues.apache.org/jira/browse/HDFS-14418) | Remove redundant super user priveledge checks from namenode. | Major | . | Ayush Saxena | Ayush Saxena | +| [HDFS-14216](https://issues.apache.org/jira/browse/HDFS-14216) | NullPointerException happens in NamenodeWebHdfs | Critical | . | lujie | lujie | +| [HDFS-14637](https://issues.apache.org/jira/browse/HDFS-14637) | Namenode may not replicate blocks to meet the policy after enabling upgradeDomain | Major | namenode | Stephen O'Donnell | Stephen O'Donnell | +| [HDFS-14879](https://issues.apache.org/jira/browse/HDFS-14879) | Header was wrong in Snapshot web UI | Major | . | Hemanth Boyina | Hemanth Boyina | +| [HDFS-14655](https://issues.apache.org/jira/browse/HDFS-14655) | [SBN Read] Namenode crashes if one of The JN is down | Critical | . | Harshakiran Reddy | Ayush Saxena | +| [HDFS-14859](https://issues.apache.org/jira/browse/HDFS-14859) | Prevent unnecessary evaluation of costly operation getNumLiveDataNodes when dfs.namenode.safemode.min.datanodes is not zero | Major | hdfs | Srinivasu Majeti | Srinivasu Majeti | +| [YARN-6715](https://issues.apache.org/jira/browse/YARN-6715) | Fix documentation about NodeHealthScriptRunner | Major | documentation, nodemanager | Peter Bacsko | Peter Bacsko | +| [YARN-9552](https://issues.apache.org/jira/browse/YARN-9552) | FairScheduler: NODE\_UPDATE can cause NoSuchElementException | Major | fairscheduler | Peter Bacsko | Peter Bacsko | +| [HDFS-14754](https://issues.apache.org/jira/browse/HDFS-14754) | Erasure Coding : The number of Under-Replicated Blocks never reduced | Critical | ec | Hemanth Boyina | Hemanth Boyina | +| [HDFS-14245](https://issues.apache.org/jira/browse/HDFS-14245) | Class cast error in GetGroups with ObserverReadProxyProvider | Major | . | Shen Yinjie | Erik Krogen | +| [HDFS-14373](https://issues.apache.org/jira/browse/HDFS-14373) | EC : Decoding is failing when block group last incomplete cell fall in to AlignedStripe | Critical | ec, hdfs-client | Surendra Singh Lilhore | Surendra Singh Lilhore | +| [HDFS-14509](https://issues.apache.org/jira/browse/HDFS-14509) | DN throws InvalidToken due to inequality of password when upgrade NN 2.x to 3.x | Blocker | . | Yuxuan Wang | Yuxuan Wang | +| [HDFS-14886](https://issues.apache.org/jira/browse/HDFS-14886) | In NameNode Web UI's Startup Progress page, Loading edits always shows 0 sec | Major | . | Hemanth Boyina | Hemanth Boyina | +| [YARN-8453](https://issues.apache.org/jira/browse/YARN-8453) | Additional Unit tests to verify queue limit and max-limit with multiple resource types | Major | capacity scheduler | Sunil G | Adam Antal | +| [HDFS-14890](https://issues.apache.org/jira/browse/HDFS-14890) | Setting permissions on name directory fails on non posix compliant filesystems | Blocker | . | hirik | Siddharth Wagle | +| [HADOOP-16580](https://issues.apache.org/jira/browse/HADOOP-16580) | Disable retry of FailoverOnNetworkExceptionRetry in case of AccessControlException | Major | common | Adam Antal | Adam Antal | +| [HDFS-14909](https://issues.apache.org/jira/browse/HDFS-14909) | DFSNetworkTopology#chooseRandomWithStorageType() should not decrease storage count for excluded node which is already part of excluded scope | Major | namenode | Surendra Singh Lilhore | Surendra Singh Lilhore | +| [HADOOP-16600](https://issues.apache.org/jira/browse/HADOOP-16600) | StagingTestBase uses methods not available in Mockito 1.8.5 in branch-3.1 | Major | . | Lisheng Sun | Duo Zhang | +| [MAPREDUCE-6441](https://issues.apache.org/jira/browse/MAPREDUCE-6441) | Improve temporary directory name generation in LocalDistributedCacheManager for concurrent processes | Major | . | William Watson | Haibo Chen | +| [HADOOP-16662](https://issues.apache.org/jira/browse/HADOOP-16662) | Remove unnecessary InnerNode check in NetworkTopology#add() | Minor | . | Lisheng Sun | Lisheng Sun | +| [HDFS-14847](https://issues.apache.org/jira/browse/HDFS-14847) | Erasure Coding: Blocks are over-replicated while EC decommissioning | Critical | ec | Fei Hui | Fei Hui | +| [HDFS-14913](https://issues.apache.org/jira/browse/HDFS-14913) | Correct the value of available count in DFSNetworkTopology#chooseRandomWithStorageType() | Major | . | Ayush Saxena | Ayush Saxena | +| [YARN-9915](https://issues.apache.org/jira/browse/YARN-9915) | Fix FindBug issue in QueueMetrics | Minor | . | Prabhu Joseph | Prabhu Joseph | +| [HDFS-12749](https://issues.apache.org/jira/browse/HDFS-12749) | DN may not send block report to NN after NN restart | Major | datanode | TanYuxin | Xiaoqiao He | +| [HDFS-13901](https://issues.apache.org/jira/browse/HDFS-13901) | INode access time is ignored because of race between open and rename | Major | . | Jinglun | Jinglun | +| [YARN-9921](https://issues.apache.org/jira/browse/YARN-9921) | Issue in PlacementConstraint when YARN Service AM retries allocation on component failure. | Major | . | Tarun Parimi | Tarun Parimi | +| [HDFS-14910](https://issues.apache.org/jira/browse/HDFS-14910) | Rename Snapshot with Pre Descendants Fail With IllegalArgumentException. | Blocker | . | Íñigo Goiri | Wei-Chiu Chuang | +| [HDFS-14308](https://issues.apache.org/jira/browse/HDFS-14308) | DFSStripedInputStream curStripeBuf is not freed by unbuffer() | Major | ec | Joe McDonnell | Zhao Yi Ming | +| [HDFS-14931](https://issues.apache.org/jira/browse/HDFS-14931) | hdfs crypto commands limit column width | Major | . | Eric Badger | Eric Badger | +| [HADOOP-16669](https://issues.apache.org/jira/browse/HADOOP-16669) | TestRawLocalFileSystemContract.testPermission fails if no native library | Minor | common, test | Steve Loughran | Steve Loughran | +| [HDFS-14920](https://issues.apache.org/jira/browse/HDFS-14920) | Erasure Coding: Decommission may hang If one or more datanodes are out of service during decommission | Major | ec | Fei Hui | Fei Hui | +| [HDFS-13736](https://issues.apache.org/jira/browse/HDFS-13736) | BlockPlacementPolicyDefault can not choose favored nodes when 'dfs.namenode.block-placement-policy.default.prefer-local-node' set to false | Major | . | hu xiaodong | hu xiaodong | +| [HDFS-14925](https://issues.apache.org/jira/browse/HDFS-14925) | rename operation should check nest snapshot | Major | namenode | Junwang Zhao | Junwang Zhao | +| [YARN-9949](https://issues.apache.org/jira/browse/YARN-9949) | Add missing queue configs for root queue in RMWebService#CapacitySchedulerInfo | Minor | capacity scheduler | Prabhu Joseph | Prabhu Joseph | +| [HDFS-14945](https://issues.apache.org/jira/browse/HDFS-14945) | Revise PacketResponder's log. | Minor | datanode | Xudong Cao | Xudong Cao | +| [HDFS-14946](https://issues.apache.org/jira/browse/HDFS-14946) | Erasure Coding: Block recovery failed during decommissioning | Major | . | Fei Hui | Fei Hui | +| [HDFS-14384](https://issues.apache.org/jira/browse/HDFS-14384) | When lastLocatedBlock token expire, it will take 1~3s second to refetch it. | Major | hdfs-client | Surendra Singh Lilhore | Surendra Singh Lilhore | +| [HDFS-14806](https://issues.apache.org/jira/browse/HDFS-14806) | Bootstrap standby may fail if used in-progress tailing | Major | namenode | Chen Liang | Chen Liang | +| [HDFS-14958](https://issues.apache.org/jira/browse/HDFS-14958) | TestBalancerWithNodeGroup is not using NetworkTopologyWithNodeGroup | Minor | hdfs | Jim Brennan | Jim Brennan | +| [HDFS-14720](https://issues.apache.org/jira/browse/HDFS-14720) | DataNode shouldn't report block as bad block if the block length is Long.MAX\_VALUE. | Major | datanode | Surendra Singh Lilhore | Hemanth Boyina | +| [HADOOP-16677](https://issues.apache.org/jira/browse/HADOOP-16677) | Recalculate the remaining timeout millis correctly while throwing an InterupptedException in SocketIOWithTimeout. | Minor | common | Xudong Cao | Xudong Cao | +| [HDFS-14884](https://issues.apache.org/jira/browse/HDFS-14884) | Add sanity check that zone key equals feinfo key while setting Xattrs | Major | encryption, hdfs | Mukul Kumar Singh | Mukul Kumar Singh | +| [HADOOP-15097](https://issues.apache.org/jira/browse/HADOOP-15097) | AbstractContractDeleteTest::testDeleteNonEmptyDirRecursive with misleading path | Minor | fs, test | zhoutai.zt | Xieming Li | +| [YARN-9984](https://issues.apache.org/jira/browse/YARN-9984) | FSPreemptionThread can cause NullPointerException while app is unregistered with containers running on a node | Major | fairscheduler | Wilfred Spiegelenburg | Wilfred Spiegelenburg | +| [YARN-9983](https://issues.apache.org/jira/browse/YARN-9983) | Typo in YARN Service overview documentation | Trivial | documentation | Denes Gerencser | Denes Gerencser | +| [HADOOP-16719](https://issues.apache.org/jira/browse/HADOOP-16719) | Remove the disallowed element config within maven-checkstyle-plugin | Major | . | Wanqiang Ji | Wanqiang Ji | +| [HADOOP-16700](https://issues.apache.org/jira/browse/HADOOP-16700) | RpcQueueTime may be negative when the response has to be sent later | Minor | . | xuzq | xuzq | +| [HADOOP-15686](https://issues.apache.org/jira/browse/HADOOP-15686) | Supress bogus AbstractWadlGeneratorGrammarGenerator in KMS stderr | Major | kms | Wei-Chiu Chuang | Wei-Chiu Chuang | +| [HDFS-14940](https://issues.apache.org/jira/browse/HDFS-14940) | HDFS Balancer : Do not allow to set balancer maximum network bandwidth more than 1TB | Minor | balancer & mover | Souryakanta Dwivedy | Hemanth Boyina | +| [YARN-9838](https://issues.apache.org/jira/browse/YARN-9838) | Fix resource inconsistency for queues when moving app with reserved container to another queue | Critical | capacity scheduler | jiulongzhu | jiulongzhu | +| [YARN-9968](https://issues.apache.org/jira/browse/YARN-9968) | Public Localizer is exiting in NodeManager due to NullPointerException | Major | nodemanager | Tarun Parimi | Tarun Parimi | +| [YARN-9011](https://issues.apache.org/jira/browse/YARN-9011) | Race condition during decommissioning | Major | nodemanager | Peter Bacsko | Peter Bacsko | +| [HDFS-14973](https://issues.apache.org/jira/browse/HDFS-14973) | Balancer getBlocks RPC dispersal does not function properly | Major | balancer & mover | Erik Krogen | Erik Krogen | +| [MAPREDUCE-7240](https://issues.apache.org/jira/browse/MAPREDUCE-7240) | Exception ' Invalid event: TA\_TOO\_MANY\_FETCH\_FAILURE at SUCCESS\_FINISHING\_CONTAINER' cause job error | Critical | . | luhuachao | luhuachao | +| [HDFS-14986](https://issues.apache.org/jira/browse/HDFS-14986) | ReplicaCachingGetSpaceUsed throws ConcurrentModificationException | Major | datanode, performance | Ryan Wu | Aiphago | +| [MAPREDUCE-7249](https://issues.apache.org/jira/browse/MAPREDUCE-7249) | Invalid event TA\_TOO\_MANY\_FETCH\_FAILURE at SUCCESS\_CONTAINER\_CLEANUP causes job failure | Critical | applicationmaster, mrv2 | Wilfred Spiegelenburg | Wilfred Spiegelenburg | +| [HDFS-15010](https://issues.apache.org/jira/browse/HDFS-15010) | BlockPoolSlice#addReplicaThreadPool static pool should be initialized by static method | Major | datanode | Surendra Singh Lilhore | Surendra Singh Lilhore | +| [HADOOP-16744](https://issues.apache.org/jira/browse/HADOOP-16744) | Fix building instruction to enable zstd | Minor | documentation | Masatake Iwasaki | Masatake Iwasaki | +| [HDFS-14869](https://issues.apache.org/jira/browse/HDFS-14869) | Data loss in case of distcp using snapshot diff. Replication should include rename records if file was skipped in the previous iteration | Major | distcp | Aasha Medhi | Aasha Medhi | +| [HADOOP-16754](https://issues.apache.org/jira/browse/HADOOP-16754) | Fix docker failed to build yetus/hadoop | Blocker | build | Kevin Su | Kevin Su | +| [HDFS-15032](https://issues.apache.org/jira/browse/HDFS-15032) | Balancer crashes when it fails to contact an unavailable NN via ObserverReadProxyProvider | Major | balancer & mover | Erik Krogen | Erik Krogen | +| [HDFS-15036](https://issues.apache.org/jira/browse/HDFS-15036) | Active NameNode should not silently fail the image transfer | Major | namenode | Konstantin Shvachko | Chen Liang | +| [HDFS-14519](https://issues.apache.org/jira/browse/HDFS-14519) | NameQuota is not update after concat operation, so namequota is wrong | Major | . | Ranith Sardar | Ranith Sardar | +| [YARN-10055](https://issues.apache.org/jira/browse/YARN-10055) | bower install fails | Blocker | build, yarn-ui-v2 | Akira Ajisaka | Akira Ajisaka | +| [HDFS-15076](https://issues.apache.org/jira/browse/HDFS-15076) | Fix tests that hold FSDirectory lock, without holding FSNamesystem lock. | Major | test | Konstantin Shvachko | Konstantin Shvachko | +| [HDFS-15073](https://issues.apache.org/jira/browse/HDFS-15073) | Replace curator-shaded guava import with the standard one | Minor | hdfs-client | Akira Ajisaka | Chandra Sanivarapu | +| [HADOOP-16042](https://issues.apache.org/jira/browse/HADOOP-16042) | Update the link to HadoopJavaVersion | Minor | documentation | Akira Ajisaka | Chandra Sanivarapu | +| [HDFS-14934](https://issues.apache.org/jira/browse/HDFS-14934) | [SBN Read] Standby NN throws many InterruptedExceptions when dfs.ha.tail-edits.period is 0 | Major | . | Takanobu Asanuma | Ayush Saxena | +| [YARN-10053](https://issues.apache.org/jira/browse/YARN-10053) | Placement rules do not use correct group service init | Major | yarn | Wilfred Spiegelenburg | Wilfred Spiegelenburg | +| [HDFS-15068](https://issues.apache.org/jira/browse/HDFS-15068) | DataNode could meet deadlock if invoke refreshVolumes when register | Major | datanode | Xiaoqiao He | Aiphago | +| [MAPREDUCE-7255](https://issues.apache.org/jira/browse/MAPREDUCE-7255) | Fix typo in MapReduce documentaion example | Trivial | documentation | Sergey Pogorelov | Sergey Pogorelov | +| [HDFS-15072](https://issues.apache.org/jira/browse/HDFS-15072) | HDFS MiniCluster fails to start when run in directory path with a % | Minor | . | Geoffrey Jacoby | Masatake Iwasaki | +| [HDFS-15077](https://issues.apache.org/jira/browse/HDFS-15077) | Fix intermittent failure of TestDFSClientRetries#testLeaseRenewSocketTimeout | Minor | test | Masatake Iwasaki | Masatake Iwasaki | +| [HDFS-15080](https://issues.apache.org/jira/browse/HDFS-15080) | Fix the issue in reading persistent memory cached data with an offset | Major | caching, datanode | Feilong He | Feilong He | +| [YARN-7387](https://issues.apache.org/jira/browse/YARN-7387) | org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.TestIncreaseAllocationExpirer fails intermittently | Major | . | Miklos Szegedi | Jim Brennan | +| [YARN-8672](https://issues.apache.org/jira/browse/YARN-8672) | TestContainerManager#testLocalingResourceWhileContainerRunning occasionally times out | Major | nodemanager | Jason Darrell Lowe | Chandni Singh | +| [HDFS-14957](https://issues.apache.org/jira/browse/HDFS-14957) | INodeReference Space Consumed was not same in QuotaUsage and ContentSummary | Major | namenode | Hemanth Boyina | Hemanth Boyina | +| [MAPREDUCE-7252](https://issues.apache.org/jira/browse/MAPREDUCE-7252) | Handling 0 progress in SimpleExponential task runtime estimator | Minor | . | Ahmed Hussein | Ahmed Hussein | +| [HADOOP-16749](https://issues.apache.org/jira/browse/HADOOP-16749) | Configuration parsing of CDATA values are blank | Major | conf | Jonathan Turner Eagles | Daryn Sharp | +| [HDFS-15095](https://issues.apache.org/jira/browse/HDFS-15095) | Fix accidental comment in flaky test TestDecommissioningStatus | Major | hdfs | Ahmed Hussein | Ahmed Hussein | +| [HDFS-15099](https://issues.apache.org/jira/browse/HDFS-15099) | [SBN Read] checkOperation(WRITE) should throw ObserverRetryOnActiveException on ObserverNode | Major | namenode | Konstantin Shvachko | Chen Liang | +| [HDFS-14578](https://issues.apache.org/jira/browse/HDFS-14578) | AvailableSpaceBlockPlacementPolicy always prefers local node | Major | block placement | Wei-Chiu Chuang | Ayush Saxena | +| [HADOOP-16683](https://issues.apache.org/jira/browse/HADOOP-16683) | Disable retry of FailoverOnNetworkExceptionRetry in case of wrapped AccessControlException | Major | common | Adam Antal | Adam Antal | +| [MAPREDUCE-7256](https://issues.apache.org/jira/browse/MAPREDUCE-7256) | Fix javadoc error in SimpleExponentialSmoothing | Minor | documentation | Masatake Iwasaki | Masatake Iwasaki | +| [YARN-8373](https://issues.apache.org/jira/browse/YARN-8373) | RM Received RMFatalEvent of type CRITICAL\_THREAD\_CRASH | Major | fairscheduler, resourcemanager | Girish Bhat | Wilfred Spiegelenburg | +| [MAPREDUCE-7247](https://issues.apache.org/jira/browse/MAPREDUCE-7247) | Modify HistoryServerRest.html content,change The job attempt id‘s datatype from string to int | Major | documentation | zhaoshengjie | | +| [YARN-8148](https://issues.apache.org/jira/browse/YARN-8148) | Update decimal values for queue capacities shown on queue status CLI | Major | client | Prabhu Joseph | Prabhu Joseph | +| [HADOOP-16808](https://issues.apache.org/jira/browse/HADOOP-16808) | Use forkCount and reuseForks parameters instead of forkMode in the config of maven surefire plugin | Minor | build | Akira Ajisaka | Xieming Li | +| [HADOOP-16793](https://issues.apache.org/jira/browse/HADOOP-16793) | Remove WARN log when ipc connection interrupted in Client#handleSaslConnectionFailure() | Minor | . | Lisheng Sun | Lisheng Sun | +| [YARN-9790](https://issues.apache.org/jira/browse/YARN-9790) | Failed to set default-application-lifetime if maximum-application-lifetime is less than or equal to zero | Major | . | kyungwan nam | kyungwan nam | +| [HDFS-14993](https://issues.apache.org/jira/browse/HDFS-14993) | checkDiskError doesn't work during datanode startup | Major | datanode | Yang Yun | Yang Yun | +| [HDFS-13179](https://issues.apache.org/jira/browse/HDFS-13179) | TestLazyPersistReplicaRecovery#testDnRestartWithSavedReplicas fails intermittently | Critical | fs | Gabor Bota | Ahmed Hussein | +| [MAPREDUCE-7259](https://issues.apache.org/jira/browse/MAPREDUCE-7259) | testSpeculateSuccessfulWithUpdateEvents fails Intermittently | Minor | . | Ahmed Hussein | Ahmed Hussein | +| [MAPREDUCE-7079](https://issues.apache.org/jira/browse/MAPREDUCE-7079) | JobHistory#ServiceStop implementation is incorrect | Major | . | Jason Darrell Lowe | Ahmed Hussein | +| [HDFS-15118](https://issues.apache.org/jira/browse/HDFS-15118) | [SBN Read] Slow clients when Observer reads are enabled but there are no Observers on the cluster. | Major | hdfs-client | Konstantin Shvachko | Chen Liang | +| [HDFS-7175](https://issues.apache.org/jira/browse/HDFS-7175) | Client-side SocketTimeoutException during Fsck | Major | namenode | Carl Steinbach | Stephen O'Donnell | +| [HDFS-15148](https://issues.apache.org/jira/browse/HDFS-15148) | dfs.namenode.send.qop.enabled should not apply to primary NN port | Major | . | Chen Liang | Chen Liang | +| [HDFS-15115](https://issues.apache.org/jira/browse/HDFS-15115) | Namenode crash caused by NPE in BlockPlacementPolicyDefault when dynamically change logger to debug | Major | . | wangzhixiang | wangzhixiang | +| [HDFS-15157](https://issues.apache.org/jira/browse/HDFS-15157) | Fix compilation failure for branch-3.1 | Major | . | Ayush Saxena | Ayush Saxena | +| [HDFS-15158](https://issues.apache.org/jira/browse/HDFS-15158) | The number of failed volumes mismatch with volumeFailures of Datanode metrics | Minor | datanode | Yang Yun | Yang Yun | +| [HADOOP-16849](https://issues.apache.org/jira/browse/HADOOP-16849) | start-build-env.sh behaves incorrectly when username is numeric only | Minor | build | Jihyun Cho | Jihyun Cho | +| [HDFS-15161](https://issues.apache.org/jira/browse/HDFS-15161) | When evictableMmapped or evictable size is zero, do not throw NoSuchElementException in ShortCircuitCache#close() | Major | . | Lisheng Sun | Lisheng Sun | +| [HDFS-15164](https://issues.apache.org/jira/browse/HDFS-15164) | Fix TestDelegationTokensWithHA | Major | . | Ayush Saxena | Ayush Saxena | +| [HADOOP-16868](https://issues.apache.org/jira/browse/HADOOP-16868) | ipc.Server readAndProcess threw NullPointerException | Major | rpc-server | Tsz-wo Sze | Tsz-wo Sze | +| [HADOOP-16869](https://issues.apache.org/jira/browse/HADOOP-16869) | Upgrade findbugs-maven-plugin to 3.0.5 to fix mvn findbugs:findbugs failure | Major | build | Akira Ajisaka | Akira Ajisaka | +| [HDFS-15052](https://issues.apache.org/jira/browse/HDFS-15052) | WebHDFS getTrashRoot leads to OOM due to FileSystem object creation | Major | webhdfs | Wei-Chiu Chuang | Masatake Iwasaki | +| [HDFS-15185](https://issues.apache.org/jira/browse/HDFS-15185) | StartupProgress reports edits segments until the entire startup completes | Major | namenode | Konstantin Shvachko | Konstantin Shvachko | +| [HDFS-15166](https://issues.apache.org/jira/browse/HDFS-15166) | Remove redundant field fStream in ByteStringLog | Major | . | Konstantin Shvachko | Xieming Li | +| [HADOOP-16841](https://issues.apache.org/jira/browse/HADOOP-16841) | The description of hadoop.http.authentication.signature.secret.file contains outdated information | Minor | documentation | Akira Ajisaka | Xieming Li | +| [YARN-10156](https://issues.apache.org/jira/browse/YARN-10156) | Fix typo 'complaint' which means quite different in Federation.md | Minor | documentation, federation | Sungpeo Kook | Sungpeo Kook | +| [HDFS-15147](https://issues.apache.org/jira/browse/HDFS-15147) | LazyPersistTestCase wait logic is error-prone | Minor | . | Ahmed Hussein | Ahmed Hussein | +| [HDFS-14668](https://issues.apache.org/jira/browse/HDFS-14668) | Support Fuse with Users from multiple Security Realms | Critical | fuse-dfs | Sailesh Patel | Istvan Fajth | +| [HDFS-15111](https://issues.apache.org/jira/browse/HDFS-15111) | stopStandbyServices() should log which service state it is transitioning from. | Major | hdfs, logging | Konstantin Shvachko | Xieming Li | +| [HDFS-15199](https://issues.apache.org/jira/browse/HDFS-15199) | NPE in BlockSender | Major | . | Ayush Saxena | Ayush Saxena | +| [HADOOP-16891](https://issues.apache.org/jira/browse/HADOOP-16891) | Upgrade jackson-databind to 2.9.10.3 | Blocker | . | Siyao Meng | Siyao Meng | +| [HADOOP-16840](https://issues.apache.org/jira/browse/HADOOP-16840) | AliyunOSS: getFileStatus throws FileNotFoundException in versioning bucket | Major | fs/oss | wujinhu | wujinhu | +| [YARN-9427](https://issues.apache.org/jira/browse/YARN-9427) | TestContainerSchedulerQueuing.testKillOnlyRequiredOpportunisticContainers fails sporadically | Major | scheduler, test | Prabhu Joseph | Ahmed Hussein | +| [HDFS-15216](https://issues.apache.org/jira/browse/HDFS-15216) | Wrong Use Case of -showprogress in fsck | Major | . | Ravuri Sushma sree | Ravuri Sushma sree | +| [HDFS-15211](https://issues.apache.org/jira/browse/HDFS-15211) | EC: File write hangs during close in case of Exception during updatePipeline | Critical | . | Ayush Saxena | Ayush Saxena | +| [HDFS-15208](https://issues.apache.org/jira/browse/HDFS-15208) | Suppress bogus AbstractWadlGeneratorGrammarGenerator in KMS stderr in hdfs | Trivial | . | Wei-Chiu Chuang | Wei-Chiu Chuang | +| [HDFS-15219](https://issues.apache.org/jira/browse/HDFS-15219) | DFS Client will stuck when ResponseProcessor.run throw Error | Major | hdfs-client | zhengchenyu | zhengchenyu | +| [HADOOP-16949](https://issues.apache.org/jira/browse/HADOOP-16949) | pylint fails in the build environment | Critical | build | Akira Ajisaka | Akira Ajisaka | +| [HADOOP-14836](https://issues.apache.org/jira/browse/HADOOP-14836) | Upgrade maven-clean-plugin to 3.1.0 | Major | build | Allen Wittenauer | Akira Ajisaka | +| [MAPREDUCE-7272](https://issues.apache.org/jira/browse/MAPREDUCE-7272) | TaskAttemptListenerImpl excessive log messages | Major | . | Ahmed Hussein | Ahmed Hussein | +| [HDFS-15283](https://issues.apache.org/jira/browse/HDFS-15283) | Cache pool MAXTTL is not persisted and restored on cluster restart | Major | namenode | Stephen O'Donnell | Stephen O'Donnell | +| [HADOOP-16944](https://issues.apache.org/jira/browse/HADOOP-16944) | Use Yetus 0.12.0 in GitHub PR | Major | build | Akira Ajisaka | Akira Ajisaka | +| [HDFS-15276](https://issues.apache.org/jira/browse/HDFS-15276) | Concat on INodeRefernce fails with illegal state exception | Critical | . | Hemanth Boyina | Hemanth Boyina | +| [HDFS-15281](https://issues.apache.org/jira/browse/HDFS-15281) | ZKFC ignores dfs.namenode.rpc-bind-host and uses dfs.namenode.rpc-address to bind to host address | Major | ha, namenode | Dhiraj Hegde | Dhiraj Hegde | +| [HDFS-15297](https://issues.apache.org/jira/browse/HDFS-15297) | TestNNHandlesBlockReportPerStorage::blockReport\_02 fails intermittently in trunk | Major | datanode, test | Mingliang Liu | Ayush Saxena | +| [HADOOP-17014](https://issues.apache.org/jira/browse/HADOOP-17014) | Upgrade jackson-databind to 2.9.10.4 | Blocker | . | Siyao Meng | Siyao Meng | +| [HDFS-15286](https://issues.apache.org/jira/browse/HDFS-15286) | Concat on a same files deleting the file | Critical | . | Hemanth Boyina | Hemanth Boyina | +| [HADOOP-15565](https://issues.apache.org/jira/browse/HADOOP-15565) | ViewFileSystem.close doesn't close child filesystems and causes FileSystem objects leak. | Major | . | Jinglun | Jinglun | +| [HADOOP-17052](https://issues.apache.org/jira/browse/HADOOP-17052) | NetUtils.connect() throws unchecked exception (UnresolvedAddressException) causing clients to abort | Major | net | Dhiraj Hegde | Dhiraj Hegde | +| [YARN-10347](https://issues.apache.org/jira/browse/YARN-10347) | Fix double locking in CapacityScheduler#reinitialize in branch-3.1 | Critical | capacity scheduler | Masatake Iwasaki | Masatake Iwasaki | + + +### TESTS: + +| JIRA | Summary | Priority | Component | Reporter | Contributor | +|:---- |:---- | :--- |:---- |:---- |:---- | +| [YARN-10072](https://issues.apache.org/jira/browse/YARN-10072) | TestCSAllocateCustomResource failures | Major | yarn | Jim Brennan | Jim Brennan | +| [YARN-10161](https://issues.apache.org/jira/browse/YARN-10161) | TestRouterWebServicesREST is corrupting STDOUT | Minor | yarn | Jim Brennan | Jim Brennan | +| [HADOOP-14206](https://issues.apache.org/jira/browse/HADOOP-14206) | TestSFTPFileSystem#testFileExists failure: Invalid encoding for signature | Major | fs, test | John Zhuge | Jim Brennan | + + +### SUB-TASKS: + +| JIRA | Summary | Priority | Component | Reporter | Contributor | +|:---- |:---- | :--- |:---- |:---- |:---- | +| [HADOOP-15636](https://issues.apache.org/jira/browse/HADOOP-15636) | Add ITestDynamoDBMetadataStore | Minor | fs/s3, test | Sean Mackrory | Gabor Bota | +| [HADOOP-15787](https://issues.apache.org/jira/browse/HADOOP-15787) | [JDK11] TestIPC.testRTEDuringConnectionSetup fails | Major | . | Akira Ajisaka | Zsolt Venczel | +| [HDFS-14262](https://issues.apache.org/jira/browse/HDFS-14262) | [SBN read] Unclear Log.WARN message in GlobalStateIdContext | Major | hdfs | Shweta | Shweta | +| [HDFS-14590](https://issues.apache.org/jira/browse/HDFS-14590) | [SBN Read] Add the document link to the top page | Major | documentation | Takanobu Asanuma | Takanobu Asanuma | +| [YARN-9791](https://issues.apache.org/jira/browse/YARN-9791) | Queue Mutation API does not allow to remove a config | Major | capacity scheduler | Prabhu Joseph | Prabhu Joseph | +| [HDFS-14822](https://issues.apache.org/jira/browse/HDFS-14822) | [SBN read] Revisit GlobalStateIdContext locking when getting server state id | Major | hdfs | Chen Liang | Chen Liang | +| [HDFS-14785](https://issues.apache.org/jira/browse/HDFS-14785) | [SBN read] Change client logging to be less aggressive | Major | hdfs | Chen Liang | Chen Liang | +| [YARN-9864](https://issues.apache.org/jira/browse/YARN-9864) | Format CS Configuration present in Configuration Store | Major | capacity scheduler | Prabhu Joseph | Prabhu Joseph | +| [YARN-9801](https://issues.apache.org/jira/browse/YARN-9801) | SchedConfCli does not work with https mode | Major | . | Prabhu Joseph | Prabhu Joseph | +| [HDFS-14858](https://issues.apache.org/jira/browse/HDFS-14858) | [SBN read] Allow configurably enable/disable AlignmentContext on NameNode | Major | hdfs | Chen Liang | Chen Liang | +| [HDFS-12979](https://issues.apache.org/jira/browse/HDFS-12979) | StandbyNode should upload FsImage to ObserverNode after checkpointing. | Major | hdfs | Konstantin Shvachko | Chen Liang | +| [YARN-9873](https://issues.apache.org/jira/browse/YARN-9873) | Mutation API Config Change need to update Version Number | Major | . | Prabhu Joseph | Prabhu Joseph | +| [HDFS-14162](https://issues.apache.org/jira/browse/HDFS-14162) | Balancer should work with ObserverNode | Major | . | Konstantin Shvachko | Erik Krogen | +| [YARN-9773](https://issues.apache.org/jira/browse/YARN-9773) | Add QueueMetrics for Custom Resources | Major | . | Manikandan R | Manikandan R | +| [HADOOP-16598](https://issues.apache.org/jira/browse/HADOOP-16598) | Backport "HADOOP-16558 [COMMON+HDFS] use protobuf-maven-plugin to generate protobuf classes" to all active branches | Major | common | Duo Zhang | Duo Zhang | +| [YARN-9950](https://issues.apache.org/jira/browse/YARN-9950) | Unset Ordering Policy of Leaf/Parent queue converted from Parent/Leaf queue respectively | Major | capacity scheduler | Prabhu Joseph | Prabhu Joseph | +| [YARN-9900](https://issues.apache.org/jira/browse/YARN-9900) | Revert to previous state when Invalid Config is applied and Refresh Support in SchedulerConfig Format | Major | capacity scheduler | Prabhu Joseph | Prabhu Joseph | +| [HADOOP-16610](https://issues.apache.org/jira/browse/HADOOP-16610) | Upgrade to yetus 0.11.1 and use emoji vote on github pre commit | Major | build | Duo Zhang | Duo Zhang | +| [YARN-9909](https://issues.apache.org/jira/browse/YARN-9909) | Offline format of YarnConfigurationStore | Major | capacity scheduler | Prabhu Joseph | Prabhu Joseph | +| [YARN-9836](https://issues.apache.org/jira/browse/YARN-9836) | General usability improvements in showSimulationTrace.html | Minor | scheduler-load-simulator | Adam Antal | Adam Antal | +| [HADOOP-16758](https://issues.apache.org/jira/browse/HADOOP-16758) | Refine testing.md to tell user better how to use auth-keys.xml | Minor | fs/s3 | Mingliang Liu | Mingliang Liu | +| [HADOOP-16609](https://issues.apache.org/jira/browse/HADOOP-16609) | Add Jenkinsfile for all active branches | Major | build | Duo Zhang | Akira Ajisaka | +| [YARN-10022](https://issues.apache.org/jira/browse/YARN-10022) | Create RM Rest API to validate a CapacityScheduler Configuration | Major | . | Kinga Marton | Kinga Marton | +| [HDFS-15173](https://issues.apache.org/jira/browse/HDFS-15173) | RBF: Delete repeated configuration 'dfs.federation.router.metrics.enable' | Minor | documentation, rbf | panlijie | panlijie | +| [YARN-10139](https://issues.apache.org/jira/browse/YARN-10139) | ValidateAndGetSchedulerConfiguration API fails when cluster max allocation \> default 8GB | Major | . | Prabhu Joseph | Prabhu Joseph | +| [HDFS-14731](https://issues.apache.org/jira/browse/HDFS-14731) | [FGL] Remove redundant locking on NameNode. | Major | namenode | Konstantin Shvachko | Konstantin Shvachko | +| [HADOOP-14918](https://issues.apache.org/jira/browse/HADOOP-14918) | Remove the Local Dynamo DB test option | Major | fs/s3 | Steve Loughran | Gabor Bota | +| [HDFS-15305](https://issues.apache.org/jira/browse/HDFS-15305) | Extend ViewFS and provide ViewFSOverloadScheme implementation with scheme configurable. | Major | fs, hadoop-client, hdfs-client, viewfs | Uma Maheswara Rao G | Uma Maheswara Rao G | + + +### OTHER: + +| JIRA | Summary | Priority | Component | Reporter | Contributor | +|:---- |:---- | :--- |:---- |:---- |:---- | +| [YARN-8016](https://issues.apache.org/jira/browse/YARN-8016) | Refine PlacementRule interface and add a app-name queue mapping rule as an example | Major | . | Zian Chen | Zian Chen | +| [HADOOP-16491](https://issues.apache.org/jira/browse/HADOOP-16491) | Upgrade jetty version to 9.3.27 | Major | . | Hrishikesh Gadre | Hrishikesh Gadre | +| [HADOOP-16542](https://issues.apache.org/jira/browse/HADOOP-16542) | Update commons-beanutils version to 1.9.4 | Major | . | Wei-Chiu Chuang | Kevin Su | +| [HADOOP-16551](https://issues.apache.org/jira/browse/HADOOP-16551) | The changelog\*.md seems not generated when create-release | Blocker | . | Zhankun Tang | | +| [YARN-9730](https://issues.apache.org/jira/browse/YARN-9730) | Support forcing configured partitions to be exclusive based on app node label | Major | . | Jonathan Hung | Jonathan Hung | +| [HDFS-14959](https://issues.apache.org/jira/browse/HDFS-14959) | [SBNN read] access time should be turned off | Major | documentation | Wei-Chiu Chuang | Chao Sun | +| [HADOOP-16784](https://issues.apache.org/jira/browse/HADOOP-16784) | Update the year to 2020 | Major | . | Ayush Saxena | Ayush Saxena | +| [HADOOP-16871](https://issues.apache.org/jira/browse/HADOOP-16871) | Upgrade Netty version to 4.1.45.Final to handle CVE-2019-20444,CVE-2019-16869 | Major | . | Aray Chenchu Sukesh | Aray Chenchu Sukesh | +| [HADOOP-16982](https://issues.apache.org/jira/browse/HADOOP-16982) | Update Netty to 4.1.48.Final | Blocker | . | Wei-Chiu Chuang | Lisheng Sun | + + diff --git a/hadoop-common-project/hadoop-common/src/site/markdown/release/3.1.4/RELEASENOTES.3.1.4.md b/hadoop-common-project/hadoop-common/src/site/markdown/release/3.1.4/RELEASENOTES.3.1.4.md new file mode 100644 index 0000000000000..b22827bf8c8de --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/site/markdown/release/3.1.4/RELEASENOTES.3.1.4.md @@ -0,0 +1,72 @@ + + +# Apache Hadoop 3.1.4 Release Notes + +These release notes cover new developer and user-facing incompatibilities, important issues, features, and major improvements. + + +--- + +* [HDFS-13806](https://issues.apache.org/jira/browse/HDFS-13806) | *Minor* | **EC: No error message for unsetting EC policy of the directory inherits the erasure coding policy from an ancestor directory** + +After this change, attempt to unsetErasureCodingPolicy() on a directory without EC policy explicitly set on it, will get NoECPolicySetException. + + +--- + +* [HDFS-14890](https://issues.apache.org/jira/browse/HDFS-14890) | *Blocker* | **Setting permissions on name directory fails on non posix compliant filesystems** + +- Fixed namenode/journal startup on Windows. + + +--- + +* [HDFS-14745](https://issues.apache.org/jira/browse/HDFS-14745) | *Major* | **Backport HDFS persistent memory read cache support to branch-3.1** + +Non-volatile storage class memory (SCM, also known as persistent memory) is supported in HDFS cache. To enable SCM cache, user just needs to configure SCM volume for property “dfs.datanode.cache.pmem.dirs” in hdfs-site.xml. And all HDFS cache directives keep unchanged. There are two implementations for HDFS SCM Cache, one is pure java code implementation and the other is native PMDK based implementation. The latter implementation can bring user better performance gain in cache write and cache read. If PMDK native libs could be loaded, it will use PMDK based implementation otherwise it will fallback to java code implementation. To enable PMDK based implementation, user should install PMDK library by referring to the official site http://pmem.io/. Then, build Hadoop with PMDK support by referring to "PMDK library build options" section in \`BUILDING.txt\` in the source code. If multiple SCM volumes are configured, a round-robin policy is used to select an available volume for caching a block. Consistent with DRAM cache, SCM cache also has no cache eviction mechanism. When DataNode receives a data read request from a client, if the corresponding block is cached into SCM, DataNode will instantiate an InputStream with the block location path on SCM (pure java implementation) or cache address on SCM (PMDK based implementation). Once the InputStream is created, DataNode will send the cached data to the client. Please refer "Centralized Cache Management" guide for more details. + + +--- + +* [HDFS-12943](https://issues.apache.org/jira/browse/HDFS-12943) | *Major* | **Consistent Reads from Standby Node** + +Observer is a new type of a NameNode in addition to Active and Standby Nodes in HA settings. An Observer Node maintains a replica of the namespace same as a Standby Node. It additionally allows execution of clients read requests. + +To ensure read-after-write consistency within a single client, a state ID is introduced in RPC headers. The Observer responds to the client request only after its own state has caught up with the client’s state ID, which it previously received from the Active NameNode. + +Clients can explicitly invoke a new client protocol call msync(), which ensures that subsequent reads by this client from an Observer are consistent. + +A new client-side ObserverReadProxyProvider is introduced to provide automatic switching between Active and Observer NameNodes for submitting respectively write and read requests. + + +--- + +* [HADOOP-16771](https://issues.apache.org/jira/browse/HADOOP-16771) | *Major* | **Update checkstyle to 8.26 and maven-checkstyle-plugin to 3.1.0** + +Updated checkstyle to 8.26 and updated maven-checkstyle-plugin to 3.1.0. + + +--- + +* [HDFS-15281](https://issues.apache.org/jira/browse/HDFS-15281) | *Major* | **ZKFC ignores dfs.namenode.rpc-bind-host and uses dfs.namenode.rpc-address to bind to host address** + +ZKFC binds host address to "dfs.namenode.servicerpc-bind-host", if configured. Otherwise, it binds to "dfs.namenode.rpc-bind-host". If neither of those is configured, ZKFC binds itself to NameNode RPC server address (effectively "dfs.namenode.rpc-address"). + + + diff --git a/hadoop-common-project/hadoop-common/src/site/markdown/release/3.3.0/CHANGELOG.3.3.0.md b/hadoop-common-project/hadoop-common/src/site/markdown/release/3.3.0/CHANGELOG.3.3.0.md new file mode 100644 index 0000000000000..6ac3f1df25862 --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/site/markdown/release/3.3.0/CHANGELOG.3.3.0.md @@ -0,0 +1,2186 @@ + + +# Apache Hadoop Changelog + +## Release 3.3.0 - 2020-07-06 + + + +### IMPORTANT ISSUES: + +| JIRA | Summary | Priority | Component | Reporter | Contributor | +|:---- |:---- | :--- |:---- |:---- |:---- | +| [HDFS-14339](https://issues.apache.org/jira/browse/HDFS-14339) | Inconsistent log level practices in RpcProgramNfs3.java | Major | nfs | Anuhan Torgonshar | Anuhan Torgonshar | +| [HDFS-15186](https://issues.apache.org/jira/browse/HDFS-15186) | Erasure Coding: Decommission may generate the parity block's content with all 0 in some case | Critical | datanode, erasure-coding | Yao Guangdong | Yao Guangdong | + + +### NEW FEATURES: + +| JIRA | Summary | Priority | Component | Reporter | Contributor | +|:---- |:---- | :--- |:---- |:---- |:---- | +| [HADOOP-15885](https://issues.apache.org/jira/browse/HADOOP-15885) | Add base64 (urlString) support to DTUtil | Minor | . | Íñigo Goiri | Íñigo Goiri | +| [HADOOP-15950](https://issues.apache.org/jira/browse/HADOOP-15950) | Failover for LdapGroupsMapping | Major | common, security | Lukas Majercak | Lukas Majercak | +| [HDFS-14001](https://issues.apache.org/jira/browse/HDFS-14001) | [PROVIDED Storage] bootstrapStandby should manage the InMemoryAliasMap | Major | . | Íñigo Goiri | Virajith Jalaparti | +| [YARN-8762](https://issues.apache.org/jira/browse/YARN-8762) | [Umbrella] Support Interactive Docker Shell to running Containers | Major | . | Zian Chen | Eric Yang | +| [HADOOP-15996](https://issues.apache.org/jira/browse/HADOOP-15996) | Plugin interface to support more complex usernames in Hadoop | Major | security | Eric Yang | Bolke de Bruin | +| [HADOOP-15229](https://issues.apache.org/jira/browse/HADOOP-15229) | Add FileSystem builder-based openFile() API to match createFile(); S3A to implement S3 Select through this API. | Major | fs, fs/azure, fs/s3 | Steve Loughran | Steve Loughran | +| [HDFS-14118](https://issues.apache.org/jira/browse/HDFS-14118) | Support using DNS to resolve nameservices to IP addresses | Major | . | Fengnan Li | Fengnan Li | +| [HADOOP-16125](https://issues.apache.org/jira/browse/HADOOP-16125) | Support multiple bind users in LdapGroupsMapping | Major | common, security | Lukas Majercak | Lukas Majercak | +| [YARN-9228](https://issues.apache.org/jira/browse/YARN-9228) | [Umbrella] Docker image life cycle management on HDFS | Major | . | Eric Yang | Eric Yang | +| [YARN-9016](https://issues.apache.org/jira/browse/YARN-9016) | DocumentStore as a backend for ATSv2 | Major | ATSv2 | Sushil Ks | Sushil Ks | +| [HDFS-14234](https://issues.apache.org/jira/browse/HDFS-14234) | Limit WebHDFS to specifc user, host, directory triples | Trivial | webhdfs | Clay B. | Clay B. | +| [HADOOP-16095](https://issues.apache.org/jira/browse/HADOOP-16095) | Support impersonation for AuthenticationFilter | Major | security | Eric Yang | Eric Yang | +| [HDFS-12345](https://issues.apache.org/jira/browse/HDFS-12345) | Scale testing HDFS NameNode with real metadata and workloads (Dynamometer) | Major | namenode, test | Zhe Zhang | Erik Krogen | +| [YARN-9473](https://issues.apache.org/jira/browse/YARN-9473) | [Umbrella] Support Vector Engine ( a new accelerator hardware) based on pluggable device framework | Major | nodemanager | Zhankun Tang | Peter Bacsko | +| [HDFS-13783](https://issues.apache.org/jira/browse/HDFS-13783) | Balancer: make balancer to be a long service process for easy to monitor it. | Major | balancer & mover | maobaolong | Chen Zhang | +| [HADOOP-16398](https://issues.apache.org/jira/browse/HADOOP-16398) | Exports Hadoop metrics to Prometheus | Major | metrics | Akira Ajisaka | Akira Ajisaka | +| [HADOOP-16527](https://issues.apache.org/jira/browse/HADOOP-16527) | Add a whitelist of endpoints to skip Kerberos authentication | Major | security | Akira Ajisaka | Akira Ajisaka | +| [HDFS-12904](https://issues.apache.org/jira/browse/HDFS-12904) | Add DataTransferThrottler to the Datanode transfers | Minor | datanode | Íñigo Goiri | Lisheng Sun | +| [YARN-9761](https://issues.apache.org/jira/browse/YARN-9761) | Allow overriding application submissions based on server side configs | Major | . | Jonathan Hung | pralabhkumar | +| [YARN-9808](https://issues.apache.org/jira/browse/YARN-9808) | Zero length files in container log output haven't got a header | Major | log-aggregation, yarn | Adam Antal | Adam Antal | +| [HADOOP-15691](https://issues.apache.org/jira/browse/HADOOP-15691) | Add PathCapabilities to FS and FC to complement StreamCapabilities | Major | . | Steve Loughran | Steve Loughran | +| [HADOOP-15616](https://issues.apache.org/jira/browse/HADOOP-15616) | Incorporate Tencent Cloud COS File System Implementation | Major | fs/cos | Junping Du | Yang Yu | +| [YARN-9760](https://issues.apache.org/jira/browse/YARN-9760) | Support configuring application priorities on a workflow level | Major | . | Jonathan Hung | Varun Saxena | +| [HDFS-13762](https://issues.apache.org/jira/browse/HDFS-13762) | Support non-volatile storage class memory(SCM) in HDFS cache directives | Major | caching, datanode | Sammi Chen | Feilong He | +| [HDFS-12943](https://issues.apache.org/jira/browse/HDFS-12943) | Consistent Reads from Standby Node | Major | hdfs | Konstantin Shvachko | Konstantin Shvachko | +| [HDFS-13571](https://issues.apache.org/jira/browse/HDFS-13571) | Deadnode detection | Major | hdfs-client | Gang Xie | Lisheng Sun | +| [YARN-9923](https://issues.apache.org/jira/browse/YARN-9923) | Introduce HealthReporter interface to support multiple health checker files | Major | nodemanager, yarn | Adam Antal | Adam Antal | +| [YARN-8851](https://issues.apache.org/jira/browse/YARN-8851) | [Umbrella] A pluggable device plugin framework to ease vendor plugin development | Major | yarn | Zhankun Tang | Zhankun Tang | +| [YARN-9414](https://issues.apache.org/jira/browse/YARN-9414) | Application Catalog for YARN applications | Major | . | Eric Yang | Eric Yang | +| [YARN-5542](https://issues.apache.org/jira/browse/YARN-5542) | Scheduling of opportunistic containers | Major | . | Konstantinos Karanasos | | +| [HDFS-13616](https://issues.apache.org/jira/browse/HDFS-13616) | Batch listing of multiple directories | Major | . | Andrew Wang | Chao Sun | +| [HDFS-14743](https://issues.apache.org/jira/browse/HDFS-14743) | Enhance INodeAttributeProvider/ AccessControlEnforcer Interface in HDFS to support Authorization of mkdir, rm, rmdir, copy, move etc... | Critical | hdfs | Ramesh Mani | Wei-Chiu Chuang | +| [MAPREDUCE-7237](https://issues.apache.org/jira/browse/MAPREDUCE-7237) | Supports config the shuffle's path cache related parameters | Major | mrv2 | Wanqiang Ji | Wanqiang Ji | +| [HADOOP-16661](https://issues.apache.org/jira/browse/HADOOP-16661) | Support TLS 1.3 | Major | security | Wei-Chiu Chuang | Wei-Chiu Chuang | +| [HADOOP-16912](https://issues.apache.org/jira/browse/HADOOP-16912) | Emit per priority RPC queue time and processing time from DecayRpcScheduler | Major | common | Fengnan Li | Fengnan Li | + + +### IMPROVEMENTS: + +| JIRA | Summary | Priority | Component | Reporter | Contributor | +|:---- |:---- | :--- |:---- |:---- |:---- | +| [YARN-8226](https://issues.apache.org/jira/browse/YARN-8226) | Improve anti-affinity section description in YARN Service API doc | Major | docs, documentation | Charan Hebri | Gour Saha | +| [HADOOP-15356](https://issues.apache.org/jira/browse/HADOOP-15356) | Make HTTP timeout configurable in ADLS Connector | Major | fs/adl | Atul Sikaria | Atul Sikaria | +| [HADOOP-15586](https://issues.apache.org/jira/browse/HADOOP-15586) | Fix wrong log statement in AbstractService | Minor | util | Szilard Nemeth | Szilard Nemeth | +| [HADOOP-15657](https://issues.apache.org/jira/browse/HADOOP-15657) | Registering MutableQuantiles via Metric annotation | Major | metrics | Sushil Ks | Sushil Ks | +| [YARN-8621](https://issues.apache.org/jira/browse/YARN-8621) | Add test coverage of custom Resource Types for the apps/\ REST API endpoint | Major | . | Szilard Nemeth | Szilard Nemeth | +| [HDFS-13947](https://issues.apache.org/jira/browse/HDFS-13947) | Review of DirectoryScanner Class | Major | datanode | David Mollitor | David Mollitor | +| [YARN-8732](https://issues.apache.org/jira/browse/YARN-8732) | Add unit tests of min/max allocation for custom resource types in FairScheduler | Minor | . | Szilard Nemeth | Szilard Nemeth | +| [YARN-8750](https://issues.apache.org/jira/browse/YARN-8750) | Refactor TestQueueMetrics | Minor | resourcemanager | Szilard Nemeth | Szilard Nemeth | +| [HDFS-13950](https://issues.apache.org/jira/browse/HDFS-13950) | ACL documentation update to indicate that ACL entries are capped by 32 | Minor | hdfs | Adam Antal | Adam Antal | +| [HDFS-13958](https://issues.apache.org/jira/browse/HDFS-13958) | Miscellaneous Improvements for FsVolumeSpi | Major | datanode | David Mollitor | David Mollitor | +| [YARN-8644](https://issues.apache.org/jira/browse/YARN-8644) | Improve unit test for RMAppImpl.FinalTransition | Minor | . | Szilard Nemeth | Szilard Nemeth | +| [HDFS-13967](https://issues.apache.org/jira/browse/HDFS-13967) | HDFS Router Quota Class Review | Minor | federation, hdfs | David Mollitor | David Mollitor | +| [HADOOP-15832](https://issues.apache.org/jira/browse/HADOOP-15832) | Upgrade BouncyCastle to 1.60 | Major | . | Robert Kanter | Robert Kanter | +| [HDFS-13882](https://issues.apache.org/jira/browse/HDFS-13882) | Set a maximum delay for retrying locateFollowingBlock | Major | . | Kitti Nanasi | Kitti Nanasi | +| [MAPREDUCE-7149](https://issues.apache.org/jira/browse/MAPREDUCE-7149) | javadocs for FileInputFormat and OutputFormat to mention DT collection | Minor | client | Steve Loughran | Steve Loughran | +| [HDFS-13968](https://issues.apache.org/jira/browse/HDFS-13968) | BlockReceiver Array-Based Queue | Minor | datanode | David Mollitor | David Mollitor | +| [HADOOP-15717](https://issues.apache.org/jira/browse/HADOOP-15717) | TGT renewal thread does not log IOException | Major | . | Szilard Nemeth | Szilard Nemeth | +| [HADOOP-15831](https://issues.apache.org/jira/browse/HADOOP-15831) | Include modificationTime in the toString method of CopyListingFileStatus | Minor | . | Ted Yu | Ted Yu | +| [HDFS-13156](https://issues.apache.org/jira/browse/HDFS-13156) | HDFS Block Placement Policy - Client Local Rack | Minor | documentation | David Mollitor | Ayush Saxena | +| [HADOOP-15849](https://issues.apache.org/jira/browse/HADOOP-15849) | Upgrade netty version to 3.10.6 | Major | . | Xiao Chen | Xiao Chen | +| [YARN-8836](https://issues.apache.org/jira/browse/YARN-8836) | Add tags and attributes in resource definition | Major | . | Weiwei Yang | Weiwei Yang | +| [HDFS-13987](https://issues.apache.org/jira/browse/HDFS-13987) | RBF: Review of RandomResolver Class | Minor | federation | David Mollitor | David Mollitor | +| [MAPREDUCE-7150](https://issues.apache.org/jira/browse/MAPREDUCE-7150) | Optimize collections used by MR JHS to reduce its memory | Major | jobhistoryserver, mrv2 | Misha Dmitriev | Misha Dmitriev | +| [HADOOP-15854](https://issues.apache.org/jira/browse/HADOOP-15854) | AuthToken Use StringBuilder instead of StringBuffer | Trivial | auth | David Mollitor | David Mollitor | +| [HADOOP-11100](https://issues.apache.org/jira/browse/HADOOP-11100) | Support to configure ftpClient.setControlKeepAliveTimeout | Minor | fs | Krishnamoorthy Dharmalingam | Adam Antal | +| [YARN-8899](https://issues.apache.org/jira/browse/YARN-8899) | TestCleanupAfterKIll is failing due to unsatisfied dependencies | Blocker | yarn-native-services | Eric Yang | Robert Kanter | +| [YARN-8618](https://issues.apache.org/jira/browse/YARN-8618) | Yarn Service: When all the components of a service have restart policy NEVER then initiation of service upgrade should fail | Major | . | Chandni Singh | Chandni Singh | +| [HADOOP-15804](https://issues.apache.org/jira/browse/HADOOP-15804) | upgrade to commons-compress 1.18 | Major | . | PJ Fanning | Akira Ajisaka | +| [YARN-8916](https://issues.apache.org/jira/browse/YARN-8916) | Define a constant "docker" string in "ContainerRuntimeConstants.java" for better maintainability | Minor | . | Zhankun Tang | Zhankun Tang | +| [YARN-8908](https://issues.apache.org/jira/browse/YARN-8908) | Fix errors in yarn-default.xml related to GPU/FPGA | Major | . | Zhankun Tang | Zhankun Tang | +| [HDFS-13994](https://issues.apache.org/jira/browse/HDFS-13994) | Improve DataNode BlockSender waitForMinLength | Minor | datanode | David Mollitor | David Mollitor | +| [HADOOP-15821](https://issues.apache.org/jira/browse/HADOOP-15821) | Move Hadoop YARN Registry to Hadoop Registry | Major | . | Íñigo Goiri | Íñigo Goiri | +| [YARN-8542](https://issues.apache.org/jira/browse/YARN-8542) | Yarn Service: Add component name to container json | Major | . | Chandni Singh | Chandni Singh | +| [YARN-8923](https://issues.apache.org/jira/browse/YARN-8923) | Cleanup references to ENV file type in YARN service code | Minor | yarn-native-services | Suma Shivaprasad | Suma Shivaprasad | +| [YARN-6586](https://issues.apache.org/jira/browse/YARN-6586) | YARN to facilitate HTTPS in AM web server | Major | yarn | Haibo Chen | Robert Kanter | +| [HDFS-13941](https://issues.apache.org/jira/browse/HDFS-13941) | make storageId in BlockPoolTokenSecretManager.checkAccess optional | Major | . | Ajay Kumar | Ajay Kumar | +| [HDFS-14026](https://issues.apache.org/jira/browse/HDFS-14026) | Overload BlockPoolTokenSecretManager.checkAccess to make storageId and storageType optional | Major | . | Ajay Kumar | Ajay Kumar | +| [HDFS-14029](https://issues.apache.org/jira/browse/HDFS-14029) | Sleep in TestLazyPersistFiles should be put into a loop | Trivial | hdfs | Adam Antal | Adam Antal | +| [HADOOP-9567](https://issues.apache.org/jira/browse/HADOOP-9567) | Provide auto-renewal for keytab based logins | Minor | security | Harsh J | Hrishikesh Gadre | +| [YARN-8915](https://issues.apache.org/jira/browse/YARN-8915) | Update the doc about the default value of "maximum-container-assignments" for capacity scheduler | Minor | . | Zhankun Tang | Zhankun Tang | +| [HDFS-14008](https://issues.apache.org/jira/browse/HDFS-14008) | NN should log snapshotdiff report | Major | namenode | Pranay Singh | Pranay Singh | +| [HDFS-13996](https://issues.apache.org/jira/browse/HDFS-13996) | Make HttpFS' ACLs RegEx configurable | Major | httpfs | Siyao Meng | Siyao Meng | +| [YARN-8954](https://issues.apache.org/jira/browse/YARN-8954) | Reservations list field in ReservationListInfo is not accessible | Minor | resourcemanager, restapi | Oleksandr Shevchenko | Oleksandr Shevchenko | +| [YARN-7225](https://issues.apache.org/jira/browse/YARN-7225) | Add queue and partition info to RM audit log | Major | resourcemanager | Jonathan Hung | Eric Payne | +| [HADOOP-15687](https://issues.apache.org/jira/browse/HADOOP-15687) | Credentials class should allow access to aliases | Trivial | . | Lars Francke | Lars Francke | +| [YARN-8969](https://issues.apache.org/jira/browse/YARN-8969) | AbstractYarnScheduler#getNodeTracker should return generic type to avoid type casting | Major | . | Wanqiang Ji | Wanqiang Ji | +| [HDFS-14053](https://issues.apache.org/jira/browse/HDFS-14053) | Provide ability for NN to re-replicate based on topology changes. | Major | . | ellen johansen | Hrishikesh Gadre | +| [HDFS-14051](https://issues.apache.org/jira/browse/HDFS-14051) | Refactor NameNodeHttpServer#initWebHdfs to specify local keytab | Major | . | Íñigo Goiri | CR Hota | +| [YARN-8957](https://issues.apache.org/jira/browse/YARN-8957) | Add Serializable interface to ComponentContainers | Minor | . | Zhankun Tang | Zhankun Tang | +| [YARN-8976](https://issues.apache.org/jira/browse/YARN-8976) | Remove redundant modifiers in interface "ApplicationConstants" | Trivial | . | Zhankun Tang | Zhankun Tang | +| [HADOOP-15907](https://issues.apache.org/jira/browse/HADOOP-15907) | Add missing maven modules in BUILDING.txt | Trivial | . | Wanqiang Ji | Wanqiang Ji | +| [MAPREDUCE-7148](https://issues.apache.org/jira/browse/MAPREDUCE-7148) | Fast fail jobs when exceeds dfs quota limitation | Major | task | Wang Yan | Wang Yan | +| [YARN-8977](https://issues.apache.org/jira/browse/YARN-8977) | Remove unnecessary type casting when calling AbstractYarnScheduler#getSchedulerNode | Trivial | . | Wanqiang Ji | Wanqiang Ji | +| [YARN-8997](https://issues.apache.org/jira/browse/YARN-8997) | [Submarine] Small refactors of modifier, condition check and redundant local variables | Minor | . | Zhankun Tang | Zhankun Tang | +| [HDFS-14070](https://issues.apache.org/jira/browse/HDFS-14070) | Refactor NameNodeWebHdfsMethods to allow better extensibility | Major | . | CR Hota | CR Hota | +| [HADOOP-15926](https://issues.apache.org/jira/browse/HADOOP-15926) | Document upgrading the section in NOTICE.txt when upgrading the version of AWS SDK | Minor | documentation | Akira Ajisaka | Dinesh Chitlangia | +| [HDFS-14045](https://issues.apache.org/jira/browse/HDFS-14045) | Use different metrics in DataNode to better measure latency of heartbeat/blockReports/incrementalBlockReports of Active/Standby NN | Major | datanode | Jiandan Yang | Jiandan Yang | +| [HADOOP-12558](https://issues.apache.org/jira/browse/HADOOP-12558) | distcp documentation is woefully out of date | Critical | documentation, tools/distcp | Allen Wittenauer | Dinesh Chitlangia | +| [HDFS-14063](https://issues.apache.org/jira/browse/HDFS-14063) | Support noredirect param for CREATE/APPEND/OPEN/GETFILECHECKSUM in HttpFS | Major | . | Íñigo Goiri | Íñigo Goiri | +| [YARN-8860](https://issues.apache.org/jira/browse/YARN-8860) | Federation client intercepter class contains unwanted character | Minor | router | Rakesh Shah | Abhishek Modi | +| [HDFS-14015](https://issues.apache.org/jira/browse/HDFS-14015) | Improve error handling in hdfsThreadDestructor in native thread local storage | Major | native | Daniel Templeton | Daniel Templeton | +| [HADOOP-15919](https://issues.apache.org/jira/browse/HADOOP-15919) | AliyunOSS: Enable Yarn to use OSS | Major | fs/oss | wujinhu | wujinhu | +| [HADOOP-14739](https://issues.apache.org/jira/browse/HADOOP-14739) | Update start-build-env.sh and build instruction for docker for Mac instead of docker toolbox. | Minor | build, documentation | Akira Ajisaka | Dinesh Chitlangia | +| [HDFS-14064](https://issues.apache.org/jira/browse/HDFS-14064) | WEBHDFS: Support Enable/Disable EC Policy | Major | erasure-coding, webhdfs | Ayush Saxena | Ayush Saxena | +| [YARN-8964](https://issues.apache.org/jira/browse/YARN-8964) | [UI2] YARN ui2 should use clusters/{cluster name} for all ATSv2 REST APIs | Major | . | Rohith Sharma K S | Akhil PB | +| [HADOOP-15943](https://issues.apache.org/jira/browse/HADOOP-15943) | AliyunOSS: add missing owner & group attributes for oss FileStatus | Major | fs/oss | wujinhu | wujinhu | +| [HDFS-14108](https://issues.apache.org/jira/browse/HDFS-14108) | Performance improvement in BlockManager Data Structures | Minor | hdfs | David Mollitor | David Mollitor | +| [HDFS-14102](https://issues.apache.org/jira/browse/HDFS-14102) | Performance improvement in BlockPlacementPolicyDefault | Minor | . | David Mollitor | David Mollitor | +| [MAPREDUCE-7164](https://issues.apache.org/jira/browse/MAPREDUCE-7164) | FileOutputCommitter does not report progress while merging paths. | Major | . | Kuhu Shukla | Kuhu Shukla | +| [YARN-8975](https://issues.apache.org/jira/browse/YARN-8975) | [Submarine] Use predefined Charset object StandardCharsets.UTF\_8 instead of String "UTF-8" | Trivial | . | Zhankun Tang | Zhankun Tang | +| [YARN-8974](https://issues.apache.org/jira/browse/YARN-8974) | Improve the assertion message in TestGPUResourceHandler | Trivial | . | Zhankun Tang | Zhankun Tang | +| [YARN-9061](https://issues.apache.org/jira/browse/YARN-9061) | Improve the GPU/FPGA module log message of container-executor | Minor | . | Zhankun Tang | Zhankun Tang | +| [YARN-9069](https://issues.apache.org/jira/browse/YARN-9069) | Fix SchedulerInfo#getSchedulerType for custom schedulers | Minor | . | Bilwa S T | Bilwa S T | +| [HDFS-14095](https://issues.apache.org/jira/browse/HDFS-14095) | EC: Track Erasure Coding commands in DFS statistics | Major | erasure-coding | Ayush Saxena | Ayush Saxena | +| [YARN-9036](https://issues.apache.org/jira/browse/YARN-9036) | Escape newlines in health report in YARN UI | Major | . | Jonathan Hung | Keqiu Hu | +| [HDFS-14106](https://issues.apache.org/jira/browse/HDFS-14106) | Refactor NamenodeFsck#copyBlock | Minor | namenode | David Mollitor | David Mollitor | +| [HDFS-12946](https://issues.apache.org/jira/browse/HDFS-12946) | Add a tool to check rack configuration against EC policies | Major | erasure-coding | Xiao Chen | Kitti Nanasi | +| [HDFS-13818](https://issues.apache.org/jira/browse/HDFS-13818) | Extend OIV to detect FSImage corruption | Major | hdfs | Adam Antal | Adam Antal | +| [HDFS-14119](https://issues.apache.org/jira/browse/HDFS-14119) | Improve GreedyPlanner Parameter Logging | Trivial | hdfs | David Mollitor | David Mollitor | +| [HDFS-14105](https://issues.apache.org/jira/browse/HDFS-14105) | Replace TreeSet in NamenodeFsck with HashSet | Trivial | . | David Mollitor | David Mollitor | +| [YARN-8985](https://issues.apache.org/jira/browse/YARN-8985) | Improve debug log in FSParentQueue when assigning container | Minor | fairscheduler | Wilfred Spiegelenburg | Wilfred Spiegelenburg | +| [HDFS-14113](https://issues.apache.org/jira/browse/HDFS-14113) | EC : Add Configuration to restrict UserDefined Policies | Major | erasure-coding | Ayush Saxena | Ayush Saxena | +| [YARN-9085](https://issues.apache.org/jira/browse/YARN-9085) | Add Guaranteed and MaxCapacity to CSQueueMetrics | Major | . | Jonathan Hung | Jonathan Hung | +| [HDFS-14124](https://issues.apache.org/jira/browse/HDFS-14124) | EC : Support EC Commands (set/get/unset EcPolicy) via WebHdfs | Major | erasure-coding, httpfs, webhdfs | Souryakanta Dwivedy | Ayush Saxena | +| [YARN-9051](https://issues.apache.org/jira/browse/YARN-9051) | Integrate multiple CustomResourceTypesConfigurationProvider implementations into one | Minor | . | Szilard Nemeth | Szilard Nemeth | +| [YARN-9008](https://issues.apache.org/jira/browse/YARN-9008) | Extend YARN distributed shell with file localization feature | Major | yarn | Peter Bacsko | Peter Bacsko | +| [HDFS-13985](https://issues.apache.org/jira/browse/HDFS-13985) | Clearer error message for ReplicaNotFoundException | Major | hdfs | Adam Antal | Adam Antal | +| [HDFS-13970](https://issues.apache.org/jira/browse/HDFS-13970) | Use MultiMap for CacheManager Directives to simplify the code | Minor | caching, hdfs | David Mollitor | David Mollitor | +| [HDFS-14006](https://issues.apache.org/jira/browse/HDFS-14006) | Refactor name node to allow different token verification implementations | Major | . | CR Hota | CR Hota | +| [YARN-9122](https://issues.apache.org/jira/browse/YARN-9122) | Add table of contents to YARN Service API document | Minor | documentation | Akira Ajisaka | Zhankun Tang | +| [HADOOP-16000](https://issues.apache.org/jira/browse/HADOOP-16000) | Remove TLSv1 and SSLv2Hello from the default value of hadoop.ssl.enabled.protocols | Major | security | Akira Ajisaka | Gabor Bota | +| [YARN-9095](https://issues.apache.org/jira/browse/YARN-9095) | Removed Unused field from Resource: NUM\_MANDATORY\_RESOURCES | Minor | . | Szilard Nemeth | Vidura Bhathiya Mudalige | +| [MAPREDUCE-7166](https://issues.apache.org/jira/browse/MAPREDUCE-7166) | map-only job should ignore node lost event when task is already succeeded | Major | mrv2 | Zhaohui Xin | Lei Li | +| [YARN-9130](https://issues.apache.org/jira/browse/YARN-9130) | Add Bind\_HOST configuration for Yarn Web Proxy | Major | yarn | Rong Tang | Rong Tang | +| [HADOOP-15965](https://issues.apache.org/jira/browse/HADOOP-15965) | Upgrade to ADLS SDK which has major performance improvement for ingress/egress | Major | fs/adl | Vishwajeet Dusane | Vishwajeet Dusane | +| [HADOOP-16014](https://issues.apache.org/jira/browse/HADOOP-16014) | Fix test, checkstyle and javadoc issues in TestKerberosAuthenticationHandler | Major | test | Dinesh Chitlangia | Dinesh Chitlangia | +| [HDFS-13946](https://issues.apache.org/jira/browse/HDFS-13946) | Log longest FSN write/read lock held stack trace | Minor | . | Yiqun Lin | Yiqun Lin | +| [HADOOP-15962](https://issues.apache.org/jira/browse/HADOOP-15962) | The buffer size is small when unpacking tar archives | Minor | common, util | David Mollitor | David Mollitor | +| [YARN-8878](https://issues.apache.org/jira/browse/YARN-8878) | Remove StringBuffer from ManagedParentQueue.java | Trivial | resourcemanager | David Mollitor | David Mollitor | +| [YARN-8894](https://issues.apache.org/jira/browse/YARN-8894) | Improve InMemoryPlan#toString | Minor | reservation system | David Mollitor | David Mollitor | +| [HDFS-14171](https://issues.apache.org/jira/browse/HDFS-14171) | Performance improvement in Tailing EditLog | Major | namenode | Kenneth Yang | Kenneth Yang | +| [HDFS-14184](https://issues.apache.org/jira/browse/HDFS-14184) | [SPS] Add support for URI based path in satisfystoragepolicy command | Major | . | Ayush Saxena | Ayush Saxena | +| [HDFS-14163](https://issues.apache.org/jira/browse/HDFS-14163) | Debug Admin Command Should Support Generic Options. | Major | . | Harshakiran Reddy | Ayush Saxena | +| [YARN-6523](https://issues.apache.org/jira/browse/YARN-6523) | Optimize system credentials sent in node heartbeat responses | Major | RM | Naganarasimha G R | Manikandan R | +| [HADOOP-15909](https://issues.apache.org/jira/browse/HADOOP-15909) | KeyProvider class should implement Closeable | Major | kms | Kuhu Shukla | Kuhu Shukla | +| [HADOOP-16029](https://issues.apache.org/jira/browse/HADOOP-16029) | Consecutive StringBuilder.append can be reused | Major | . | Ayush Saxena | Ayush Saxena | +| [HADOOP-15481](https://issues.apache.org/jira/browse/HADOOP-15481) | Emit FairCallQueue stats as metrics | Major | metrics, rpc-server | Erik Krogen | Christopher Gregorian | +| [HADOOP-15994](https://issues.apache.org/jira/browse/HADOOP-15994) | Upgrade Jackson2 to 2.9.8 | Major | security | Akira Ajisaka | lqjacklee | +| [HDFS-14213](https://issues.apache.org/jira/browse/HDFS-14213) | Remove Jansson from BUILDING.txt | Minor | documentation | Akira Ajisaka | Dinesh Chitlangia | +| [HDFS-14221](https://issues.apache.org/jira/browse/HDFS-14221) | Replace Guava Optional with Java Optional | Major | . | Arpit Agarwal | Arpit Agarwal | +| [HDFS-14222](https://issues.apache.org/jira/browse/HDFS-14222) | Make ThrottledAsyncChecker constructor public | Major | . | Arpit Agarwal | Arpit Agarwal | +| [HDFS-14153](https://issues.apache.org/jira/browse/HDFS-14153) | [SPS] : Add Support for Storage Policy Satisfier in WEBHDFS | Major | . | Ayush Saxena | Ayush Saxena | +| [HDFS-14061](https://issues.apache.org/jira/browse/HDFS-14061) | Check if the cluster topology supports the EC policy before setting, enabling or adding it | Major | erasure-coding, hdfs | Kitti Nanasi | Kitti Nanasi | +| [HDFS-14185](https://issues.apache.org/jira/browse/HDFS-14185) | Cleanup method calls to static Assert methods in TestAddStripedBlocks | Minor | hdfs | Shweta | Shweta | +| [HADOOP-16075](https://issues.apache.org/jira/browse/HADOOP-16075) | Upgrade checkstyle version to 8.16 | Minor | build | Dinesh Chitlangia | Dinesh Chitlangia | +| [HDFS-14187](https://issues.apache.org/jira/browse/HDFS-14187) | Make warning message more clear when there are not enough data nodes for EC write | Major | erasure-coding | Kitti Nanasi | Kitti Nanasi | +| [HADOOP-16089](https://issues.apache.org/jira/browse/HADOOP-16089) | AliyunOSS: update oss-sdk version to 3.4.1 | Major | fs/oss | wujinhu | wujinhu | +| [HDFS-14125](https://issues.apache.org/jira/browse/HDFS-14125) | Use parameterized log format in ECTopologyVerifier | Trivial | erasure-coding | Kitti Nanasi | Kitti Nanasi | +| [HDFS-14231](https://issues.apache.org/jira/browse/HDFS-14231) | DataXceiver#run() should not log exceptions caused by InvalidToken exception as an error | Major | hdfs | Kitti Nanasi | Kitti Nanasi | +| [HDFS-14250](https://issues.apache.org/jira/browse/HDFS-14250) | [SBN read] msync should sync with active NameNode to fetch the latest stateID | Major | namenode | Chao Sun | Chao Sun | +| [YARN-8219](https://issues.apache.org/jira/browse/YARN-8219) | Add application launch time to ATSV2 | Major | timelineserver | Kanwaljeet Sachdev | Abhishek Modi | +| [YARN-7171](https://issues.apache.org/jira/browse/YARN-7171) | RM UI should sort memory / cores numerically | Major | . | Eric Maynard | Ahmed Hussein | +| [HDFS-14172](https://issues.apache.org/jira/browse/HDFS-14172) | Avoid NPE when SectionName#fromString returns null | Minor | . | Xiang Li | Xiang Li | +| [YARN-9282](https://issues.apache.org/jira/browse/YARN-9282) | Typo in javadoc of class LinuxContainerExecutor: hadoop.security.authetication should be 'authentication' | Trivial | . | Szilard Nemeth | Charan Hebri | +| [HDFS-14260](https://issues.apache.org/jira/browse/HDFS-14260) | Replace synchronized method in BlockReceiver with atomic value | Minor | datanode | David Mollitor | David Mollitor | +| [HADOOP-16097](https://issues.apache.org/jira/browse/HADOOP-16097) | Provide proper documentation for FairCallQueue | Major | documentation, ipc | Erik Krogen | Erik Krogen | +| [HADOOP-16108](https://issues.apache.org/jira/browse/HADOOP-16108) | Tail Follow Interval Should Allow To Specify The Sleep Interval To Save Unnecessary RPC's | Major | . | Harshakiran Reddy | Ayush Saxena | +| [HDFS-14241](https://issues.apache.org/jira/browse/HDFS-14241) | Provide feedback on successful renameSnapshot and deleteSnapshot | Minor | hdfs, shell | Siyao Meng | Siyao Meng | +| [HDFS-13209](https://issues.apache.org/jira/browse/HDFS-13209) | DistributedFileSystem.create should allow an option to provide StoragePolicy | Major | hdfs | Jean-Marc Spaggiari | Ayush Saxena | +| [YARN-8295](https://issues.apache.org/jira/browse/YARN-8295) | [UI2] Improve "Resource Usage" tab error message when there are no data available. | Minor | yarn-ui-v2 | Gergely Novák | Charan Hebri | +| [YARN-8927](https://issues.apache.org/jira/browse/YARN-8927) | Support trust top-level image like "centos" when "library" is configured in "docker.trusted.registries" | Major | . | Zhankun Tang | Zhankun Tang | +| [HDFS-14258](https://issues.apache.org/jira/browse/HDFS-14258) | Introduce Java Concurrent Package To DataXceiverServer Class | Minor | datanode | David Mollitor | David Mollitor | +| [YARN-7824](https://issues.apache.org/jira/browse/YARN-7824) | [UI2] Yarn Component Instance page should include link to container logs | Major | yarn-ui-v2 | Yesha Vora | Akhil PB | +| [HADOOP-15281](https://issues.apache.org/jira/browse/HADOOP-15281) | Distcp to add no-rename copy option | Major | tools/distcp | Steve Loughran | Andrew Olson | +| [HDFS-9596](https://issues.apache.org/jira/browse/HDFS-9596) | Remove Shuffle Method From DFSUtil | Trivial | . | David Mollitor | David Mollitor | +| [HDFS-14296](https://issues.apache.org/jira/browse/HDFS-14296) | Prefer ArrayList over LinkedList in VolumeScanner | Minor | datanode | David Mollitor | David Mollitor | +| [YARN-9309](https://issues.apache.org/jira/browse/YARN-9309) | Improve graph text in SLS to avoid overlapping | Minor | . | Bilwa S T | Bilwa S T | +| [HDFS-14188](https://issues.apache.org/jira/browse/HDFS-14188) | Make hdfs ec -verifyClusterSetup command accept an erasure coding policy as a parameter | Major | erasure-coding | Kitti Nanasi | Kitti Nanasi | +| [HADOOP-15967](https://issues.apache.org/jira/browse/HADOOP-15967) | KMS Benchmark Tool | Major | . | Wei-Chiu Chuang | George Huang | +| [HDFS-14286](https://issues.apache.org/jira/browse/HDFS-14286) | Logging stale datanode information | Trivial | hdfs | Karthik Palanisamy | Karthik Palanisamy | +| [HDFS-14235](https://issues.apache.org/jira/browse/HDFS-14235) | Handle ArrayIndexOutOfBoundsException in DataNodeDiskMetrics#slowDiskDetectionDaemon | Major | . | Surendra Singh Lilhore | Ranith Sardar | +| [HDFS-14267](https://issues.apache.org/jira/browse/HDFS-14267) | Add test\_libhdfs\_ops to libhdfs tests, mark libhdfs\_read/write.c as examples | Major | libhdfs, native, test | Sahil Takiar | Sahil Takiar | +| [HDFS-14302](https://issues.apache.org/jira/browse/HDFS-14302) | Refactor NameNodeWebHdfsMethods#generateDelegationToken() to allow better extensibility | Major | . | CR Hota | CR Hota | +| [HADOOP-16035](https://issues.apache.org/jira/browse/HADOOP-16035) | Jenkinsfile for Hadoop | Major | build | Allen Wittenauer | Allen Wittenauer | +| [HDFS-14298](https://issues.apache.org/jira/browse/HDFS-14298) | Improve log messages of ECTopologyVerifier | Minor | . | Kitti Nanasi | Kitti Nanasi | +| [YARN-9168](https://issues.apache.org/jira/browse/YARN-9168) | DistributedShell client timeout should be -1 by default | Minor | . | Zhankun Tang | Zhankun Tang | +| [HDFS-7133](https://issues.apache.org/jira/browse/HDFS-7133) | Support clearing namespace quota on "/" | Major | namenode | Guo Ruijing | Ayush Saxena | +| [HADOOP-16126](https://issues.apache.org/jira/browse/HADOOP-16126) | ipc.Client.stop() may sleep too long to wait for all connections | Major | ipc | Tsz-wo Sze | Tsz-wo Sze | +| [YARN-9287](https://issues.apache.org/jira/browse/YARN-9287) | Consecutive StringBuilder append should be reused | Major | . | Ayush Saxena | Ayush Saxena | +| [YARN-9087](https://issues.apache.org/jira/browse/YARN-9087) | Improve logging for initialization of Resource plugins | Major | yarn | Szilard Nemeth | Szilard Nemeth | +| [YARN-9121](https://issues.apache.org/jira/browse/YARN-9121) | Replace GpuDiscoverer.getInstance() to a readable object for easy access control | Major | . | Szilard Nemeth | Szilard Nemeth | +| [YARN-9322](https://issues.apache.org/jira/browse/YARN-9322) | Store metrics for custom resource types into FSQueueMetrics and query them in FairSchedulerQueueInfo | Major | . | Szilard Nemeth | Szilard Nemeth | +| [YARN-9139](https://issues.apache.org/jira/browse/YARN-9139) | Simplify initializer code of GpuDiscoverer | Major | . | Szilard Nemeth | Szilard Nemeth | +| [HDFS-14247](https://issues.apache.org/jira/browse/HDFS-14247) | Repeat adding node description into network topology | Minor | datanode | HuangTao | HuangTao | +| [YARN-9332](https://issues.apache.org/jira/browse/YARN-9332) | RackResolver tool should accept multiple hosts | Minor | yarn | Lantao Jin | Lantao Jin | +| [HDFS-14182](https://issues.apache.org/jira/browse/HDFS-14182) | Datanode usage histogram is clicked to show ip list | Major | . | fengchuang | fengchuang | +| [HDFS-14321](https://issues.apache.org/jira/browse/HDFS-14321) | Fix -Xcheck:jni issues in libhdfs, run ctest with -Xcheck:jni enabled | Major | hdfs-client, libhdfs, native | Sahil Takiar | Sahil Takiar | +| [HADOOP-16148](https://issues.apache.org/jira/browse/HADOOP-16148) | Cleanup LineReader Unit Test | Trivial | common | David Mollitor | David Mollitor | +| [HADOOP-16162](https://issues.apache.org/jira/browse/HADOOP-16162) | Remove unused Job Summary Appender configurations from log4j.properties | Major | conf | Chen Zhi | Chen Zhi | +| [HDFS-14336](https://issues.apache.org/jira/browse/HDFS-14336) | Fix checkstyle for NameNodeMXBean | Trivial | namenode | Danny Becker | Danny Becker | +| [YARN-9298](https://issues.apache.org/jira/browse/YARN-9298) | Implement FS placement rules using PlacementRule interface | Major | scheduler | Wilfred Spiegelenburg | Wilfred Spiegelenburg | +| [HDFS-14326](https://issues.apache.org/jira/browse/HDFS-14326) | Add CorruptFilesCount to JMX | Minor | fs, metrics, namenode | Danny Becker | Danny Becker | +| [YARN-9138](https://issues.apache.org/jira/browse/YARN-9138) | Improve test coverage for nvidia-smi binary execution of GpuDiscoverer | Major | . | Szilard Nemeth | Szilard Nemeth | +| [YARN-8218](https://issues.apache.org/jira/browse/YARN-8218) | Add application launch time to ATSV1 | Major | . | Kanwaljeet Sachdev | Abhishek Modi | +| [YARN-9150](https://issues.apache.org/jira/browse/YARN-9150) | Making TimelineSchemaCreator support different backends for Timeline Schema Creation in ATSv2 | Major | ATSv2 | Sushil Ks | Sushil Ks | +| [HADOOP-16157](https://issues.apache.org/jira/browse/HADOOP-16157) | [Clean-up] Remove NULL check before instanceof in AzureNativeFileSystemStore | Minor | tools | Shweta | Shweta | +| [MAPREDUCE-7191](https://issues.apache.org/jira/browse/MAPREDUCE-7191) | JobHistoryServer should log exception when loading/parsing history file failed | Minor | mrv2 | Jiandan Yang | Jiandan Yang | +| [YARN-9381](https://issues.apache.org/jira/browse/YARN-9381) | The yarn-default.xml has two identical property named yarn.timeline-service.http-cross-origin.enabled | Trivial | yarn | jenny | Abhishek Modi | +| [MAPREDUCE-7192](https://issues.apache.org/jira/browse/MAPREDUCE-7192) | JobHistoryServer attempts page support jump to containers log page in NM when logAggregation is disable | Major | mrv2 | Jiandan Yang | Jiandan Yang | +| [HDFS-14346](https://issues.apache.org/jira/browse/HDFS-14346) | Better time precision in getTimeDuration | Minor | namenode | Chao Sun | Chao Sun | +| [HDFS-14366](https://issues.apache.org/jira/browse/HDFS-14366) | Improve HDFS append performance | Major | hdfs | Chao Sun | Chao Sun | +| [YARN-4404](https://issues.apache.org/jira/browse/YARN-4404) | Typo in comment in SchedulerUtils | Trivial | resourcemanager | Daniel Templeton | Yesha Vora | +| [MAPREDUCE-7188](https://issues.apache.org/jira/browse/MAPREDUCE-7188) | [Clean-up] Remove NULL check before instanceof and fix checkstyle issue in TaskResult | Minor | . | Shweta | Shweta | +| [YARN-9340](https://issues.apache.org/jira/browse/YARN-9340) | [Clean-up] Remove NULL check before instanceof in ResourceRequestSetKey | Minor | yarn | Shweta | Shweta | +| [HDFS-14328](https://issues.apache.org/jira/browse/HDFS-14328) | [Clean-up] Remove NULL check before instanceof in TestGSet | Minor | . | Shweta | Shweta | +| [HADOOP-16167](https://issues.apache.org/jira/browse/HADOOP-16167) | "hadoop CLASSFILE" prints error messages on Ubuntu 18 | Major | scripts | Daniel Templeton | Daniel Templeton | +| [YARN-9385](https://issues.apache.org/jira/browse/YARN-9385) | YARN Services with simple authentication doesn't respect current UGI | Major | security, yarn-native-services | Todd Lipcon | Eric Yang | +| [HDFS-14211](https://issues.apache.org/jira/browse/HDFS-14211) | [Consistent Observer Reads] Allow for configurable "always msync" mode | Major | hdfs-client | Erik Krogen | Erik Krogen | +| [YARN-9370](https://issues.apache.org/jira/browse/YARN-9370) | Better logging in recoverAssignedGpus in class GpuResourceAllocator | Trivial | . | Szilard Nemeth | Yesha Vora | +| [HADOOP-16147](https://issues.apache.org/jira/browse/HADOOP-16147) | Allow CopyListing sequence file keys and values to be more easily customized | Major | tools/distcp | Andrew Olson | Andrew Olson | +| [YARN-9358](https://issues.apache.org/jira/browse/YARN-9358) | Add javadoc to new methods introduced in FSQueueMetrics with YARN-9322 | Major | . | Szilard Nemeth | Zoltan Siegl | +| [YARN-8967](https://issues.apache.org/jira/browse/YARN-8967) | Change FairScheduler to use PlacementRule interface | Major | capacityscheduler, fairscheduler | Wilfred Spiegelenburg | Wilfred Spiegelenburg | +| [HDFS-14304](https://issues.apache.org/jira/browse/HDFS-14304) | High lock contention on hdfsHashMutex in libhdfs | Major | hdfs-client, libhdfs, native | Sahil Takiar | Sahil Takiar | +| [HDFS-14295](https://issues.apache.org/jira/browse/HDFS-14295) | Add Threadpool for DataTransfers | Major | datanode | David Mollitor | David Mollitor | +| [HDFS-14395](https://issues.apache.org/jira/browse/HDFS-14395) | Remove WARN Logging From Interrupts | Minor | hdfs-client | David Mollitor | David Mollitor | +| [YARN-9264](https://issues.apache.org/jira/browse/YARN-9264) | [Umbrella] Follow-up on IntelOpenCL FPGA plugin | Major | nodemanager | Peter Bacsko | Peter Bacsko | +| [MAPREDUCE-7190](https://issues.apache.org/jira/browse/MAPREDUCE-7190) | Add SleepJob additional parameter to make parallel runs distinguishable | Major | . | Adam Antal | Adam Antal | +| [YARN-9214](https://issues.apache.org/jira/browse/YARN-9214) | Add AbstractYarnScheduler#getValidQueues method to remove duplication | Major | . | Wanqiang Ji | Wanqiang Ji | +| [HDFS-13960](https://issues.apache.org/jira/browse/HDFS-13960) | hdfs dfs -checksum command should optionally show block size in output | Minor | hdfs | Adam Antal | Lokesh Jain | +| [HDFS-14327](https://issues.apache.org/jira/browse/HDFS-14327) | Using FQDN instead of IP to access servers with DNS resolving | Major | . | Fengnan Li | Fengnan Li | +| [YARN-9394](https://issues.apache.org/jira/browse/YARN-9394) | Use new API of RackResolver to get better performance | Major | yarn | Lantao Jin | Lantao Jin | +| [HADOOP-16208](https://issues.apache.org/jira/browse/HADOOP-16208) | Do Not Log InterruptedException in Client | Minor | common | David Mollitor | David Mollitor | +| [HDFS-14371](https://issues.apache.org/jira/browse/HDFS-14371) | Improve Logging in FSNamesystem by adding parameterized logging | Minor | hdfs | Shweta | Shweta | +| [HADOOP-10848](https://issues.apache.org/jira/browse/HADOOP-10848) | Cleanup calling of sun.security.krb5.Config | Minor | . | Kai Zheng | Akira Ajisaka | +| [YARN-9463](https://issues.apache.org/jira/browse/YARN-9463) | Add queueName info when failing with queue capacity sanity check | Trivial | capacity scheduler | Aihua Xu | Aihua Xu | +| [HADOOP-16179](https://issues.apache.org/jira/browse/HADOOP-16179) | hadoop-common pom should not depend on kerb-simplekdc | Major | common | Todd Lipcon | Todd Lipcon | +| [HADOOP-16052](https://issues.apache.org/jira/browse/HADOOP-16052) | Remove Subversion and Forrest from Dockerfile | Minor | build | Akira Ajisaka | Xieming Li | +| [HADOOP-16243](https://issues.apache.org/jira/browse/HADOOP-16243) | Change Log Level to trace in NetUtils.java | Major | . | Bharat Viswanadham | chencan | +| [HADOOP-16227](https://issues.apache.org/jira/browse/HADOOP-16227) | Upgrade checkstyle to 8.19 | Major | build | Akira Ajisaka | Akira Ajisaka | +| [HADOOP-16249](https://issues.apache.org/jira/browse/HADOOP-16249) | Make CallerContext LimitedPrivate scope to Public | Minor | ipc | Kenneth Yang | Kenneth Yang | +| [HADOOP-15014](https://issues.apache.org/jira/browse/HADOOP-15014) | KMS should log the IP address of the clients | Major | kms | Zsombor Gegesy | Zsombor Gegesy | +| [YARN-9123](https://issues.apache.org/jira/browse/YARN-9123) | Clean up and split testcases in TestNMWebServices for GPU support | Minor | . | Szilard Nemeth | Szilard Nemeth | +| [YARN-9349](https://issues.apache.org/jira/browse/YARN-9349) | When doTransition() method occurs exception, the log level practices are inconsistent | Major | nodemanager | Anuhan Torgonshar | | +| [HDFS-14432](https://issues.apache.org/jira/browse/HDFS-14432) | dfs.datanode.shared.file.descriptor.paths duplicated in hdfs-default.xml | Minor | hdfs | puleya7 | puleya7 | +| [HDFS-14374](https://issues.apache.org/jira/browse/HDFS-14374) | Expose total number of delegation tokens in AbstractDelegationTokenSecretManager | Major | . | CR Hota | CR Hota | +| [HADOOP-16026](https://issues.apache.org/jira/browse/HADOOP-16026) | Replace incorrect use of system property user.name | Major | . | Dinesh Chitlangia | Dinesh Chitlangia | +| [YARN-9081](https://issues.apache.org/jira/browse/YARN-9081) | Update jackson from 1.9.13 to 2.x in hadoop-yarn-services-core | Major | . | Akira Ajisaka | Akira Ajisaka | +| [HDFS-3246](https://issues.apache.org/jira/browse/HDFS-3246) | pRead equivalent for direct read path | Major | hdfs-client, performance | Henry Robinson | Sahil Takiar | +| [HDFS-14463](https://issues.apache.org/jira/browse/HDFS-14463) | Add Log Level link under NameNode and DataNode Web UI Utilities dropdown | Trivial | webhdfs | Siyao Meng | Siyao Meng | +| [HDFS-14460](https://issues.apache.org/jira/browse/HDFS-14460) | DFSUtil#getNamenodeWebAddr should return HTTPS address based on policy configured | Major | . | CR Hota | CR Hota | +| [HADOOP-16282](https://issues.apache.org/jira/browse/HADOOP-16282) | Avoid FileStream to improve performance | Major | . | Ayush Saxena | Ayush Saxena | +| [HDFS-14453](https://issues.apache.org/jira/browse/HDFS-14453) | Improve Bad Sequence Number Error Message | Minor | ipc | David Mollitor | Shweta | +| [HADOOP-16059](https://issues.apache.org/jira/browse/HADOOP-16059) | Use SASL Factories Cache to Improve Performance | Critical | . | Ayush Saxena | Ayush Saxena | +| [HADOOP-16292](https://issues.apache.org/jira/browse/HADOOP-16292) | Refactor checkTrustAndSend in SaslDataTransferClient to make it cleaner | Major | . | Sherwood Zheng | Sherwood Zheng | +| [YARN-9529](https://issues.apache.org/jira/browse/YARN-9529) | Log correct cpu controller path on error while initializing CGroups. | Major | nodemanager | Jonathan Hung | Jonathan Hung | +| [HADOOP-16289](https://issues.apache.org/jira/browse/HADOOP-16289) | Allow extra jsvc startup option in hadoop\_start\_secure\_daemon in hadoop-functions.sh | Major | scripts | Siyao Meng | Siyao Meng | +| [HADOOP-16238](https://issues.apache.org/jira/browse/HADOOP-16238) | Add the possbility to set SO\_REUSEADDR in IPC Server Listener | Minor | ipc | Peter Bacsko | Peter Bacsko | +| [YARN-9453](https://issues.apache.org/jira/browse/YARN-9453) | Clean up code long if-else chain in ApplicationCLI#run | Major | . | Szilard Nemeth | Wanqiang Ji | +| [YARN-9546](https://issues.apache.org/jira/browse/YARN-9546) | Add configuration option for YARN Native services AM classpath | Major | . | Gergely Pollak | Gergely Pollak | +| [HDFS-14507](https://issues.apache.org/jira/browse/HDFS-14507) | Document -blockingDecommission option for hdfs dfsadmin -listOpenFiles | Minor | documentation | Siyao Meng | Siyao Meng | +| [YARN-9145](https://issues.apache.org/jira/browse/YARN-9145) | [Umbrella] Dynamically add or remove auxiliary services | Major | nodemanager | Billie Rinaldi | Billie Rinaldi | +| [HDFS-14451](https://issues.apache.org/jira/browse/HDFS-14451) | Incorrect header or version mismatch log message | Minor | ipc | David Mollitor | Shweta | +| [HDFS-14502](https://issues.apache.org/jira/browse/HDFS-14502) | keepResults option in NNThroughputBenchmark should call saveNamespace() | Major | benchmarks, hdfs | Konstantin Shvachko | Konstantin Shvachko | +| [HADOOP-16323](https://issues.apache.org/jira/browse/HADOOP-16323) | https everywhere in Maven settings | Minor | build | Akira Ajisaka | Akira Ajisaka | +| [YARN-9563](https://issues.apache.org/jira/browse/YARN-9563) | Resource report REST API could return NaN or Inf | Minor | . | Ahmed Hussein | Ahmed Hussein | +| [HDFS-13654](https://issues.apache.org/jira/browse/HDFS-13654) | Use a random secret when a secret file doesn't exist in HttpFS. This should be default. | Major | httpfs, security | Pulkit Bhardwaj | Takanobu Asanuma | +| [YARN-9592](https://issues.apache.org/jira/browse/YARN-9592) | Use Logger format in ContainersMonitorImpl | Major | . | Íñigo Goiri | Íñigo Goiri | +| [YARN-9545](https://issues.apache.org/jira/browse/YARN-9545) | Create healthcheck REST endpoint for ATSv2 | Major | ATSv2 | Zoltan Siegl | Zoltan Siegl | +| [HADOOP-16344](https://issues.apache.org/jira/browse/HADOOP-16344) | Make DurationInfo "public unstable" | Minor | util | Kevin Risden | Kevin Su | +| [YARN-9471](https://issues.apache.org/jira/browse/YARN-9471) | Cleanup in TestLogAggregationIndexFileController | Major | log-aggregation, yarn | Adam Antal | Adam Antal | +| [HDFS-10659](https://issues.apache.org/jira/browse/HDFS-10659) | Namenode crashes after Journalnode re-installation in an HA cluster due to missing paxos directory | Major | ha, journal-node | Amit Anand | star | +| [HDFS-10210](https://issues.apache.org/jira/browse/HDFS-10210) | Remove the defunct startKdc profile from hdfs | Major | test | Wei-Chiu Chuang | Wei-Chiu Chuang | +| [YARN-9569](https://issues.apache.org/jira/browse/YARN-9569) | Auto-created leaf queues do not honor cluster-wide min/max memory/vcores | Major | capacity scheduler | Craig Condit | Craig Condit | +| [YARN-9602](https://issues.apache.org/jira/browse/YARN-9602) | Use logger format in Container Executor. | Major | . | Abhishek Modi | Abhishek Modi | +| [HDFS-14513](https://issues.apache.org/jira/browse/HDFS-14513) | FSImage which is saving should be clean while NameNode shutdown | Major | namenode | Xiaoqiao He | Xiaoqiao He | +| [YARN-9543](https://issues.apache.org/jira/browse/YARN-9543) | [UI2] Handle ATSv2 server down or failures cases gracefully in YARN UI v2 | Major | ATSv2, yarn-ui-v2 | Zoltan Siegl | Zoltan Siegl | +| [HADOOP-16369](https://issues.apache.org/jira/browse/HADOOP-16369) | Fix zstandard shortname misspelled as zts | Major | . | Jonathan Turner Eagles | Jonathan Turner Eagles | +| [HDFS-14560](https://issues.apache.org/jira/browse/HDFS-14560) | Allow block replication parameters to be refreshable | Major | namenode | Stephen O'Donnell | Stephen O'Donnell | +| [HDFS-14203](https://issues.apache.org/jira/browse/HDFS-14203) | Refactor OIV Delimited output entry building mechanism | Minor | tools | Adam Antal | Adam Antal | +| [HADOOP-14807](https://issues.apache.org/jira/browse/HADOOP-14807) | should prevent the possibility of NPE about ReconfigurableBase.java | Minor | . | hu xiaodong | hu xiaodong | +| [HDFS-12770](https://issues.apache.org/jira/browse/HDFS-12770) | Add doc about how to disable client socket cache | Trivial | hdfs-client | Weiwei Yang | Weiwei Yang | +| [HADOOP-9157](https://issues.apache.org/jira/browse/HADOOP-9157) | Better option for curl in hadoop-auth-examples | Minor | documentation | Jingguo Yao | Andras Bokor | +| [HDFS-14340](https://issues.apache.org/jira/browse/HDFS-14340) | Lower the log level when can't get postOpAttr | Minor | nfs | Anuhan Torgonshar | Anuhan Torgonshar | +| [HADOOP-15914](https://issues.apache.org/jira/browse/HADOOP-15914) | hadoop jar command has no help argument | Major | common | Adam Antal | Adam Antal | +| [YARN-9630](https://issues.apache.org/jira/browse/YARN-9630) | [UI2] Add a link in docs's top page | Major | documentation, yarn-ui-v2 | Wanqiang Ji | Wanqiang Ji | +| [HADOOP-16156](https://issues.apache.org/jira/browse/HADOOP-16156) | [Clean-up] Remove NULL check before instanceof and fix checkstyle in InnerNodeImpl | Minor | . | Shweta | Shweta | +| [HDFS-14201](https://issues.apache.org/jira/browse/HDFS-14201) | Ability to disallow safemode NN to become active | Major | auto-failover | Xiao Liang | Xiao Liang | +| [HDFS-14487](https://issues.apache.org/jira/browse/HDFS-14487) | Missing Space in Client Error Message | Minor | hdfs-client | David Mollitor | Shweta | +| [HDFS-14398](https://issues.apache.org/jira/browse/HDFS-14398) | Update HAState.java to fix typos. | Trivial | namenode | bianqi | Nikhil Navadiya | +| [HDFS-14103](https://issues.apache.org/jira/browse/HDFS-14103) | Review Logging of BlockPlacementPolicyDefault | Minor | . | David Mollitor | David Mollitor | +| [YARN-9631](https://issues.apache.org/jira/browse/YARN-9631) | hadoop-yarn-applications-catalog-webapp doesn't respect mvn test -D parameter | Major | . | Wei-Chiu Chuang | Eric Yang | +| [HADOOP-14385](https://issues.apache.org/jira/browse/HADOOP-14385) | HttpExceptionUtils#validateResponse swallows exceptions | Trivial | . | Wei-Chiu Chuang | Wei-Chiu Chuang | +| [HDFS-12564](https://issues.apache.org/jira/browse/HDFS-12564) | Add the documents of swebhdfs configurations on the client side | Major | documentation, webhdfs | Takanobu Asanuma | Takanobu Asanuma | +| [HDFS-14403](https://issues.apache.org/jira/browse/HDFS-14403) | Cost-Based RPC FairCallQueue | Major | ipc, namenode | Erik Krogen | Christopher Gregorian | +| [HADOOP-16266](https://issues.apache.org/jira/browse/HADOOP-16266) | Add more fine-grained processing time metrics to the RPC layer | Minor | ipc | Christopher Gregorian | Erik Krogen | +| [HADOOP-16350](https://issues.apache.org/jira/browse/HADOOP-16350) | Ability to tell HDFS client not to request KMS Information from NameNode | Major | common, kms | Greg Senia | | +| [HADOOP-16396](https://issues.apache.org/jira/browse/HADOOP-16396) | Allow authoritative mode on a subdirectory | Major | fs/s3 | Sean Mackrory | Sean Mackrory | +| [YARN-9629](https://issues.apache.org/jira/browse/YARN-9629) | Support configurable MIN\_LOG\_ROLLING\_INTERVAL | Minor | log-aggregation, nodemanager, yarn | Adam Antal | Adam Antal | +| [HDFS-13694](https://issues.apache.org/jira/browse/HDFS-13694) | Making md5 computing being in parallel with image loading | Major | . | zhouyingchao | Lisheng Sun | +| [HADOOP-16409](https://issues.apache.org/jira/browse/HADOOP-16409) | Allow authoritative mode on non-qualified paths | Major | fs/s3 | Sean Mackrory | Sean Mackrory | +| [HDFS-14632](https://issues.apache.org/jira/browse/HDFS-14632) | Reduce useless #getNumLiveDataNodes call in SafeModeMonitor | Major | namenode | Xiaoqiao He | Xiaoqiao He | +| [HDFS-14624](https://issues.apache.org/jira/browse/HDFS-14624) | When decommissioning a node, log remaining blocks to replicate periodically | Major | namenode | Stephen O'Donnell | Stephen O'Donnell | +| [YARN-9573](https://issues.apache.org/jira/browse/YARN-9573) | DistributedShell cannot specify LogAggregationContext | Major | distributed-shell, log-aggregation, yarn | Adam Antal | Adam Antal | +| [YARN-9337](https://issues.apache.org/jira/browse/YARN-9337) | GPU auto-discovery script runs even when the resource is given by hand | Major | yarn | Adam Antal | Adam Antal | +| [YARN-9360](https://issues.apache.org/jira/browse/YARN-9360) | Do not expose innards of QueueMetrics object into FSLeafQueue#computeMaxAMResource | Major | . | Szilard Nemeth | Peter Bacsko | +| [YARN-9127](https://issues.apache.org/jira/browse/YARN-9127) | Create more tests to verify GpuDeviceInformationParser | Major | . | Szilard Nemeth | Peter Bacsko | +| [YARN-9326](https://issues.apache.org/jira/browse/YARN-9326) | Fair Scheduler configuration defaults are not documented in case of min and maxResources | Major | docs, documentation, fairscheduler, yarn | Adam Antal | Adam Antal | +| [HDFS-14547](https://issues.apache.org/jira/browse/HDFS-14547) | DirectoryWithQuotaFeature.quota costs additional memory even the storage type quota is not set. | Major | . | Jinglun | Jinglun | +| [HDFS-13693](https://issues.apache.org/jira/browse/HDFS-13693) | Remove unnecessary search in INodeDirectory.addChild during image loading | Major | namenode | zhouyingchao | Lisheng Sun | +| [HADOOP-16431](https://issues.apache.org/jira/browse/HADOOP-16431) | Remove useless log in IOUtils.java and ExceptionDiags.java | Minor | . | Lisheng Sun | Lisheng Sun | +| [HDFS-14673](https://issues.apache.org/jira/browse/HDFS-14673) | The console log is noisy when using DNSDomainNameResolver to resolve NameNode. | Minor | . | Akira Ajisaka | Kevin Su | +| [HDFS-12967](https://issues.apache.org/jira/browse/HDFS-12967) | NNBench should support multi-cluster access | Major | benchmarks | Chen Zhang | Chen Zhang | +| [HADOOP-16452](https://issues.apache.org/jira/browse/HADOOP-16452) | Increase ipc.maximum.data.length default from 64MB to 128MB | Major | ipc | Wei-Chiu Chuang | Siyao Meng | +| [HDFS-14449](https://issues.apache.org/jira/browse/HDFS-14449) | Expose total number of DT in JMX for Namenode | Major | . | Fengnan Li | Fengnan Li | +| [HDFS-14419](https://issues.apache.org/jira/browse/HDFS-14419) | Avoid repeated calls to the listOpenFiles function | Minor | namenode, performance | HuangTao | HuangTao | +| [HDFS-14683](https://issues.apache.org/jira/browse/HDFS-14683) | WebHDFS: Add erasureCodingPolicy field to GETCONTENTSUMMARY response | Major | . | Siyao Meng | Siyao Meng | +| [YARN-9375](https://issues.apache.org/jira/browse/YARN-9375) | Use Configured in GpuDiscoverer and FpgaDiscoverer | Major | nodemanager, yarn | Adam Antal | Adam Antal | +| [YARN-9093](https://issues.apache.org/jira/browse/YARN-9093) | Remove commented code block from the beginning of TestDefaultContainerExecutor | Trivial | . | Szilard Nemeth | Vidura Bhathiya Mudalige | +| [HADOOP-15942](https://issues.apache.org/jira/browse/HADOOP-15942) | Change the logging level form DEBUG to ERROR for RuntimeErrorException in JMXJsonServlet | Major | common | Anuhan Torgonshar | Anuhan Torgonshar | +| [YARN-9667](https://issues.apache.org/jira/browse/YARN-9667) | Container-executor.c duplicates messages to stdout | Major | nodemanager, yarn | Adam Antal | Peter Bacsko | +| [YARN-9678](https://issues.apache.org/jira/browse/YARN-9678) | TestGpuResourceHandler / TestFpgaResourceHandler should be renamed | Major | . | Szilard Nemeth | Kevin Su | +| [HDFS-14652](https://issues.apache.org/jira/browse/HDFS-14652) | HealthMonitor connection retry times should be configurable | Major | . | Chen Zhang | Chen Zhang | +| [HDFS-14313](https://issues.apache.org/jira/browse/HDFS-14313) | Get hdfs used space from FsDatasetImpl#volumeMap#ReplicaInfo in memory instead of df/du | Major | datanode, performance | Lisheng Sun | Lisheng Sun | +| [HDFS-14608](https://issues.apache.org/jira/browse/HDFS-14608) | DataNode#DataTransfer should be named | Minor | . | Íñigo Goiri | Íñigo Goiri | +| [HDFS-14616](https://issues.apache.org/jira/browse/HDFS-14616) | Add the warn log when the volume available space isn't enough | Minor | hdfs | liying | liying | +| [YARN-9711](https://issues.apache.org/jira/browse/YARN-9711) | Missing spaces in NMClientImpl | Trivial | client | Charles Xu | Charles Xu | +| [HDFS-14662](https://issues.apache.org/jira/browse/HDFS-14662) | Document the usage of the new Balancer "asService" parameter | Major | . | Chen Zhang | Chen Zhang | +| [HDFS-14701](https://issues.apache.org/jira/browse/HDFS-14701) | Change Log Level to warn in SlotReleaser | Minor | . | Lisheng Sun | Lisheng Sun | +| [HDFS-14705](https://issues.apache.org/jira/browse/HDFS-14705) | Remove unused configuration dfs.min.replication | Trivial | . | Wei-Chiu Chuang | CR Hota | +| [HDFS-14693](https://issues.apache.org/jira/browse/HDFS-14693) | NameNode should log a warning when EditLog IPC logger's pending size exceeds limit. | Minor | namenode | Xudong Cao | Xudong Cao | +| [YARN-9094](https://issues.apache.org/jira/browse/YARN-9094) | Remove unused interface method: NodeResourceUpdaterPlugin#handleUpdatedResourceFromRM | Trivial | . | Szilard Nemeth | Gergely Pollak | +| [YARN-9096](https://issues.apache.org/jira/browse/YARN-9096) | Some GpuResourcePlugin and ResourcePluginManager methods are synchronized unnecessarily | Major | . | Szilard Nemeth | Gergely Pollak | +| [YARN-9092](https://issues.apache.org/jira/browse/YARN-9092) | Create an object for cgroups mount enable and cgroups mount path as they belong together | Minor | . | Szilard Nemeth | Gergely Pollak | +| [YARN-9124](https://issues.apache.org/jira/browse/YARN-9124) | Resolve contradiction in ResourceUtils: addMandatoryResources / checkMandatoryResources work differently | Minor | . | Szilard Nemeth | Adam Antal | +| [YARN-8199](https://issues.apache.org/jira/browse/YARN-8199) | Logging fileSize of log files under NM Local Dir | Major | log-aggregation | Prabhu Joseph | Prabhu Joseph | +| [HDFS-14195](https://issues.apache.org/jira/browse/HDFS-14195) | OIV: print out storage policy id in oiv Delimited output | Minor | tools | Wang, Xinglong | Wang, Xinglong | +| [YARN-9729](https://issues.apache.org/jira/browse/YARN-9729) | [UI2] Fix error message for logs when ATSv2 is offline | Major | yarn-ui-v2 | Zoltan Siegl | Zoltan Siegl | +| [YARN-9657](https://issues.apache.org/jira/browse/YARN-9657) | AbstractLivelinessMonitor add serviceName to PingChecker thread | Minor | . | Bibin Chundatt | Bilwa S T | +| [YARN-9715](https://issues.apache.org/jira/browse/YARN-9715) | [UI2] yarn-container-log URI need to be encoded to avoid potential misuses | Major | . | Prabhu Joseph | Akhil PB | +| [HADOOP-16453](https://issues.apache.org/jira/browse/HADOOP-16453) | Update how exceptions are handled in NetUtils | Minor | . | Lisheng Sun | Lisheng Sun | +| [YARN-9464](https://issues.apache.org/jira/browse/YARN-9464) | Support "Pending Resource" metrics in RM's RESTful API | Major | . | Zhankun Tang | Prabhu Joseph | +| [YARN-9135](https://issues.apache.org/jira/browse/YARN-9135) | NM State store ResourceMappings serialization are tested with Strings instead of real Device objects | Major | . | Szilard Nemeth | Peter Bacsko | +| [HDFS-13505](https://issues.apache.org/jira/browse/HDFS-13505) | Turn on HDFS ACLs by default. | Major | . | Ajay Kumar | Siyao Meng | +| [HDFS-14370](https://issues.apache.org/jira/browse/HDFS-14370) | Edit log tailing fast-path should allow for backoff | Major | namenode, qjm | Erik Krogen | Erik Krogen | +| [YARN-9442](https://issues.apache.org/jira/browse/YARN-9442) | container working directory has group read permissions | Minor | yarn | Jim Brennan | Jim Brennan | +| [HDFS-14625](https://issues.apache.org/jira/browse/HDFS-14625) | Make DefaultAuditLogger class in FSnamesystem to Abstract | Major | . | hemanthboyina | hemanthboyina | +| [HDFS-14491](https://issues.apache.org/jira/browse/HDFS-14491) | More Clarity on Namenode UI Around Blocks and Replicas | Minor | . | Alan Jackoway | Siyao Meng | +| [YARN-9134](https://issues.apache.org/jira/browse/YARN-9134) | No test coverage for redefining FPGA / GPU resource types in TestResourceUtils | Major | . | Szilard Nemeth | Peter Bacsko | +| [YARN-9133](https://issues.apache.org/jira/browse/YARN-9133) | Make tests more easy to comprehend in TestGpuResourceHandler | Major | . | Szilard Nemeth | Peter Bacsko | +| [YARN-9140](https://issues.apache.org/jira/browse/YARN-9140) | Code cleanup in ResourcePluginManager.initialize and in TestResourcePluginManager | Trivial | . | Szilard Nemeth | Peter Bacsko | +| [YARN-9676](https://issues.apache.org/jira/browse/YARN-9676) | Add DEBUG and TRACE level messages to AppLogAggregatorImpl and connected classes | Major | . | Adam Antal | Adam Antal | +| [YARN-9488](https://issues.apache.org/jira/browse/YARN-9488) | Skip YARNFeatureNotEnabledException from ClientRMService | Minor | resourcemanager | Prabhu Joseph | Prabhu Joseph | +| [YARN-9679](https://issues.apache.org/jira/browse/YARN-9679) | Regular code cleanup in TestResourcePluginManager | Major | . | Szilard Nemeth | Adam Antal | +| [HADOOP-16504](https://issues.apache.org/jira/browse/HADOOP-16504) | Increase ipc.server.listen.queue.size default from 128 to 256 | Major | . | Lisheng Sun | Lisheng Sun | +| [YARN-8586](https://issues.apache.org/jira/browse/YARN-8586) | Extract log aggregation related fields and methods from RMAppImpl | Major | . | Szilard Nemeth | Peter Bacsko | +| [YARN-9100](https://issues.apache.org/jira/browse/YARN-9100) | Add tests for GpuResourceAllocator and do minor code cleanup | Major | . | Szilard Nemeth | Peter Bacsko | +| [HDFS-14678](https://issues.apache.org/jira/browse/HDFS-14678) | Allow triggerBlockReport to a specific namenode | Major | datanode | Leon Gao | Leon Gao | +| [HDFS-14523](https://issues.apache.org/jira/browse/HDFS-14523) | Remove excess read lock for NetworkToplogy | Major | . | Wu Weiwei | Wu Weiwei | +| [HADOOP-15246](https://issues.apache.org/jira/browse/HADOOP-15246) | SpanReceiverInfo - Prefer ArrayList over LinkedList | Trivial | common | David Mollitor | David Mollitor | +| [HADOOP-16158](https://issues.apache.org/jira/browse/HADOOP-16158) | DistCp to support checksum validation when copy blocks in parallel | Major | tools/distcp | Kai Xie | Kai Xie | +| [HADOOP-14784](https://issues.apache.org/jira/browse/HADOOP-14784) | [KMS] Improve KeyAuthorizationKeyProvider#toString() | Trivial | . | Wei-Chiu Chuang | Yeliang Cang | +| [HDFS-14746](https://issues.apache.org/jira/browse/HDFS-14746) | Trivial test code update after HDFS-14687 | Trivial | ec | Wei-Chiu Chuang | Kevin Su | +| [HDFS-13709](https://issues.apache.org/jira/browse/HDFS-13709) | Report bad block to NN when transfer block encounter EIO exception | Major | datanode | Chen Zhang | Chen Zhang | +| [HDFS-14665](https://issues.apache.org/jira/browse/HDFS-14665) | HttpFS: LISTSTATUS response is missing HDFS-specific fields | Major | httpfs | Siyao Meng | Siyao Meng | +| [HADOOP-16523](https://issues.apache.org/jira/browse/HADOOP-16523) | Minor spell mistake in comment (PR#388) | Major | . | Wei-Chiu Chuang | | +| [HDFS-14276](https://issues.apache.org/jira/browse/HDFS-14276) | [SBN read] Reduce tailing overhead | Major | ha, namenode | Wei-Chiu Chuang | Ayush Saxena | +| [HADOOP-16061](https://issues.apache.org/jira/browse/HADOOP-16061) | Update Apache Yetus to 0.10.0 | Major | build | Akira Ajisaka | Akira Ajisaka | +| [HDFS-14358](https://issues.apache.org/jira/browse/HDFS-14358) | Provide LiveNode and DeadNode filter in DataNode UI | Major | . | Ravuri Sushma sree | hemanthboyina | +| [HDFS-14675](https://issues.apache.org/jira/browse/HDFS-14675) | Increase Balancer Defaults Further | Major | balancer & mover | Stephen O'Donnell | Stephen O'Donnell | +| [HDFS-14617](https://issues.apache.org/jira/browse/HDFS-14617) | Improve fsimage load time by writing sub-sections to the fsimage index | Major | namenode | Stephen O'Donnell | Stephen O'Donnell | +| [HDFS-14497](https://issues.apache.org/jira/browse/HDFS-14497) | Write lock held by metasave impact following RPC processing | Major | namenode | Xiaoqiao He | Xiaoqiao He | +| [HDFS-14760](https://issues.apache.org/jira/browse/HDFS-14760) | Log INFO mode if snapshot usage and actual usage differ | Major | . | CR Hota | CR Hota | +| [HDFS-14710](https://issues.apache.org/jira/browse/HDFS-14710) | RBF: Improve some RPC performance by using previous block | Minor | rbf | xuzq | xuzq | +| [YARN-9756](https://issues.apache.org/jira/browse/YARN-9756) | Create metric that sums total memory/vcores preempted per round | Major | capacity scheduler | Eric Payne | Manikandan R | +| [HDFS-14104](https://issues.apache.org/jira/browse/HDFS-14104) | Review getImageTxIdToRetain | Minor | namenode | David Mollitor | David Mollitor | +| [HDFS-14256](https://issues.apache.org/jira/browse/HDFS-14256) | Review Logging of NameNode Class | Minor | namenode | David Mollitor | David Mollitor | +| [YARN-9783](https://issues.apache.org/jira/browse/YARN-9783) | Remove low-level zookeeper test to be able to build Hadoop against zookeeper 3.5.5 | Major | test | Mate Szalay-Beko | Mate Szalay-Beko | +| [HDFS-14748](https://issues.apache.org/jira/browse/HDFS-14748) | Make DataNodePeerMetrics#minOutlierDetectionSamples configurable | Major | . | Lisheng Sun | Lisheng Sun | +| [HADOOP-15998](https://issues.apache.org/jira/browse/HADOOP-15998) | Ensure jar validation works on Windows. | Blocker | build | Brian Grunkemeyer | Brian Grunkemeyer | +| [HDFS-13843](https://issues.apache.org/jira/browse/HDFS-13843) | RBF: Add optional parameter -d for detailed listing of mount points. | Major | federation | Soumyapn | Ayush Saxena | +| [YARN-9400](https://issues.apache.org/jira/browse/YARN-9400) | Remove unnecessary if at EntityGroupFSTimelineStore#parseApplicationId | Minor | . | Prabhu Joseph | Prabhu Joseph | +| [HDFS-14633](https://issues.apache.org/jira/browse/HDFS-14633) | The StorageType quota and consume in QuotaFeature is not handled for rename | Major | . | Jinglun | Jinglun | +| [HADOOP-16268](https://issues.apache.org/jira/browse/HADOOP-16268) | Allow custom wrapped exception to be thrown by server if RPC call queue is filled up | Major | . | CR Hota | CR Hota | +| [HDFS-14812](https://issues.apache.org/jira/browse/HDFS-14812) | RBF: MountTableRefresherService should load cache when refresh | Major | . | xuzq | xuzq | +| [YARN-9810](https://issues.apache.org/jira/browse/YARN-9810) | Add queue capacity/maxcapacity percentage metrics | Major | . | Jonathan Hung | Shubham Gupta | +| [HDFS-14784](https://issues.apache.org/jira/browse/HDFS-14784) | Add more methods to WebHdfsTestUtil to support tests outside of package | Major | . | Chen Zhang | Chen Zhang | +| [HDFS-13913](https://issues.apache.org/jira/browse/HDFS-13913) | LazyPersistFileScrubber.run() should log meaningful warn message | Minor | namenode | Daniel Templeton | Daniel Green | +| [HADOOP-16531](https://issues.apache.org/jira/browse/HADOOP-16531) | Log more detail for slow RPC | Major | . | Chen Zhang | Chen Zhang | +| [YARN-9763](https://issues.apache.org/jira/browse/YARN-9763) | Print application tags in application summary | Major | . | Jonathan Hung | Manoj Kumar | +| [YARN-9795](https://issues.apache.org/jira/browse/YARN-9795) | ClusterMetrics to include AM allocation delay | Minor | . | Fengnan Li | Fengnan Li | +| [YARN-8995](https://issues.apache.org/jira/browse/YARN-8995) | Log events info in AsyncDispatcher when event queue size cumulatively reaches a certain number every time. | Major | metrics, nodemanager, resourcemanager | zhuqi | zhuqi | +| [YARN-9787](https://issues.apache.org/jira/browse/YARN-9787) | Typo in analysesErrorMsg | Trivial | yarn | David Mollitor | Kevin Su | +| [YARN-9764](https://issues.apache.org/jira/browse/YARN-9764) | Print application submission context label in application summary | Major | . | Jonathan Hung | Manoj Kumar | +| [HADOOP-16549](https://issues.apache.org/jira/browse/HADOOP-16549) | Remove Unsupported SSL/TLS Versions from Docs/Properties | Minor | documentation, security | Daisuke Kobayashi | Daisuke Kobayashi | +| [YARN-9824](https://issues.apache.org/jira/browse/YARN-9824) | Fall back to configured queue ordering policy class name | Major | . | Jonathan Hung | Jonathan Hung | +| [HADOOP-15184](https://issues.apache.org/jira/browse/HADOOP-15184) | Add GitHub pull request template | Major | . | Akira Ajisaka | Akira Ajisaka | +| [YARN-9815](https://issues.apache.org/jira/browse/YARN-9815) | ReservationACLsTestBase fails with NPE | Minor | yarn | Ahmed Hussein | Ahmed Hussein | +| [HDFS-14840](https://issues.apache.org/jira/browse/HDFS-14840) | Use Java Conccurent Instead of Synchronization in BlockPoolTokenSecretManager | Minor | hdfs | David Mollitor | David Mollitor | +| [HDFS-14799](https://issues.apache.org/jira/browse/HDFS-14799) | Do Not Call Map containsKey In Conjunction with get | Minor | namenode | David Mollitor | hemanthboyina | +| [HDFS-14795](https://issues.apache.org/jira/browse/HDFS-14795) | Add Throttler for writing block | Minor | . | Lisheng Sun | Lisheng Sun | +| [HADOOP-16556](https://issues.apache.org/jira/browse/HADOOP-16556) | Fix some LGTM alerts | Minor | common | Malcolm Taylor | Malcolm Taylor | +| [HADOOP-16069](https://issues.apache.org/jira/browse/HADOOP-16069) | Support configure ZK\_DTSM\_ZK\_KERBEROS\_PRINCIPAL in ZKDelegationTokenSecretManager using principal with Schema /\_HOST | Minor | common | luhuachao | luhuachao | +| [HDFS-14844](https://issues.apache.org/jira/browse/HDFS-14844) | Make buffer of BlockReaderRemote#newBlockReader#BufferedOutputStream configurable | Minor | . | Lisheng Sun | Lisheng Sun | +| [HADOOP-16445](https://issues.apache.org/jira/browse/HADOOP-16445) | Allow separate custom signing algorithms for S3 and DDB | Major | fs/s3 | Siddharth Seth | Siddharth Seth | +| [YARN-9762](https://issues.apache.org/jira/browse/YARN-9762) | Add submission context label to audit logs | Major | . | Jonathan Hung | Manoj Kumar | +| [HDFS-14837](https://issues.apache.org/jira/browse/HDFS-14837) | Review of Block.java | Minor | hdfs-client | David Mollitor | David Mollitor | +| [HDFS-14843](https://issues.apache.org/jira/browse/HDFS-14843) | Double Synchronization in BlockReportLeaseManager | Minor | . | David Mollitor | David Mollitor | +| [HDFS-14832](https://issues.apache.org/jira/browse/HDFS-14832) | RBF : Add Icon for ReadOnly False | Minor | . | hemanthboyina | hemanthboyina | +| [HDFS-11934](https://issues.apache.org/jira/browse/HDFS-11934) | Add assertion to TestDefaultNameNodePort#testGetAddressFromConf | Minor | hdfs | legend | Nikhil Navadiya | +| [YARN-9857](https://issues.apache.org/jira/browse/YARN-9857) | TestDelegationTokenRenewer throws NPE but tests pass | Minor | . | Ahmed Hussein | Ahmed Hussein | +| [HDFS-14564](https://issues.apache.org/jira/browse/HDFS-14564) | Add libhdfs APIs for readFully; add readFully to ByteBufferPositionedReadable | Major | hdfs-client, libhdfs, native | Sahil Takiar | Sahil Takiar | +| [HDFS-14850](https://issues.apache.org/jira/browse/HDFS-14850) | Optimize FileSystemAccessService#getFileSystemConfiguration | Major | httpfs, performance | Lisheng Sun | Lisheng Sun | +| [HDFS-14192](https://issues.apache.org/jira/browse/HDFS-14192) | Track missing DFS operations in Statistics and StorageStatistics | Major | . | Ayush Saxena | Ayush Saxena | +| [HADOOP-16599](https://issues.apache.org/jira/browse/HADOOP-16599) | Allow a SignerInitializer to be specified along with a Custom Signer | Major | fs/s3 | Siddharth Seth | Siddharth Seth | +| [HDFS-14888](https://issues.apache.org/jira/browse/HDFS-14888) | RBF: Enable Parallel Test Profile for builds | Major | . | Ayush Saxena | Ayush Saxena | +| [HADOOP-16624](https://issues.apache.org/jira/browse/HADOOP-16624) | Upgrade hugo to the latest version in Dockerfile | Minor | build | Akira Ajisaka | Kevin Su | +| [HDFS-14814](https://issues.apache.org/jira/browse/HDFS-14814) | RBF: RouterQuotaUpdateService supports inherited rule. | Major | . | Jinglun | Jinglun | +| [YARN-9356](https://issues.apache.org/jira/browse/YARN-9356) | Add more tests to ratio method in TestResourceCalculator | Major | . | Szilard Nemeth | Zoltan Siegl | +| [HDFS-14898](https://issues.apache.org/jira/browse/HDFS-14898) | Use Relative URLS in Hadoop HDFS HTTP FS | Major | httpfs | David Mollitor | David Mollitor | +| [YARN-7291](https://issues.apache.org/jira/browse/YARN-7291) | Better input parsing for resource in allocation file | Minor | fairscheduler | Yufei Gu | Zoltan Siegl | +| [YARN-9860](https://issues.apache.org/jira/browse/YARN-9860) | Enable service mode for Docker containers on YARN | Major | . | Prabhu Joseph | Prabhu Joseph | +| [HDFS-14899](https://issues.apache.org/jira/browse/HDFS-14899) | Use Relative URLS in Hadoop HDFS RBF | Major | rbf | David Mollitor | David Mollitor | +| [HDFS-14238](https://issues.apache.org/jira/browse/HDFS-14238) | A log in NNThroughputBenchmark should change log level to "INFO" instead of "ERROR" | Major | . | Shen Yinjie | Shen Yinjie | +| [HADOOP-16643](https://issues.apache.org/jira/browse/HADOOP-16643) | Update netty4 to the latest 4.1.42 | Major | . | Wei-Chiu Chuang | Lisheng Sun | +| [HDFS-14810](https://issues.apache.org/jira/browse/HDFS-14810) | Review FSNameSystem editlog sync | Major | namenode | Xiaoqiao He | Xiaoqiao He | +| [HADOOP-16152](https://issues.apache.org/jira/browse/HADOOP-16152) | Upgrade Eclipse Jetty version to 9.4.x | Major | . | Yuming Wang | Siyao Meng | +| [HADOOP-16579](https://issues.apache.org/jira/browse/HADOOP-16579) | Upgrade to Apache Curator 4.2.0 and ZooKeeper 3.5.6 in Hadoop | Major | . | Mate Szalay-Beko | Norbert Kalmár | +| [HDFS-14918](https://issues.apache.org/jira/browse/HDFS-14918) | Remove useless getRedundancyThread from BlockManagerTestUtil | Minor | . | Fei Hui | Fei Hui | +| [HDFS-14915](https://issues.apache.org/jira/browse/HDFS-14915) | Move Superuser Check Before Taking Lock For Encryption API | Major | . | Ayush Saxena | Ayush Saxena | +| [HDFS-14921](https://issues.apache.org/jira/browse/HDFS-14921) | Remove SuperUser Check in Setting Storage Policy in FileStatus During Listing | Major | . | Ayush Saxena | Ayush Saxena | +| [HADOOP-16615](https://issues.apache.org/jira/browse/HADOOP-16615) | Add password check for credential provider | Major | . | hong dongdong | hong dongdong | +| [HDFS-14917](https://issues.apache.org/jira/browse/HDFS-14917) | Change the ICON of "Decommissioned & dead" datanode on "dfshealth.html" | Trivial | ui | Xieming Li | Xieming Li | +| [HDFS-14923](https://issues.apache.org/jira/browse/HDFS-14923) | Remove dead code from HealthMonitor | Minor | . | Fei Hui | Fei Hui | +| [YARN-9914](https://issues.apache.org/jira/browse/YARN-9914) | Use separate configs for free disk space checking for full and not-full disks | Minor | yarn | Jim Brennan | Jim Brennan | +| [HDFS-14935](https://issues.apache.org/jira/browse/HDFS-14935) | Refactor DFSNetworkTopology#isNodeInScope | Minor | . | Lisheng Sun | Lisheng Sun | +| [YARN-2442](https://issues.apache.org/jira/browse/YARN-2442) | ResourceManager JMX UI does not give HA State | Major | resourcemanager | Nishan Shetty | Rohith Sharma K S | +| [YARN-9889](https://issues.apache.org/jira/browse/YARN-9889) | [UI] Add Application Tag column to RM All Applications table | Major | . | Kinga Marton | Kinga Marton | +| [HDFS-14936](https://issues.apache.org/jira/browse/HDFS-14936) | Add getNumOfChildren() for interface InnerNode | Minor | . | Lisheng Sun | Lisheng Sun | +| [HDFS-14927](https://issues.apache.org/jira/browse/HDFS-14927) | RBF: Add metrics for async callers thread pool | Minor | rbf | Leon Gao | Leon Gao | +| [HADOOP-16678](https://issues.apache.org/jira/browse/HADOOP-16678) | Review of ArrayWritable | Minor | common | David Mollitor | David Mollitor | +| [HDFS-14775](https://issues.apache.org/jira/browse/HDFS-14775) | Add Timestamp for longest FSN write/read lock held log | Major | . | Chen Zhang | Chen Zhang | +| [MAPREDUCE-7208](https://issues.apache.org/jira/browse/MAPREDUCE-7208) | Tuning TaskRuntimeEstimator | Minor | . | Ahmed Hussein | Ahmed Hussein | +| [HDFS-14942](https://issues.apache.org/jira/browse/HDFS-14942) | Change Log Level to debug in JournalNodeSyncer#syncWithJournalAtIndex | Minor | . | Lisheng Sun | Lisheng Sun | +| [HDFS-14815](https://issues.apache.org/jira/browse/HDFS-14815) | RBF: Update the quota in MountTable when calling setQuota on a MountTable src. | Major | . | Jinglun | Jinglun | +| [YARN-9677](https://issues.apache.org/jira/browse/YARN-9677) | Make FpgaDevice and GpuDevice classes more similar to each other | Major | . | Szilard Nemeth | Kevin Su | +| [YARN-9890](https://issues.apache.org/jira/browse/YARN-9890) | [UI2] Add Application tag to the app table and app detail page. | Major | . | Kinga Marton | Kinga Marton | +| [HDFS-14928](https://issues.apache.org/jira/browse/HDFS-14928) | UI: unifying the WebUI across different components. | Trivial | ui | Xieming Li | Xieming Li | +| [HDFS-14975](https://issues.apache.org/jira/browse/HDFS-14975) | Add LF for SetECPolicyCommand usage | Trivial | . | Fei Hui | Fei Hui | +| [YARN-9537](https://issues.apache.org/jira/browse/YARN-9537) | Add configuration to disable AM preemption | Major | fairscheduler | zhoukang | zhoukang | +| [HDFS-14979](https://issues.apache.org/jira/browse/HDFS-14979) | [Observer Node] Balancer should submit getBlocks to Observer Node when possible | Major | balancer & mover, hdfs | Erik Krogen | Erik Krogen | +| [HADOOP-16705](https://issues.apache.org/jira/browse/HADOOP-16705) | MBeanInfoBuilder puts unnecessary memory pressure on the system with a debug log | Major | metrics | Lukas Majercak | Lukas Majercak | +| [HADOOP-16691](https://issues.apache.org/jira/browse/HADOOP-16691) | Unify Logging in UserGroupInformation | Minor | common | David Mollitor | David Mollitor | +| [HDFS-14882](https://issues.apache.org/jira/browse/HDFS-14882) | Consider DataNode load when #getBlockLocation | Major | namenode | Xiaoqiao He | Xiaoqiao He | +| [HDFS-14974](https://issues.apache.org/jira/browse/HDFS-14974) | RBF: Make tests use free ports | Major | . | Íñigo Goiri | Íñigo Goiri | +| [HDFS-14955](https://issues.apache.org/jira/browse/HDFS-14955) | RBF: getQuotaUsage() on mount point should return global quota. | Minor | . | Jinglun | Jinglun | +| [HADOOP-16712](https://issues.apache.org/jira/browse/HADOOP-16712) | Config ha.failover-controller.active-standby-elector.zk.op.retries is not in core-default.xml | Trivial | . | Wei-Chiu Chuang | Xieming Li | +| [YARN-9886](https://issues.apache.org/jira/browse/YARN-9886) | Queue mapping based on userid passed through application tag | Major | scheduler | Kinga Marton | Szilard Nemeth | +| [HDFS-14952](https://issues.apache.org/jira/browse/HDFS-14952) | Skip safemode if blockTotal is 0 in new NN | Trivial | namenode | Rajesh Balamohan | Xiaoqiao He | +| [HDFS-14995](https://issues.apache.org/jira/browse/HDFS-14995) | Use log variable directly instead of passing as argument in InvalidateBlocks#printBlockDeletionTime() | Minor | . | Lisheng Sun | Lisheng Sun | +| [HDFS-14949](https://issues.apache.org/jira/browse/HDFS-14949) | Add getServerDefaults() support to HttpFS | Major | . | Kihwal Lee | hemanthboyina | +| [HADOOP-15852](https://issues.apache.org/jira/browse/HADOOP-15852) | Refactor QuotaUsage | Minor | common | David Mollitor | David Mollitor | +| [HDFS-15002](https://issues.apache.org/jira/browse/HDFS-15002) | RBF: Fix annotation in RouterAdmin | Trivial | . | Jinglun | Jinglun | +| [YARN-8842](https://issues.apache.org/jira/browse/YARN-8842) | Expose metrics for custom resource types in QueueMetrics | Major | . | Szilard Nemeth | Szilard Nemeth | +| [HADOOP-16717](https://issues.apache.org/jira/browse/HADOOP-16717) | Remove GenericsUtil isLog4jLogger dependency on Log4jLoggerAdapter | Major | . | David Mollitor | Xieming Li | +| [YARN-9966](https://issues.apache.org/jira/browse/YARN-9966) | Code duplication in UserGroupMappingPlacementRule | Major | . | Szilard Nemeth | Kevin Su | +| [YARN-9991](https://issues.apache.org/jira/browse/YARN-9991) | Queue mapping based on userid passed through application tag: Change prefix to 'userid' | Major | scheduler | Szilard Nemeth | Szilard Nemeth | +| [YARN-9362](https://issues.apache.org/jira/browse/YARN-9362) | Code cleanup in TestNMLeveldbStateStoreService | Minor | . | Szilard Nemeth | Denes Gerencser | +| [YARN-9937](https://issues.apache.org/jira/browse/YARN-9937) | Add missing queue configs in RMWebService#CapacitySchedulerQueueInfo | Major | capacity scheduler | Prabhu Joseph | Prabhu Joseph | +| [HDFS-15013](https://issues.apache.org/jira/browse/HDFS-15013) | Reduce NameNode overview tab response time | Minor | namenode | HuangTao | HuangTao | +| [HADOOP-16718](https://issues.apache.org/jira/browse/HADOOP-16718) | Allow disabling Server Name Indication (SNI) for Jetty | Major | . | Siyao Meng | Aravindan Vijayan | +| [YARN-9958](https://issues.apache.org/jira/browse/YARN-9958) | Remove the invalid lock in ContainerExecutor | Major | . | Wanqiang Ji | Wanqiang Ji | +| [HADOOP-16729](https://issues.apache.org/jira/browse/HADOOP-16729) | Extract version numbers to head of pom.xml | Minor | build | Tamas Penzes | Tamas Penzes | +| [MAPREDUCE-7250](https://issues.apache.org/jira/browse/MAPREDUCE-7250) | FrameworkUploader: skip replication check entirely if timeout == 0 | Major | mrv2 | Peter Bacsko | Peter Bacsko | +| [YARN-9052](https://issues.apache.org/jira/browse/YARN-9052) | Replace all MockRM submit method definitions with a builder | Minor | . | Szilard Nemeth | Szilard Nemeth | +| [YARN-9128](https://issues.apache.org/jira/browse/YARN-9128) | Use SerializationUtils from apache commons to serialize / deserialize ResourceMappings | Major | . | Szilard Nemeth | Adam Antal | +| [HDFS-15023](https://issues.apache.org/jira/browse/HDFS-15023) | [SBN read] ZKFC should check the state before joining the election | Major | . | Fei Hui | Fei Hui | +| [HADOOP-16735](https://issues.apache.org/jira/browse/HADOOP-16735) | Make it clearer in config default that EnvironmentVariableCredentialsProvider supports AWS\_SESSION\_TOKEN | Minor | documentation, fs/s3 | Mingliang Liu | Mingliang Liu | +| [HDFS-15028](https://issues.apache.org/jira/browse/HDFS-15028) | Keep the capacity of volume and reduce a system call | Minor | datanode | Yang Yun | Yang Yun | +| [YARN-10012](https://issues.apache.org/jira/browse/YARN-10012) | Guaranteed and max capacity queue metrics for custom resources | Major | . | Jonathan Hung | Manikandan R | +| [HDFS-14522](https://issues.apache.org/jira/browse/HDFS-14522) | Allow compact property description in xml in httpfs | Major | httpfs | Akira Ajisaka | Masatake Iwasaki | +| [YARN-5106](https://issues.apache.org/jira/browse/YARN-5106) | Provide a builder interface for FairScheduler allocations for use in tests | Major | fairscheduler | Karthik Kambatla | Adam Antal | +| [HDFS-14854](https://issues.apache.org/jira/browse/HDFS-14854) | Create improved decommission monitor implementation | Major | namenode | Stephen O'Donnell | Stephen O'Donnell | +| [HDFS-15050](https://issues.apache.org/jira/browse/HDFS-15050) | Optimize log information when DFSInputStream meet CannotObtainBlockLengthException | Major | dfsclient | Xiaoqiao He | Xiaoqiao He | +| [HDFS-15016](https://issues.apache.org/jira/browse/HDFS-15016) | RBF: getDatanodeReport() should return the latest update | Major | . | Íñigo Goiri | Íñigo Goiri | +| [YARN-10033](https://issues.apache.org/jira/browse/YARN-10033) | TestProportionalCapacityPreemptionPolicy not initializing vcores for effective max resources | Major | capacity scheduler, test | Eric Payne | Eric Payne | +| [YARN-10039](https://issues.apache.org/jira/browse/YARN-10039) | Allow disabling app submission from REST endpoints | Major | . | Jonathan Hung | Jonathan Hung | +| [YARN-9894](https://issues.apache.org/jira/browse/YARN-9894) | CapacitySchedulerPerf test for measuring hundreds of apps in a large number of queues. | Major | capacity scheduler, test | Eric Payne | Eric Payne | +| [YARN-10038](https://issues.apache.org/jira/browse/YARN-10038) | [UI] Finish Time is not correctly parsed in the RM Apps page | Minor | webapp | Íñigo Goiri | Íñigo Goiri | +| [HADOOP-16771](https://issues.apache.org/jira/browse/HADOOP-16771) | Update checkstyle to 8.26 and maven-checkstyle-plugin to 3.1.0 | Major | build | Andras Bokor | Andras Bokor | +| [HDFS-15062](https://issues.apache.org/jira/browse/HDFS-15062) | Add LOG when sendIBRs failed | Major | datanode | Fei Hui | Fei Hui | +| [YARN-10009](https://issues.apache.org/jira/browse/YARN-10009) | In Capacity Scheduler, DRC can treat minimum user limit percent as a max when custom resource is defined | Critical | capacity scheduler | Eric Payne | Eric Payne | +| [YARN-10036](https://issues.apache.org/jira/browse/YARN-10036) | Install yarnpkg and upgrade nodejs in Dockerfile | Major | buid, yarn-ui-v2 | Akira Ajisaka | Akira Ajisaka | +| [HDFS-12999](https://issues.apache.org/jira/browse/HDFS-12999) | When reach the end of the block group, it may not need to flush all the data packets(flushAllInternals) twice. | Major | erasure-coding, hdfs-client | lufei | lufei | +| [HDFS-14997](https://issues.apache.org/jira/browse/HDFS-14997) | BPServiceActor processes commands from NameNode asynchronously | Major | datanode | Xiaoqiao He | Xiaoqiao He | +| [HDFS-15003](https://issues.apache.org/jira/browse/HDFS-15003) | RBF: Make Router support storage type quota. | Major | . | Jinglun | Jinglun | +| [HDFS-15081](https://issues.apache.org/jira/browse/HDFS-15081) | Typo in RetryCache#waitForCompletion annotation | Trivial | namenode | Fei Hui | Fei Hui | +| [HDFS-15074](https://issues.apache.org/jira/browse/HDFS-15074) | DataNode.DataTransfer thread should catch all the expception and log it. | Major | datanode | Surendra Singh Lilhore | hemanthboyina | +| [HDFS-14937](https://issues.apache.org/jira/browse/HDFS-14937) | [SBN read] ObserverReadProxyProvider should throw InterruptException | Major | . | xuzq | xuzq | +| [HDFS-14740](https://issues.apache.org/jira/browse/HDFS-14740) | Recover data blocks from persistent memory read cache during datanode restarts | Major | caching, datanode | Feilong He | Feilong He | +| [HADOOP-16777](https://issues.apache.org/jira/browse/HADOOP-16777) | Add Tez to LimitedPrivate of ClusterStorageCapacityExceededException | Minor | common | Wang Yan | Wang Yan | +| [HDFS-15091](https://issues.apache.org/jira/browse/HDFS-15091) | Cache Admin and Quota Commands Should Check SuperUser Before Taking Lock | Major | . | Ayush Saxena | Ayush Saxena | +| [HDFS-15090](https://issues.apache.org/jira/browse/HDFS-15090) | RBF: MountPoint Listing Should Return Flag Values Of Destination | Major | . | Ayush Saxena | Ayush Saxena | +| [HDFS-15096](https://issues.apache.org/jira/browse/HDFS-15096) | RBF: GetServerDefaults Should be Cached At Router | Major | . | Ayush Saxena | Ayush Saxena | +| [HDFS-15094](https://issues.apache.org/jira/browse/HDFS-15094) | RBF: Reuse ugi string in ConnectionPoolID | Major | . | Ayush Saxena | Ayush Saxena | +| [HADOOP-15993](https://issues.apache.org/jira/browse/HADOOP-15993) | Upgrade Kafka version in hadoop-kafka module | Major | build, security | Akira Ajisaka | Akira Ajisaka | +| [HDFS-15107](https://issues.apache.org/jira/browse/HDFS-15107) | dfs.client.server-defaults.validity.period.ms to support time units | Major | . | Ayush Saxena | Ayush Saxena | +| [YARN-9989](https://issues.apache.org/jira/browse/YARN-9989) | Typo in CapacityScheduler documentation: Runtime Configuration | Major | . | Szilard Nemeth | Kevin Su | +| [YARN-9868](https://issues.apache.org/jira/browse/YARN-9868) | Validate %primary\_group queue in CS queue manager | Major | . | Manikandan R | Manikandan R | +| [YARN-9912](https://issues.apache.org/jira/browse/YARN-9912) | Capacity scheduler: support u:user2:%secondary\_group queue mapping | Major | capacity scheduler, capacityscheduler | Manikandan R | Manikandan R | +| [HDFS-15097](https://issues.apache.org/jira/browse/HDFS-15097) | Purge log in KMS and HttpFS | Minor | httpfs, kms | Doris Gu | Doris Gu | +| [HDFS-15112](https://issues.apache.org/jira/browse/HDFS-15112) | RBF: Do not return FileNotFoundException when a subcluster is unavailable | Major | . | Íñigo Goiri | Íñigo Goiri | +| [HADOOP-16753](https://issues.apache.org/jira/browse/HADOOP-16753) | Refactor HAAdmin | Major | ha | Akira Ajisaka | Xieming Li | +| [HDFS-14968](https://issues.apache.org/jira/browse/HDFS-14968) | Add ability to know datanode staleness | Minor | datanode, logging, namenode | Ahmed Hussein | Ahmed Hussein | +| [YARN-7913](https://issues.apache.org/jira/browse/YARN-7913) | Improve error handling when application recovery fails with exception | Major | resourcemanager | Gergo Repas | Wilfred Spiegelenburg | +| [YARN-8472](https://issues.apache.org/jira/browse/YARN-8472) | YARN Container Phase 2 | Major | . | Eric Yang | Eric Yang | +| [HDFS-15117](https://issues.apache.org/jira/browse/HDFS-15117) | EC: Add getECTopologyResultForPolicies to DistributedFileSystem | Major | . | Ayush Saxena | Ayush Saxena | +| [HDFS-15119](https://issues.apache.org/jira/browse/HDFS-15119) | Allow expiration of cached locations in DFSInputStream | Minor | dfsclient | Ahmed Hussein | Ahmed Hussein | +| [HADOOP-16811](https://issues.apache.org/jira/browse/HADOOP-16811) | Use JUnit TemporaryFolder Rule in TestFileUtils | Minor | common, test | David Mollitor | David Mollitor | +| [MAPREDUCE-7262](https://issues.apache.org/jira/browse/MAPREDUCE-7262) | MRApp helpers block for long intervals (500ms) | Minor | mr-am | Ahmed Hussein | Ahmed Hussein | +| [YARN-9768](https://issues.apache.org/jira/browse/YARN-9768) | RM Renew Delegation token thread should timeout and retry | Major | . | CR Hota | Manikandan R | +| [MAPREDUCE-7260](https://issues.apache.org/jira/browse/MAPREDUCE-7260) | Cross origin request support for Job history server web UI | Critical | jobhistoryserver | Adam Antal | Adam Antal | +| [YARN-10084](https://issues.apache.org/jira/browse/YARN-10084) | Allow inheritance of max app lifetime / default app lifetime | Major | capacity scheduler | Eric Payne | Eric Payne | +| [HDFS-12491](https://issues.apache.org/jira/browse/HDFS-12491) | Support wildcard in CLASSPATH for libhdfs | Major | libhdfs | John Zhuge | Muhammad Samir Khan | +| [YARN-10116](https://issues.apache.org/jira/browse/YARN-10116) | Expose diagnostics in RMAppManager summary | Major | . | Jonathan Hung | Jonathan Hung | +| [YARN-9624](https://issues.apache.org/jira/browse/YARN-9624) | Use switch case for ProtoUtils#convertFromProtoFormat containerState | Major | . | Bibin Chundatt | Bilwa S T | +| [HADOOP-16739](https://issues.apache.org/jira/browse/HADOOP-16739) | Fix native build failure of hadoop-pipes on CentOS 8 | Major | tools/pipes | Masatake Iwasaki | Masatake Iwasaki | +| [HADOOP-16847](https://issues.apache.org/jira/browse/HADOOP-16847) | Test TestGroupsCaching fail if HashSet iterates in a different order | Minor | test | testfixer0 | testfixer0 | +| [HDFS-15150](https://issues.apache.org/jira/browse/HDFS-15150) | Introduce read write lock to Datanode | Major | datanode | Stephen O'Donnell | Stephen O'Donnell | +| [HDFS-14758](https://issues.apache.org/jira/browse/HDFS-14758) | Decrease lease hard limit | Minor | . | Eric Payne | hemanthboyina | +| [HDFS-15127](https://issues.apache.org/jira/browse/HDFS-15127) | RBF: Do not allow writes when a subcluster is unavailable for HASH\_ALL mount points. | Major | . | Íñigo Goiri | Íñigo Goiri | +| [MAPREDUCE-7263](https://issues.apache.org/jira/browse/MAPREDUCE-7263) | Remove obsolete validateTargetPath() from FrameworkUploader | Major | mrv2 | Adam Antal | Hudáky Márton Gyula | +| [YARN-10137](https://issues.apache.org/jira/browse/YARN-10137) | UIv2 build is broken in trunk | Critical | yarn, yarn-ui-v2 | Adam Antal | Adam Antal | +| [HDFS-15086](https://issues.apache.org/jira/browse/HDFS-15086) | Block scheduled counter never get decremet if the block got deleted before replication. | Major | 3.1.1 | Surendra Singh Lilhore | hemanthboyina | +| [HADOOP-16850](https://issues.apache.org/jira/browse/HADOOP-16850) | Support getting thread info from thread group for JvmMetrics to improve the performance | Major | metrics, performance | Tao Yang | Tao Yang | +| [HADOOP-13666](https://issues.apache.org/jira/browse/HADOOP-13666) | Supporting rack exclusion in countNumOfAvailableNodes in NetworkTopology | Major | net | Íñigo Goiri | Íñigo Goiri | +| [HADOOP-16833](https://issues.apache.org/jira/browse/HADOOP-16833) | InstrumentedLock should log lock queue time | Major | util | Stephen O'Donnell | Stephen O'Donnell | +| [YARN-8374](https://issues.apache.org/jira/browse/YARN-8374) | Upgrade objenesis to 2.6 | Major | build, timelineservice | Jason Darrell Lowe | Akira Ajisaka | +| [HDFS-13739](https://issues.apache.org/jira/browse/HDFS-13739) | Add option to disable rack local write preference | Major | balancer & mover, block placement, datanode, fs, hdfs, hdfs-client, namenode, nn, performance | Hari Sekhon | Ayush Saxena | +| [HDFS-15176](https://issues.apache.org/jira/browse/HDFS-15176) | Enable GcTimePercentage Metric in NameNode's JvmMetrics. | Minor | . | Jinglun | Jinglun | +| [HDFS-15174](https://issues.apache.org/jira/browse/HDFS-15174) | Optimize ReplicaCachingGetSpaceUsed by reducing unnecessary io operations | Major | . | Lisheng Sun | Lisheng Sun | +| [YARN-10074](https://issues.apache.org/jira/browse/YARN-10074) | Update netty to 4.1.42Final in yarn-csi | Major | . | Wei-Chiu Chuang | Wei-Chiu Chuang | +| [YARN-9018](https://issues.apache.org/jira/browse/YARN-9018) | Add functionality to AuxiliaryLocalPathHandler to return all locations to read for a given path | Major | . | Kuhu Shukla | Kuhu Shukla | +| [HDFS-14861](https://issues.apache.org/jira/browse/HDFS-14861) | Reset LowRedundancyBlocks Iterator periodically | Major | namenode | Stephen O'Donnell | Stephen O'Donnell | +| [HDFS-15120](https://issues.apache.org/jira/browse/HDFS-15120) | Refresh BlockPlacementPolicy at runtime. | Major | . | Jinglun | Jinglun | +| [HDFS-15190](https://issues.apache.org/jira/browse/HDFS-15190) | HttpFS : Add Support for Storage Policy Satisfier | Major | . | hemanthboyina | hemanthboyina | +| [HDFS-15033](https://issues.apache.org/jira/browse/HDFS-15033) | Support to save replica cached files to other place and make expired time configurable | Minor | datanode | Yang Yun | Yang Yun | +| [YARN-10148](https://issues.apache.org/jira/browse/YARN-10148) | Add Unit test for queue ACL for both FS and CS | Major | scheduler | Kinga Marton | Kinga Marton | +| [HADOOP-16899](https://issues.apache.org/jira/browse/HADOOP-16899) | Update HdfsDesign.md to reduce ambiguity | Minor | documentation | Akshay Nehe | Akshay Nehe | +| [HADOOP-14630](https://issues.apache.org/jira/browse/HADOOP-14630) | Contract Tests to verify create, mkdirs and rename under a file is forbidden | Major | fs, fs/azure, fs/s3, fs/swift | Steve Loughran | Steve Loughran | +| [HDFS-12101](https://issues.apache.org/jira/browse/HDFS-12101) | DFSClient.rename() to unwrap ParentNotDirectoryException; define policy for renames under a file | Minor | fs | Steve Loughran | Steve Loughran | +| [HADOOP-16898](https://issues.apache.org/jira/browse/HADOOP-16898) | Batch listing of multiple directories to be an unstable interface | Major | fs | Steve Loughran | Steve Loughran | +| [HADOOP-16772](https://issues.apache.org/jira/browse/HADOOP-16772) | Extract version numbers to head of pom.xml (addendum) | Major | build | Tamas Penzes | Tamas Penzes | +| [YARN-9050](https://issues.apache.org/jira/browse/YARN-9050) | [Umbrella] Usability improvements for scheduler activities | Major | capacityscheduler | Tao Yang | Tao Yang | +| [HDFS-15159](https://issues.apache.org/jira/browse/HDFS-15159) | Prevent adding same DN multiple times in PendingReconstructionBlocks | Major | . | hemanthboyina | hemanthboyina | +| [HDFS-15197](https://issues.apache.org/jira/browse/HDFS-15197) | [SBN read] Change ObserverRetryOnActiveException log to debug | Minor | hdfs | Chen Liang | Chen Liang | +| [HADOOP-16927](https://issues.apache.org/jira/browse/HADOOP-16927) | Update hadoop-thirdparty dependency version to 1.0.0 | Major | . | Vinayakumar B | Vinayakumar B | +| [HADOOP-15620](https://issues.apache.org/jira/browse/HADOOP-15620) | Über-jira: S3A phase VI: Hadoop 3.3 features | Major | fs/s3 | Steve Loughran | Steve Loughran | +| [HDFS-13377](https://issues.apache.org/jira/browse/HDFS-13377) | The owner of folder can set quota for his sub folder | Minor | namenode | Yang Yun | Yang Yun | +| [HADOOP-16938](https://issues.apache.org/jira/browse/HADOOP-16938) | Make non-HA proxy providers pluggable | Major | . | Roger Liu | Roger Liu | +| [HDFS-15154](https://issues.apache.org/jira/browse/HDFS-15154) | Allow only hdfs superusers the ability to assign HDFS storage policies | Major | hdfs | Bob Cauthen | Siddharth Wagle | +| [YARN-10200](https://issues.apache.org/jira/browse/YARN-10200) | Add number of containers to RMAppManager summary | Major | . | Jonathan Hung | Jonathan Hung | +| [HDFS-15075](https://issues.apache.org/jira/browse/HDFS-15075) | Remove process command timing from BPServiceActor | Major | . | Íñigo Goiri | Xiaoqiao He | +| [YARN-10210](https://issues.apache.org/jira/browse/YARN-10210) | Add a RMFailoverProxyProvider that does DNS resolution on failover | Major | . | Roger Liu | Roger Liu | +| [HDFS-15238](https://issues.apache.org/jira/browse/HDFS-15238) | RBF：NamenodeHeartbeatService caused memory to grow rapidly | Major | . | xuzq | xuzq | +| [HDFS-15239](https://issues.apache.org/jira/browse/HDFS-15239) | Add button to go to the parent directory in the explorer | Major | . | Íñigo Goiri | hemanthboyina | +| [HADOOP-16910](https://issues.apache.org/jira/browse/HADOOP-16910) | ABFS Streams to update FileSystem.Statistics counters on IO. | Major | fs/azure | Mehakmeet Singh | Mehakmeet Singh | +| [HADOOP-15619](https://issues.apache.org/jira/browse/HADOOP-15619) | Über-JIRA: S3Guard Phase IV: Hadoop 3.3 features | Major | fs/s3 | Steve Loughran | Steve Loughran | +| [HDFS-15258](https://issues.apache.org/jira/browse/HDFS-15258) | RBF: Mark Router FSCK unstable | Major | rbf | Akira Ajisaka | Akira Ajisaka | +| [YARN-10063](https://issues.apache.org/jira/browse/YARN-10063) | Usage output of container-executor binary needs to include --http/--https argument | Minor | . | Siddharth Ahuja | Siddharth Ahuja | +| [MAPREDUCE-7266](https://issues.apache.org/jira/browse/MAPREDUCE-7266) | historyContext doesn't need to be a class attribute inside JobHistoryServer | Minor | jobhistoryserver | Siddharth Ahuja | Siddharth Ahuja | +| [YARN-10003](https://issues.apache.org/jira/browse/YARN-10003) | YarnConfigurationStore#checkVersion throws exception that belongs to RMStateStore | Major | . | Szilard Nemeth | Benjamin Teke | +| [YARN-10212](https://issues.apache.org/jira/browse/YARN-10212) | Create separate configuration for max global AM attempts | Major | . | Jonathan Hung | Bilwa S T | +| [HDFS-14476](https://issues.apache.org/jira/browse/HDFS-14476) | lock too long when fix inconsistent blocks between disk and in-memory | Major | datanode | Sean Chow | Sean Chow | +| [YARN-5277](https://issues.apache.org/jira/browse/YARN-5277) | When localizers fail due to resource timestamps being out, provide more diagnostics | Major | nodemanager | Steve Loughran | Siddharth Ahuja | +| [YARN-9995](https://issues.apache.org/jira/browse/YARN-9995) | Code cleanup in TestSchedConfCLI | Minor | . | Szilard Nemeth | Bilwa S T | +| [YARN-9354](https://issues.apache.org/jira/browse/YARN-9354) | Resources should be created with ResourceTypesTestHelper instead of TestUtils | Trivial | . | Szilard Nemeth | Andras Gyori | +| [YARN-10002](https://issues.apache.org/jira/browse/YARN-10002) | Code cleanup and improvements in ConfigurationStoreBaseTest | Minor | . | Szilard Nemeth | Benjamin Teke | +| [YARN-9954](https://issues.apache.org/jira/browse/YARN-9954) | Configurable max application tags and max tag length | Major | . | Jonathan Hung | Bilwa S T | +| [HADOOP-16972](https://issues.apache.org/jira/browse/HADOOP-16972) | Ignore AuthenticationFilterInitializer for KMSWebServer | Blocker | kms | Masatake Iwasaki | Masatake Iwasaki | +| [YARN-10001](https://issues.apache.org/jira/browse/YARN-10001) | Add explanation of unimplemented methods in InMemoryConfigurationStore | Major | . | Szilard Nemeth | Siddharth Ahuja | +| [HADOOP-17001](https://issues.apache.org/jira/browse/HADOOP-17001) | The suffix name of the unified compression class | Major | io | bianqi | bianqi | +| [YARN-9997](https://issues.apache.org/jira/browse/YARN-9997) | Code cleanup in ZKConfigurationStore | Minor | . | Szilard Nemeth | Andras Gyori | +| [YARN-9996](https://issues.apache.org/jira/browse/YARN-9996) | Code cleanup in QueueAdminConfigurationMutationACLPolicy | Major | . | Szilard Nemeth | Siddharth Ahuja | +| [HADOOP-16914](https://issues.apache.org/jira/browse/HADOOP-16914) | Adding Output Stream Counters in ABFS | Major | fs/azure | Mehakmeet Singh | Mehakmeet Singh | +| [YARN-9998](https://issues.apache.org/jira/browse/YARN-9998) | Code cleanup in LeveldbConfigurationStore | Minor | . | Szilard Nemeth | Benjamin Teke | +| [YARN-9999](https://issues.apache.org/jira/browse/YARN-9999) | TestFSSchedulerConfigurationStore: Extend from ConfigurationStoreBaseTest, general code cleanup | Minor | . | Szilard Nemeth | Benjamin Teke | +| [YARN-10189](https://issues.apache.org/jira/browse/YARN-10189) | Code cleanup in LeveldbRMStateStore | Minor | . | Benjamin Teke | Benjamin Teke | +| [YARN-10237](https://issues.apache.org/jira/browse/YARN-10237) | Add isAbsoluteResource config for queue in scheduler response | Minor | scheduler | Prabhu Joseph | Prabhu Joseph | +| [YARN-10160](https://issues.apache.org/jira/browse/YARN-10160) | Add auto queue creation related configs to RMWebService#CapacitySchedulerQueueInfo | Major | . | Prabhu Joseph | Prabhu Joseph | +| [HADOOP-17046](https://issues.apache.org/jira/browse/HADOOP-17046) | Support downstreams' existing Hadoop-rpc implementations using non-shaded protobuf classes. | Major | rpc-server | Vinayakumar B | Vinayakumar B | + + +### BUG FIXES: + +| JIRA | Summary | Priority | Component | Reporter | Contributor | +|:---- |:---- | :--- |:---- |:---- |:---- | +| [YARN-8035](https://issues.apache.org/jira/browse/YARN-8035) | Uncaught exception in ContainersMonitorImpl during relaunch due to the process ID changing | Major | . | Shane Kumpf | Shane Kumpf | +| [HDFS-13376](https://issues.apache.org/jira/browse/HDFS-13376) | Specify minimum GCC version to avoid TLS support error in Build of hadoop-hdfs-native-client | Major | build, documentation, native | LiXin Ge | LiXin Ge | +| [YARN-8388](https://issues.apache.org/jira/browse/YARN-8388) | TestCGroupElasticMemoryController.testNormalExit() hangs on Linux | Major | nodemanager | Haibo Chen | Miklos Szegedi | +| [YARN-8108](https://issues.apache.org/jira/browse/YARN-8108) | RM metrics rest API throws GSSException in kerberized environment | Blocker | . | Kshitij Badani | Sunil G | +| [HADOOP-15814](https://issues.apache.org/jira/browse/HADOOP-15814) | Maven 3.3.3 unable to parse pom file | Major | . | Wei-Chiu Chuang | Wei-Chiu Chuang | +| [HADOOP-15817](https://issues.apache.org/jira/browse/HADOOP-15817) | Reuse Object Mapper in KMSJSONReader | Major | kms | Jonathan Turner Eagles | Jonathan Turner Eagles | +| [HDFS-13957](https://issues.apache.org/jira/browse/HDFS-13957) | Fix incorrect option used in description of InMemoryAliasMap | Minor | documentation | Yiqun Lin | Yiqun Lin | +| [YARN-4254](https://issues.apache.org/jira/browse/YARN-4254) | ApplicationAttempt stuck for ever due to UnknownHostException | Major | . | Bibin Chundatt | Bibin Chundatt | +| [HDFS-13964](https://issues.apache.org/jira/browse/HDFS-13964) | RBF: TestRouterWebHDFSContractAppend fails with No Active Namenode under nameservice | Major | . | Ayush Saxena | Ayush Saxena | +| [YARN-8788](https://issues.apache.org/jira/browse/YARN-8788) | mvn package -Pyarn-ui fails on JDK9 | Major | . | Akira Ajisaka | Vidura Bhathiya Mudalige | +| [YARN-7825](https://issues.apache.org/jira/browse/YARN-7825) | [UI2] Maintain constant horizontal application info bar for Application Attempt page | Major | yarn-ui-v2 | Yesha Vora | Akhil PB | +| [YARN-8659](https://issues.apache.org/jira/browse/YARN-8659) | RMWebServices returns only RUNNING apps when filtered with queue | Major | yarn | Prabhu Joseph | Szilard Nemeth | +| [YARN-8843](https://issues.apache.org/jira/browse/YARN-8843) | updateNodeResource does not support units for memory | Minor | . | Íñigo Goiri | Manikandan R | +| [HDFS-13962](https://issues.apache.org/jira/browse/HDFS-13962) | Add null check for add-replica pool to avoid lock acquiring | Major | . | Yiqun Lin | Surendra Singh Lilhore | +| [MAPREDUCE-7035](https://issues.apache.org/jira/browse/MAPREDUCE-7035) | Skip javadoc build for auto-generated sources in hadoop-mapreduce-client | Minor | client, documentation | Mukul Kumar Singh | Mukul Kumar Singh | +| [YARN-8853](https://issues.apache.org/jira/browse/YARN-8853) | [UI2] Application Attempts tab is not shown correctly when there are no attempts | Major | yarn-ui-v2 | Charan Hebri | Akhil PB | +| [MAPREDUCE-7130](https://issues.apache.org/jira/browse/MAPREDUCE-7130) | Rumen crashes trying to handle MRAppMaster recovery events | Minor | tools/rumen | Jonathan Bender | Peter Bacsko | +| [HDFS-13949](https://issues.apache.org/jira/browse/HDFS-13949) | Correct the description of dfs.datanode.disk.check.timeout in hdfs-default.xml | Minor | documentation | Toshihiro Suzuki | Toshihiro Suzuki | +| [HADOOP-15708](https://issues.apache.org/jira/browse/HADOOP-15708) | Reading values from Configuration before adding deprecations make it impossible to read value with deprecated key | Minor | conf | Szilard Nemeth | Zoltan Siegl | +| [YARN-8666](https://issues.apache.org/jira/browse/YARN-8666) | [UI2] Remove application tab from YARN Queue Page | Major | yarn-ui-v2 | Yesha Vora | Yesha Vora | +| [YARN-8753](https://issues.apache.org/jira/browse/YARN-8753) | [UI2] Lost nodes representation missing from Nodemanagers Chart | Major | yarn-ui-v2 | Yesha Vora | Yesha Vora | +| [YARN-8710](https://issues.apache.org/jira/browse/YARN-8710) | Service AM should set a finite limit on NM container max retries | Major | yarn-native-services | Suma Shivaprasad | Suma Shivaprasad | +| [HDFS-13951](https://issues.apache.org/jira/browse/HDFS-13951) | HDFS DelegationTokenFetcher can't print non-HDFS tokens in a tokenfile | Minor | tools | Steve Loughran | Steve Loughran | +| [HDFS-13945](https://issues.apache.org/jira/browse/HDFS-13945) | TestDataNodeVolumeFailure is Flaky | Major | . | Ayush Saxena | Ayush Saxena | +| [HDFS-13802](https://issues.apache.org/jira/browse/HDFS-13802) | RBF: Remove FSCK from Router Web UI | Major | . | Fei Hui | Fei Hui | +| [YARN-8830](https://issues.apache.org/jira/browse/YARN-8830) | SLS tool fix node addition | Critical | . | Bibin Chundatt | Bibin Chundatt | +| [YARN-8869](https://issues.apache.org/jira/browse/YARN-8869) | YARN Service Client might not work correctly with RM REST API for Kerberos authentication | Blocker | . | Eric Yang | Eric Yang | +| [YARN-8775](https://issues.apache.org/jira/browse/YARN-8775) | TestDiskFailures.testLocalDirsFailures sometimes can fail on concurrent File modifications | Major | test, yarn | Antal Bálint Steinbach | Antal Bálint Steinbach | +| [HADOOP-15853](https://issues.apache.org/jira/browse/HADOOP-15853) | TestConfigurationDeprecation leaves behind a temp file, resulting in a license issue | Major | test | Robert Kanter | Ayush Saxena | +| [YARN-8879](https://issues.apache.org/jira/browse/YARN-8879) | Kerberos principal is needed when submitting a submarine job | Critical | . | Zac Zhou | Zac Zhou | +| [HDFS-13993](https://issues.apache.org/jira/browse/HDFS-13993) | TestDataNodeVolumeFailure#testTolerateVolumeFailuresAfterAddingMoreVolumes is flaky | Major | . | Ayush Saxena | Ayush Saxena | +| [YARN-8810](https://issues.apache.org/jira/browse/YARN-8810) | Yarn Service: discrepancy between hashcode and equals of ConfigFile | Minor | . | Chandni Singh | Chandni Singh | +| [YARN-8759](https://issues.apache.org/jira/browse/YARN-8759) | Copy of resource-types.xml is not deleted if test fails, causes other test failures | Major | yarn | Antal Bálint Steinbach | Antal Bálint Steinbach | +| [HDFS-14000](https://issues.apache.org/jira/browse/HDFS-14000) | RBF: Documentation should reflect right scripts for v3.0 and above | Major | . | CR Hota | CR Hota | +| [YARN-8868](https://issues.apache.org/jira/browse/YARN-8868) | Set HTTPOnly attribute to Cookie | Major | . | Chandni Singh | Chandni Singh | +| [YARN-8864](https://issues.apache.org/jira/browse/YARN-8864) | NM incorrectly logs container user as the user who sent a start/stop container request in its audit log | Major | nodemanager | Haibo Chen | Wilfred Spiegelenburg | +| [HDFS-13990](https://issues.apache.org/jira/browse/HDFS-13990) | Synchronization Issue With HashResolver | Minor | federation | David Mollitor | David Mollitor | +| [HDFS-14003](https://issues.apache.org/jira/browse/HDFS-14003) | Fix findbugs warning in trunk for FSImageFormatPBINode | Major | . | Yiqun Lin | Yiqun Lin | +| [HDFS-14005](https://issues.apache.org/jira/browse/HDFS-14005) | RBF: Web UI update to bootstrap-3.3.7 | Major | . | Íñigo Goiri | Íñigo Goiri | +| [HDFS-14002](https://issues.apache.org/jira/browse/HDFS-14002) | TestLayoutVersion#testNameNodeFeatureMinimumCompatibleLayoutVersions fails | Major | test | Takanobu Asanuma | Takanobu Asanuma | +| [HADOOP-15418](https://issues.apache.org/jira/browse/HADOOP-15418) | Hadoop KMSAuthenticationFilter needs to use getPropsByPrefix instead of iterator to avoid ConcurrentModificationException | Major | common | Suma Shivaprasad | Suma Shivaprasad | +| [HDFS-14007](https://issues.apache.org/jira/browse/HDFS-14007) | Incompatible layout when generating FSImage | Major | . | Íñigo Goiri | Íñigo Goiri | +| [HDFS-9872](https://issues.apache.org/jira/browse/HDFS-9872) | HDFS bytes-default configurations should accept multiple size units | Major | . | Yiqun Lin | Yiqun Lin | +| [YARN-8910](https://issues.apache.org/jira/browse/YARN-8910) | Misleading log statement in NM when max retries is -1 | Minor | . | Chandni Singh | Chandni Singh | +| [HADOOP-15850](https://issues.apache.org/jira/browse/HADOOP-15850) | CopyCommitter#concatFileChunks should check that the blocks per chunk is not 0 | Critical | tools/distcp | Ted Yu | Ted Yu | +| [HDFS-13983](https://issues.apache.org/jira/browse/HDFS-13983) | TestOfflineImageViewer crashes in windows | Major | . | Vinayakumar B | Vinayakumar B | +| [YARN-7502](https://issues.apache.org/jira/browse/YARN-7502) | Nodemanager restart docs should describe nodemanager supervised property | Major | documentation | Jason Darrell Lowe | Suma Shivaprasad | +| [HADOOP-15866](https://issues.apache.org/jira/browse/HADOOP-15866) | Renamed HADOOP\_SECURITY\_GROUP\_SHELL\_COMMAND\_TIMEOUT keys break compatibility | Blocker | . | Wei-Chiu Chuang | Wei-Chiu Chuang | +| [YARN-8922](https://issues.apache.org/jira/browse/YARN-8922) | Fix test-container-executor | Major | test | Robert Kanter | Robert Kanter | +| [YARN-8826](https://issues.apache.org/jira/browse/YARN-8826) | Fix lingering timeline collector after serviceStop in TimelineCollectorManager | Trivial | ATSv2 | Prabha Manepalli | Prabha Manepalli | +| [HADOOP-15873](https://issues.apache.org/jira/browse/HADOOP-15873) | Add JavaBeans Activation Framework API to LICENSE.txt | Major | . | Wei-Chiu Chuang | Akira Ajisaka | +| [YARN-8919](https://issues.apache.org/jira/browse/YARN-8919) | Some tests fail due to NoClassDefFoundError for OperatorCreationException | Critical | . | Takanobu Asanuma | Takanobu Asanuma | +| [HDFS-14004](https://issues.apache.org/jira/browse/HDFS-14004) | TestLeaseRecovery2#testCloseWhileRecoverLease fails intermittently in trunk | Major | . | Ayush Saxena | Ayush Saxena | +| [MAPREDUCE-4669](https://issues.apache.org/jira/browse/MAPREDUCE-4669) | MRAM web UI does not work with HTTPS | Major | mr-am | Alejandro Abdelnur | Robert Kanter | +| [YARN-8814](https://issues.apache.org/jira/browse/YARN-8814) | Yarn Service Upgrade: Update the swagger definition | Major | . | Chandni Singh | Chandni Singh | +| [HADOOP-15856](https://issues.apache.org/jira/browse/HADOOP-15856) | Trunk build fails to compile native on Windows | Blocker | native | Vinayakumar B | Vinayakumar B | +| [YARN-8939](https://issues.apache.org/jira/browse/YARN-8939) | Javadoc build fails in hadoop-yarn-csi | Major | . | Takanobu Asanuma | Weiwei Yang | +| [YARN-8938](https://issues.apache.org/jira/browse/YARN-8938) | Add service upgrade cancel and express examples to the service upgrade doc | Minor | . | Chandni Singh | Chandni Singh | +| [HDFS-14021](https://issues.apache.org/jira/browse/HDFS-14021) | TestReconstructStripedBlocksWithRackAwareness#testReconstructForNotEnoughRacks fails intermittently | Major | erasure-coding, test | Xiao Chen | Xiao Chen | +| [MAPREDUCE-7151](https://issues.apache.org/jira/browse/MAPREDUCE-7151) | RMContainerAllocator#handleJobPriorityChange expects application\_priority always | Major | . | Bibin Chundatt | Bilwa S T | +| [YARN-8929](https://issues.apache.org/jira/browse/YARN-8929) | DefaultOOMHandler should only pick running containers to kill upon oom events | Major | nodemanager | Haibo Chen | Haibo Chen | +| [YARN-8587](https://issues.apache.org/jira/browse/YARN-8587) | Delays are noticed to launch docker container | Major | . | Yesha Vora | dockerzhang | +| [HDFS-14025](https://issues.apache.org/jira/browse/HDFS-14025) | TestPendingReconstruction.testPendingAndInvalidate fails | Major | . | Ayush Saxena | Ayush Saxena | +| [YARN-8941](https://issues.apache.org/jira/browse/YARN-8941) | Fix findbugs warnings in hadoop-yarn-csi | Minor | . | Takanobu Asanuma | Weiwei Yang | +| [YARN-8930](https://issues.apache.org/jira/browse/YARN-8930) | CGroup-based strict container memory enforcement does not work with CGroupElasticMemoryController | Major | nodemanager | Haibo Chen | Haibo Chen | +| [HDFS-13959](https://issues.apache.org/jira/browse/HDFS-13959) | TestUpgradeDomainBlockPlacementPolicy is flaky | Major | . | Ayush Saxena | Ayush Saxena | +| [HDFS-14028](https://issues.apache.org/jira/browse/HDFS-14028) | HDFS OIV temporary dir deletes folder | Major | hdfs | Adam Antal | Adam Antal | +| [HDFS-14027](https://issues.apache.org/jira/browse/HDFS-14027) | DFSStripedOutputStream should implement both hsync methods | Critical | erasure-coding | Xiao Chen | Xiao Chen | +| [YARN-8950](https://issues.apache.org/jira/browse/YARN-8950) | Compilation fails with dependency convergence error for hbase.profile=2.0 | Blocker | . | Rohith Sharma K S | Rohith Sharma K S | +| [YARN-8854](https://issues.apache.org/jira/browse/YARN-8854) | Upgrade jquery datatable version references to v1.10.19 | Critical | . | Akhil PB | Akhil PB | +| [HADOOP-15886](https://issues.apache.org/jira/browse/HADOOP-15886) | Fix findbugs warnings in RegistryDNS.java | Major | . | Akira Ajisaka | Akira Ajisaka | +| [YARN-8897](https://issues.apache.org/jira/browse/YARN-8897) | LoadBasedRouterPolicy throws NPE in case of sub cluster unavailability | Minor | federation, router | Akshay Agarwal | Bilwa S T | +| [HDFS-14049](https://issues.apache.org/jira/browse/HDFS-14049) | TestHttpFSServerWebServer fails on Windows because of missing winutils.exe | Major | . | Íñigo Goiri | Íñigo Goiri | +| [HADOOP-15899](https://issues.apache.org/jira/browse/HADOOP-15899) | Update AWS Java SDK versions in NOTICE.txt | Major | . | Akira Ajisaka | Akira Ajisaka | +| [HADOOP-15900](https://issues.apache.org/jira/browse/HADOOP-15900) | Update JSch versions in LICENSE.txt | Major | . | Akira Ajisaka | Akira Ajisaka | +| [HDFS-14042](https://issues.apache.org/jira/browse/HDFS-14042) | Fix NPE when PROVIDED storage is missing | Major | . | Íñigo Goiri | Virajith Jalaparti | +| [HDFS-14043](https://issues.apache.org/jira/browse/HDFS-14043) | Tolerate corrupted seen\_txid file | Major | hdfs, namenode | Lukas Majercak | Lukas Majercak | +| [YARN-8858](https://issues.apache.org/jira/browse/YARN-8858) | CapacityScheduler should respect maximum node resource when per-queue maximum-allocation is being used. | Major | . | Sumana Sathish | Wangda Tan | +| [YARN-8970](https://issues.apache.org/jira/browse/YARN-8970) | Improve the debug message in CS#allocateContainerOnSingleNode | Trivial | . | Weiwei Yang | Zhankun Tang | +| [YARN-8865](https://issues.apache.org/jira/browse/YARN-8865) | RMStateStore contains large number of expired RMDelegationToken | Major | resourcemanager | Wilfred Spiegelenburg | Wilfred Spiegelenburg | +| [HDFS-14048](https://issues.apache.org/jira/browse/HDFS-14048) | DFSOutputStream close() throws exception on subsequent call after DataNode restart | Major | hdfs-client | Erik Krogen | Erik Krogen | +| [MAPREDUCE-7156](https://issues.apache.org/jira/browse/MAPREDUCE-7156) | NullPointerException when reaching max shuffle connections | Major | mrv2 | Peter Bacsko | Peter Bacsko | +| [YARN-8866](https://issues.apache.org/jira/browse/YARN-8866) | Fix a parsing error for crossdomain.xml | Major | build, yarn-ui-v2 | Takanobu Asanuma | Takanobu Asanuma | +| [YARN-6729](https://issues.apache.org/jira/browse/YARN-6729) | Clarify documentation on how to enable cgroup support | Major | nodemanager | Yufei Gu | Zhankun Tang | +| [HDFS-14039](https://issues.apache.org/jira/browse/HDFS-14039) | ec -listPolicies doesn't show correct state for the default policy when the default is not RS(6,3) | Major | erasure-coding | Xiao Chen | Kitti Nanasi | +| [HADOOP-15903](https://issues.apache.org/jira/browse/HADOOP-15903) | Allow HttpServer2 to discover resources in /static when symlinks are used | Major | . | Íñigo Goiri | Íñigo Goiri | +| [YARN-8990](https://issues.apache.org/jira/browse/YARN-8990) | Fix fair scheduler race condition in app submit and queue cleanup | Blocker | fairscheduler | Wilfred Spiegelenburg | Wilfred Spiegelenburg | +| [HADOOP-15916](https://issues.apache.org/jira/browse/HADOOP-15916) | Upgrade Maven Surefire plugin to 3.0.0-M1 | Blocker | build | Akira Ajisaka | Akira Ajisaka | +| [YARN-9002](https://issues.apache.org/jira/browse/YARN-9002) | YARN Service keytab does not support s3, wasb, gs and is restricted to HDFS and local filesystem only | Major | yarn-native-services | Gour Saha | Gour Saha | +| [YARN-8233](https://issues.apache.org/jira/browse/YARN-8233) | NPE in CapacityScheduler#tryCommit when handling allocate/reserve proposal whose allocatedOrReservedContainer is null | Critical | capacityscheduler | Tao Yang | Tao Yang | +| [HDFS-14065](https://issues.apache.org/jira/browse/HDFS-14065) | Failed Storage Locations shows nothing in the Datanode Volume Failures | Major | . | Ayush Saxena | Ayush Saxena | +| [HADOOP-15923](https://issues.apache.org/jira/browse/HADOOP-15923) | create-release script should set max-cache-ttl as well as default-cache-ttl for gpg-agent | Blocker | build | Akira Ajisaka | Akira Ajisaka | +| [HADOOP-15912](https://issues.apache.org/jira/browse/HADOOP-15912) | start-build-env.sh still creates an invalid /etc/sudoers.d/hadoop-build-${USER\_ID} file entry after HADOOP-15802 | Major | build | Akira Ajisaka | Akira Ajisaka | +| [HADOOP-15869](https://issues.apache.org/jira/browse/HADOOP-15869) | BlockDecompressorStream#decompress should not return -1 in case of IOException. | Major | . | Surendra Singh Lilhore | Surendra Singh Lilhore | +| [MAPREDUCE-7158](https://issues.apache.org/jira/browse/MAPREDUCE-7158) | Inefficient Flush Logic in JobHistory EventWriter | Major | . | Zichen Sun | Zichen Sun | +| [HADOOP-15930](https://issues.apache.org/jira/browse/HADOOP-15930) | Exclude MD5 checksum files from release artifact | Critical | build | Akira Ajisaka | Akira Ajisaka | +| [YARN-8856](https://issues.apache.org/jira/browse/YARN-8856) | TestTimelineReaderWebServicesHBaseStorage tests failing with NoClassDefFoundError | Major | . | Jason Darrell Lowe | Sushil Ks | +| [HDFS-14054](https://issues.apache.org/jira/browse/HDFS-14054) | TestLeaseRecovery2: testHardLeaseRecoveryAfterNameNodeRestart2 and testHardLeaseRecoveryWithRenameAfterNameNodeRestart are flaky | Major | . | Zsolt Venczel | Zsolt Venczel | +| [HADOOP-15925](https://issues.apache.org/jira/browse/HADOOP-15925) | The config and log of gpg-agent are removed in create-release script | Major | build | Akira Ajisaka | Dinesh Chitlangia | +| [HDFS-13963](https://issues.apache.org/jira/browse/HDFS-13963) | NN UI is broken with IE11 | Minor | namenode, ui | Daisuke Kobayashi | Ayush Saxena | +| [HDFS-14056](https://issues.apache.org/jira/browse/HDFS-14056) | Fix error messages in HDFS-12716 | Minor | hdfs | Adam Antal | Ayush Saxena | +| [HADOOP-15939](https://issues.apache.org/jira/browse/HADOOP-15939) | Filter overlapping objenesis class in hadoop-client-minicluster | Minor | build | Xiaoyu Yao | Xiaoyu Yao | +| [YARN-8936](https://issues.apache.org/jira/browse/YARN-8936) | Bump up Atsv2 hbase versions | Major | . | Rohith Sharma K S | Vrushali C | +| [YARN-8984](https://issues.apache.org/jira/browse/YARN-8984) | AMRMClient#OutstandingSchedRequests leaks when AllocationTags is null or empty | Critical | . | Yang Wang | Yang Wang | +| [YARN-9042](https://issues.apache.org/jira/browse/YARN-9042) | Javadoc error in deviceplugin package | Major | . | Rohith Sharma K S | Zhankun Tang | +| [HADOOP-15358](https://issues.apache.org/jira/browse/HADOOP-15358) | SFTPConnectionPool connections leakage | Critical | fs | Mikhail Pryakhin | Mikhail Pryakhin | +| [HADOOP-15948](https://issues.apache.org/jira/browse/HADOOP-15948) | Inconsistency in get and put syntax if filename/dirname contains space | Minor | fs | vivek kumar | Ayush Saxena | +| [HDFS-13816](https://issues.apache.org/jira/browse/HDFS-13816) | dfs.getQuotaUsage() throws NPE on non-existent dir instead of FileNotFoundException | Major | namenode | Vinayakumar B | Vinayakumar B | +| [MAPREDUCE-7162](https://issues.apache.org/jira/browse/MAPREDUCE-7162) | TestEvents#testEvents fails | Critical | jobhistoryserver, test | Zhaohui Xin | Zhaohui Xin | +| [YARN-9056](https://issues.apache.org/jira/browse/YARN-9056) | Yarn Service Upgrade: Instance state changes from UPGRADING to READY without performing a readiness check | Critical | . | Chandni Singh | Chandni Singh | +| [MAPREDUCE-6190](https://issues.apache.org/jira/browse/MAPREDUCE-6190) | If a task stucks before its first heartbeat, it never timeouts and the MR job becomes stuck | Major | . | Ankit Malhotra | Zhaohui Xin | +| [YARN-8812](https://issues.apache.org/jira/browse/YARN-8812) | Containers fail during creating a symlink which started with hyphen for a resource file | Minor | . | Oleksandr Shevchenko | Oleksandr Shevchenko | +| [YARN-9044](https://issues.apache.org/jira/browse/YARN-9044) | LogsCLI should contact ATSv2 for "-am" option | Major | . | Rohith Sharma K S | Rohith Sharma K S | +| [HADOOP-15947](https://issues.apache.org/jira/browse/HADOOP-15947) | Fix ITestDynamoDBMetadataStore test error issues | Major | fs/s3 | Gabor Bota | Gabor Bota | +| [YARN-9030](https://issues.apache.org/jira/browse/YARN-9030) | Log aggregation changes to handle filesystems which do not support setting permissions | Major | log-aggregation | Suma Shivaprasad | Suma Shivaprasad | +| [YARN-8948](https://issues.apache.org/jira/browse/YARN-8948) | PlacementRule interface should be for all YarnSchedulers | Major | . | Bibin Chundatt | Bibin Chundatt | +| [YARN-9067](https://issues.apache.org/jira/browse/YARN-9067) | YARN Resource Manager is running OOM because of leak of Configuration Object | Major | yarn-native-services | Eric Yang | Eric Yang | +| [YARN-9010](https://issues.apache.org/jira/browse/YARN-9010) | Fix the incorrect trailing slash deletion in constructor method of CGroupsHandlerImpl | Major | . | Zhankun Tang | Zhankun Tang | +| [MAPREDUCE-7165](https://issues.apache.org/jira/browse/MAPREDUCE-7165) | mapred-site.xml is misformatted in single node setup document | Major | documentation | Akira Ajisaka | Zhaohui Xin | +| [HDFS-14075](https://issues.apache.org/jira/browse/HDFS-14075) | Terminate the namenode when failed to start log segment | Critical | . | Ayush Saxena | Ayush Saxena | +| [HADOOP-15970](https://issues.apache.org/jira/browse/HADOOP-15970) | Upgrade plexus-utils from 2.0.5 to 3.1.0 | Major | security | Akira Ajisaka | Akira Ajisaka | +| [HADOOP-15974](https://issues.apache.org/jira/browse/HADOOP-15974) | Upgrade Curator version to 2.13.0 to fix ZK tests | Major | . | Jason Darrell Lowe | Akira Ajisaka | +| [YARN-9071](https://issues.apache.org/jira/browse/YARN-9071) | NM and service AM don't have updated status for reinitialized containers | Critical | . | Billie Rinaldi | Chandni Singh | +| [YARN-8994](https://issues.apache.org/jira/browse/YARN-8994) | Fix race condition between move app and queue cleanup in Fair Scheduler | Major | fairscheduler | Wilfred Spiegelenburg | Wilfred Spiegelenburg | +| [YARN-9019](https://issues.apache.org/jira/browse/YARN-9019) | Ratio calculation of ResourceCalculator implementations could return NaN | Major | . | Szilard Nemeth | Szilard Nemeth | +| [YARN-9025](https://issues.apache.org/jira/browse/YARN-9025) | TestFairScheduler#testChildMaxResources is flaky | Major | . | Szilard Nemeth | Szilard Nemeth | +| [MAPREDUCE-7159](https://issues.apache.org/jira/browse/MAPREDUCE-7159) | FrameworkUploader: ensure proper permissions of generated framework tar.gz if restrictive umask is used | Major | mrv2 | Peter Bacsko | Peter Bacsko | +| [YARN-9009](https://issues.apache.org/jira/browse/YARN-9009) | Fix flaky test TestEntityGroupFSTimelineStore.testCleanLogs | Minor | . | OrDTesters | OrDTesters | +| [YARN-8738](https://issues.apache.org/jira/browse/YARN-8738) | FairScheduler should not parse negative maxResources or minResources values as positive | Major | fairscheduler | Sen Zhao | Szilard Nemeth | +| [HDFS-14137](https://issues.apache.org/jira/browse/HDFS-14137) | TestMaintenanceState fails with ArrayIndexOutOfBound Exception | Major | . | Ayush Saxena | Ayush Saxena | +| [YARN-9114](https://issues.apache.org/jira/browse/YARN-9114) | [UI2] log service redirect url need to support user name | Major | webapp, yarn-ui-v2 | Sunil G | Akhil PB | +| [HADOOP-15995](https://issues.apache.org/jira/browse/HADOOP-15995) | Add ldap.bind.password.alias in LdapGroupsMapping to distinguish aliases when using multiple providers through CompositeGroupsMapping | Major | common | Lukas Majercak | Lukas Majercak | +| [HDFS-14144](https://issues.apache.org/jira/browse/HDFS-14144) | TestPred fails in Trunk | Major | . | Ayush Saxena | Ayush Saxena | +| [MAPREDUCE-7170](https://issues.apache.org/jira/browse/MAPREDUCE-7170) | Doc typo in PluggableShuffleAndPluggableSort.md | Minor | documentation | Zhaohui Xin | Zhaohui Xin | +| [HDFS-14145](https://issues.apache.org/jira/browse/HDFS-14145) | TestBlockStorageMovementAttemptedItems#testNoBlockMovementAttemptFinishedReportAdded fails sporadically in Trunk | Major | . | Ayush Saxena | Ayush Saxena | +| [HDFS-14088](https://issues.apache.org/jira/browse/HDFS-14088) | RequestHedgingProxyProvider can throw NullPointerException when failover due to no lock on currentUsedProxy | Major | hdfs-client | Yuxuan Wang | Yuxuan Wang | +| [YARN-9040](https://issues.apache.org/jira/browse/YARN-9040) | LevelDBCacheTimelineStore in ATS 1.5 leaks native memory | Major | timelineserver | Tarun Parimi | Tarun Parimi | +| [YARN-9084](https://issues.apache.org/jira/browse/YARN-9084) | Service Upgrade: With default readiness check, the status of upgrade is reported to be successful prematurely | Major | . | Chandni Singh | Chandni Singh | +| [HDFS-13661](https://issues.apache.org/jira/browse/HDFS-13661) | Ls command with e option fails when the filesystem is not HDFS | Major | erasure-coding, tools | Takanobu Asanuma | Takanobu Asanuma | +| [YARN-9154](https://issues.apache.org/jira/browse/YARN-9154) | Fix itemization in YARN service quickstart document | Minor | documentation | Akira Ajisaka | Ayush Saxena | +| [HDFS-14166](https://issues.apache.org/jira/browse/HDFS-14166) | Ls with -e option not giving the result in proper format | Major | . | Soumyapn | Shubham Dewan | +| [HDFS-14165](https://issues.apache.org/jira/browse/HDFS-14165) | In NameNode UI under DataNode tab ,the Capacity column is Non-Aligned | Minor | . | Shubham Dewan | Shubham Dewan | +| [HDFS-14046](https://issues.apache.org/jira/browse/HDFS-14046) | In-Maintenance ICON is missing in datanode info page | Major | datanode | Harshakiran Reddy | Ranith Sardar | +| [HDFS-14183](https://issues.apache.org/jira/browse/HDFS-14183) | [SPS] Remove the -w parameter from the -satisfystoragepolicy usage | Major | . | Ayush Saxena | Ayush Saxena | +| [MAPREDUCE-7174](https://issues.apache.org/jira/browse/MAPREDUCE-7174) | MapReduce example wordmedian should handle generic options | Major | . | Fei Hui | Fei Hui | +| [YARN-9164](https://issues.apache.org/jira/browse/YARN-9164) | Shutdown NM may cause NPE when opportunistic container scheduling is enabled | Critical | . | lujie | lujie | +| [YARN-8567](https://issues.apache.org/jira/browse/YARN-8567) | Fetching yarn logs fails for long running application if it is not present in timeline store | Major | log-aggregation | Tarun Parimi | Tarun Parimi | +| [HADOOP-15997](https://issues.apache.org/jira/browse/HADOOP-15997) | KMS client uses wrong UGI after HADOOP-14445 | Blocker | kms | Wei-Chiu Chuang | Wei-Chiu Chuang | +| [HADOOP-16028](https://issues.apache.org/jira/browse/HADOOP-16028) | Fix NetworkTopology chooseRandom function to support excluded nodes | Major | . | Sihai Ke | Sihai Ke | +| [HADOOP-16030](https://issues.apache.org/jira/browse/HADOOP-16030) | AliyunOSS: bring fixes back from HADOOP-15671 | Blocker | fs/oss | wujinhu | wujinhu | +| [YARN-9162](https://issues.apache.org/jira/browse/YARN-9162) | Fix TestRMAdminCLI#testHelp | Major | resourcemanager, test | Ayush Saxena | Ayush Saxena | +| [HADOOP-16031](https://issues.apache.org/jira/browse/HADOOP-16031) | TestSecureLogins#testValidKerberosName fails | Major | security | Akira Ajisaka | Akira Ajisaka | +| [YARN-9173](https://issues.apache.org/jira/browse/YARN-9173) | FairShare calculation broken for large values after YARN-8833 | Major | fairscheduler | Wilfred Spiegelenburg | Wilfred Spiegelenburg | +| [HDFS-14189](https://issues.apache.org/jira/browse/HDFS-14189) | Fix intermittent failure of TestNameNodeMetrics | Major | . | Ayush Saxena | Ayush Saxena | +| [HDFS-14132](https://issues.apache.org/jira/browse/HDFS-14132) | Add BlockLocation.isStriped() to determine if block is replicated or Striped | Major | hdfs | Shweta | Shweta | +| [YARN-8833](https://issues.apache.org/jira/browse/YARN-8833) | Avoid potential integer overflow when computing fair shares | Major | fairscheduler | liyakun | liyakun | +| [HADOOP-16016](https://issues.apache.org/jira/browse/HADOOP-16016) | TestSSLFactory#testServerWeakCiphers sporadically fails in precommit builds | Major | security, test | Jason Darrell Lowe | Akira Ajisaka | +| [HDFS-14198](https://issues.apache.org/jira/browse/HDFS-14198) | Upload and Create button doesn't get enabled after getting reset. | Major | . | Ayush Saxena | Ayush Saxena | +| [HADOOP-16013](https://issues.apache.org/jira/browse/HADOOP-16013) | DecayRpcScheduler decay thread should run as a daemon | Major | ipc | Erik Krogen | Erik Krogen | +| [HADOOP-16043](https://issues.apache.org/jira/browse/HADOOP-16043) | NPE in ITestDynamoDBMetadataStore when fs.s3a.s3guard.ddb.table is not set | Major | fs/s3 | Adam Antal | Gabor Bota | +| [YARN-9179](https://issues.apache.org/jira/browse/YARN-9179) | Fix NPE in AbstractYarnScheduler#updateNewContainerInfo | Major | test | Akira Ajisaka | Akira Ajisaka | +| [YARN-8747](https://issues.apache.org/jira/browse/YARN-8747) | [UI2] YARN UI2 page loading failed due to js error under some time zone configuration | Critical | webapp | collinma | collinma | +| [YARN-9203](https://issues.apache.org/jira/browse/YARN-9203) | Fix typos in yarn-default.xml | Trivial | documentation | Rahul Padmanabhan | Rahul Padmanabhan | +| [YARN-9194](https://issues.apache.org/jira/browse/YARN-9194) | Invalid event: REGISTERED and LAUNCH\_FAILED at FAILED, and NullPointerException happens in RM while shutdown a NM | Critical | . | lujie | lujie | +| [HDFS-14175](https://issues.apache.org/jira/browse/HDFS-14175) | EC: Native XOR decoder should reset the output buffer before using it. | Major | ec, hdfs | Surendra Singh Lilhore | Ayush Saxena | +| [YARN-9197](https://issues.apache.org/jira/browse/YARN-9197) | NPE in service AM when failed to launch container | Major | yarn-native-services | kyungwan nam | kyungwan nam | +| [YARN-9204](https://issues.apache.org/jira/browse/YARN-9204) | RM fails to start if absolute resource is specified for partition capacity in CS queues | Blocker | yarn | Jiandan Yang | Jiandan Yang | +| [HDFS-14207](https://issues.apache.org/jira/browse/HDFS-14207) | ZKFC should catch exception when ha configuration missing | Major | hdfs | Fei Hui | Fei Hui | +| [HADOOP-15922](https://issues.apache.org/jira/browse/HADOOP-15922) | DelegationTokenAuthenticationFilter get wrong doAsUser since it does not decode URL | Major | common, kms | Xiaoqiao He | Xiaoqiao He | +| [HDFS-14218](https://issues.apache.org/jira/browse/HDFS-14218) | EC: Ls -e throw NPE when directory ec policy is disabled | Major | . | Surendra Singh Lilhore | Ayush Saxena | +| [YARN-9210](https://issues.apache.org/jira/browse/YARN-9210) | RM nodes web page can not display node info | Blocker | yarn | Jiandan Yang | Jiandan Yang | +| [YARN-9205](https://issues.apache.org/jira/browse/YARN-9205) | When using custom resource type, application will fail to run due to the CapacityScheduler throws InvalidResourceRequestException(GREATER\_THEN\_MAX\_ALLOCATION) | Critical | . | Zhankun Tang | Zhankun Tang | +| [YARN-8961](https://issues.apache.org/jira/browse/YARN-8961) | [UI2] Flow Run End Time shows 'Invalid date' | Major | . | Charan Hebri | Akhil PB | +| [HADOOP-16065](https://issues.apache.org/jira/browse/HADOOP-16065) | -Ddynamodb should be -Ddynamo in AWS SDK testing document | Minor | documentation | Akira Ajisaka | Akira Ajisaka | +| [HDFS-14228](https://issues.apache.org/jira/browse/HDFS-14228) | Incorrect getSnapshottableDirListing() javadoc | Major | snapshots | Wei-Chiu Chuang | Dinesh Chitlangia | +| [YARN-9222](https://issues.apache.org/jira/browse/YARN-9222) | Print launchTime in ApplicationSummary | Major | . | Jonathan Hung | Jonathan Hung | +| [YARN-8901](https://issues.apache.org/jira/browse/YARN-8901) | Restart "NEVER" policy does not work with component dependency | Critical | . | Yesha Vora | Suma Shivaprasad | +| [YARN-9237](https://issues.apache.org/jira/browse/YARN-9237) | NM should ignore sending finished apps to RM during RM fail-over | Major | yarn | Jiandan Yang | Jiandan Yang | +| [YARN-6616](https://issues.apache.org/jira/browse/YARN-6616) | YARN AHS shows submitTime for jobs same as startTime | Minor | . | Prabhu Joseph | Prabhu Joseph | +| [HDFS-14236](https://issues.apache.org/jira/browse/HDFS-14236) | Lazy persist copy/ put fails with ViewFs | Major | fs | Hanisha Koneru | Hanisha Koneru | +| [YARN-9251](https://issues.apache.org/jira/browse/YARN-9251) | Build failure for -Dhbase.profile=2.0 | Blocker | . | Rohith Sharma K S | Rohith Sharma K S | +| [YARN-9099](https://issues.apache.org/jira/browse/YARN-9099) | GpuResourceAllocator#getReleasingGpus calculates number of GPUs in a wrong way | Major | . | Szilard Nemeth | Szilard Nemeth | +| [HADOOP-16084](https://issues.apache.org/jira/browse/HADOOP-16084) | Fix the comment for getClass in Configuration | Trivial | . | Fengnan Li | Fengnan Li | +| [YARN-9262](https://issues.apache.org/jira/browse/YARN-9262) | TestRMAppAttemptTransitions is failing with an NPE | Critical | resourcemanager | Sunil G | lujie | +| [YARN-9231](https://issues.apache.org/jira/browse/YARN-9231) | TestDistributedShell fix timeout | Major | distributed-shell | Prabhu Joseph | Prabhu Joseph | +| [HDFS-14232](https://issues.apache.org/jira/browse/HDFS-14232) | libhdfs is not included in binary tarball | Critical | build, libhdfs | Akira Ajisaka | Akira Ajisaka | +| [HDFS-14158](https://issues.apache.org/jira/browse/HDFS-14158) | Checkpointer ignores configured time period \> 5 minutes | Minor | namenode | Timo Walter | Timo Walter | +| [MAPREDUCE-7177](https://issues.apache.org/jira/browse/MAPREDUCE-7177) | Disable speculative execution in TestDFSIO | Major | . | Kihwal Lee | Zhaohui Xin | +| [HADOOP-16076](https://issues.apache.org/jira/browse/HADOOP-16076) | SPNEGO+SSL Client Connections with HttpClient Broken | Major | build, security | Larry McCay | Larry McCay | +| [HDFS-14202](https://issues.apache.org/jira/browse/HDFS-14202) | "dfs.disk.balancer.max.disk.throughputInMBperSec" property is not working as per set value. | Major | diskbalancer | Ranith Sardar | Ranith Sardar | +| [YARN-9149](https://issues.apache.org/jira/browse/YARN-9149) | yarn container -status misses logUrl when integrated with ATSv2 | Major | . | Rohith Sharma K S | Abhishek Modi | +| [YARN-9246](https://issues.apache.org/jira/browse/YARN-9246) | NPE when executing a command yarn node -status or -states without additional arguments | Minor | client | Masahiro Tanaka | Masahiro Tanaka | +| [HDFS-14242](https://issues.apache.org/jira/browse/HDFS-14242) | OIV WebImageViewer: NPE when param op is not specified | Major | tools | Siyao Meng | Siyao Meng | +| [YARN-7627](https://issues.apache.org/jira/browse/YARN-7627) | [ATSv2] When passing a non-number as metricslimit, the error message is wrong | Trivial | api | Grant Sohn | Charan Hebri | +| [YARN-8498](https://issues.apache.org/jira/browse/YARN-8498) | Yarn NodeManager OOM Listener Fails Compilation on Ubuntu 18.04 | Blocker | . | Jack Bearden | Ayush Saxena | +| [YARN-9206](https://issues.apache.org/jira/browse/YARN-9206) | RMServerUtils does not count SHUTDOWN as an accepted state | Major | . | Kuhu Shukla | Kuhu Shukla | +| [HADOOP-16032](https://issues.apache.org/jira/browse/HADOOP-16032) | Distcp It should clear sub directory ACL before applying new ACL on it. | Major | tools/distcp | Ranith Sardar | Ranith Sardar | +| [HDFS-14140](https://issues.apache.org/jira/browse/HDFS-14140) | JournalNodeSyncer authentication is failing in secure cluster | Major | journal-node, security | Surendra Singh Lilhore | Surendra Singh Lilhore | +| [YARN-9257](https://issues.apache.org/jira/browse/YARN-9257) | Distributed Shell client throws a NPE for a non-existent queue | Major | distributed-shell | Charan Hebri | Charan Hebri | +| [YARN-8761](https://issues.apache.org/jira/browse/YARN-8761) | Service AM support for decommissioning component instances | Major | . | Billie Rinaldi | Billie Rinaldi | +| [HADOOP-16098](https://issues.apache.org/jira/browse/HADOOP-16098) | Fix javadoc warnings in hadoop-aws | Minor | fs/s3 | Masatake Iwasaki | Masatake Iwasaki | +| [HDFS-14266](https://issues.apache.org/jira/browse/HDFS-14266) | EC : Fsck -blockId shows null for EC Blocks if One Block Is Not Available. | Major | . | Harshakiran Reddy | Ayush Saxena | +| [HADOOP-10007](https://issues.apache.org/jira/browse/HADOOP-10007) | distcp / mv is not working on ftp | Major | fs | Fabian Zimmermann | | +| [HDFS-14274](https://issues.apache.org/jira/browse/HDFS-14274) | EC: NPE While Listing EC Policy For A Directory Following Replication Policy. | Major | erasure-coding | Souryakanta Dwivedy | Ayush Saxena | +| [HDFS-14263](https://issues.apache.org/jira/browse/HDFS-14263) | Remove unnecessary block file exists check from FsDatasetImpl#getBlockInputStream() | Major | datanode | Surendra Singh Lilhore | Surendra Singh Lilhore | +| [YARN-7761](https://issues.apache.org/jira/browse/YARN-7761) | [UI2] Clicking 'master container log' or 'Link' next to 'log' under application's appAttempt goes to Old UI's Log link | Major | yarn-ui-v2 | Sumana Sathish | Akhil PB | +| [YARN-9295](https://issues.apache.org/jira/browse/YARN-9295) | [UI2] Fix label typo in Cluster Overview page | Trivial | yarn-ui-v2 | Charan Hebri | Charan Hebri | +| [YARN-9308](https://issues.apache.org/jira/browse/YARN-9308) | fairscheduler-statedump.log gets generated regardless of service again after the merge of HDFS-7240 | Blocker | fairscheduler, scheduler | Akira Ajisaka | Wilfred Spiegelenburg | +| [YARN-9284](https://issues.apache.org/jira/browse/YARN-9284) | Fix the unit of yarn.service.am-resource.memory in the document | Minor | documentation, yarn-native-services | Masahiro Tanaka | Masahiro Tanaka | +| [YARN-9283](https://issues.apache.org/jira/browse/YARN-9283) | Javadoc of LinuxContainerExecutor#addSchedPriorityCommand has a wrong property name as reference | Minor | documentation | Szilard Nemeth | Adam Antal | +| [HADOOP-16116](https://issues.apache.org/jira/browse/HADOOP-16116) | Fix Spelling Mistakes - DECOMISSIONED | Trivial | . | David Mollitor | David Mollitor | +| [HDFS-14287](https://issues.apache.org/jira/browse/HDFS-14287) | DataXceiverServer May Double-Close PeerServer | Minor | datanode | David Mollitor | David Mollitor | +| [YARN-9286](https://issues.apache.org/jira/browse/YARN-9286) | [Timeline Server] Sorting based on FinalStatus shows pop-up message | Minor | timelineserver | Nallasivan | Bilwa S T | +| [HDFS-14081](https://issues.apache.org/jira/browse/HDFS-14081) | hdfs dfsadmin -metasave metasave\_test results NPE | Major | hdfs | Shweta | Shweta | +| [HDFS-14273](https://issues.apache.org/jira/browse/HDFS-14273) | Fix checkstyle issues in BlockLocation's method javadoc | Trivial | . | Shweta | Shweta | +| [HADOOP-15813](https://issues.apache.org/jira/browse/HADOOP-15813) | Enable more reliable SSL connection reuse | Major | common | Daryn Sharp | Daryn Sharp | +| [YARN-9319](https://issues.apache.org/jira/browse/YARN-9319) | Fix compilation issue of handling typedef an existing name by gcc compiler | Blocker | . | Wei-Chiu Chuang | Zhankun Tang | +| [YARN-9238](https://issues.apache.org/jira/browse/YARN-9238) | Avoid allocating opportunistic containers to previous/removed/non-exist application attempt | Critical | . | lujie | lujie | +| [YARN-9118](https://issues.apache.org/jira/browse/YARN-9118) | Handle exceptions with parsing user defined GPU devices in GpuDiscoverer | Major | . | Szilard Nemeth | Szilard Nemeth | +| [HDFS-14279](https://issues.apache.org/jira/browse/HDFS-14279) | [SBN Read] Race condition in ObserverReadProxyProvider | Major | hdfs, namenode | Erik Krogen | Erik Krogen | +| [HADOOP-16129](https://issues.apache.org/jira/browse/HADOOP-16129) | Misc. bug fixes for KMS Benchmark | Major | kms | Wei-Chiu Chuang | George Huang | +| [HDFS-14285](https://issues.apache.org/jira/browse/HDFS-14285) | libhdfs hdfsRead copies entire array even if its only partially filled | Major | hdfs-client, libhdfs, native | Sahil Takiar | Sahil Takiar | +| [YARN-9317](https://issues.apache.org/jira/browse/YARN-9317) | Avoid repeated YarnConfiguration#timelineServiceV2Enabled check | Major | . | Bibin Chundatt | Prabhu Joseph | +| [YARN-9300](https://issues.apache.org/jira/browse/YARN-9300) | Lazy preemption should trigger an update on queue preemption metrics for CapacityScheduler | Major | capacityscheduler | Tao Yang | Tao Yang | +| [YARN-9213](https://issues.apache.org/jira/browse/YARN-9213) | RM Web UI v1 does not show custom resource allocations for containers page | Major | . | Szilard Nemeth | Szilard Nemeth | +| [YARN-9329](https://issues.apache.org/jira/browse/YARN-9329) | updatePriority is blocked when using FairScheduler | Major | . | Jiandan Yang | Jiandan Yang | +| [HDFS-14299](https://issues.apache.org/jira/browse/HDFS-14299) | ViewFs: Correct error message for read only operations | Minor | federation | hu xiaodong | hu xiaodong | +| [HADOOP-16127](https://issues.apache.org/jira/browse/HADOOP-16127) | In ipc.Client, put a new connection could happen after stop | Major | ipc | Tsz-wo Sze | Tsz-wo Sze | +| [YARN-8378](https://issues.apache.org/jira/browse/YARN-8378) | ApplicationHistoryManagerImpl#getApplications doesn't honor filters | Minor | resourcemanager, yarn | Lantao Jin | Lantao Jin | +| [YARN-9311](https://issues.apache.org/jira/browse/YARN-9311) | TestRMRestart hangs due to a deadlock | Major | . | Prabhu Joseph | Prabhu Joseph | +| [YARN-9248](https://issues.apache.org/jira/browse/YARN-9248) | RMContainerImpl:Invalid event: ACQUIRED at KILLED | Major | . | lujie | lujie | +| [YARN-9318](https://issues.apache.org/jira/browse/YARN-9318) | Resources#multiplyAndRoundUp does not consider Resource Types | Major | . | Szilard Nemeth | Szilard Nemeth | +| [HADOOP-16107](https://issues.apache.org/jira/browse/HADOOP-16107) | FilterFileSystem doesn't wrap all create() or new builder calls; may skip CRC logic | Blocker | fs | Steve Loughran | Steve Loughran | +| [HADOOP-16149](https://issues.apache.org/jira/browse/HADOOP-16149) | hadoop-mapreduce-client-app build not converging due to transient dependencies | Major | build | Steve Loughran | Steve Loughran | +| [YARN-9334](https://issues.apache.org/jira/browse/YARN-9334) | YARN Service Client does not work with SPNEGO when knox is configured | Major | yarn-native-services | Tarun Parimi | Billie Rinaldi | +| [YARN-9323](https://issues.apache.org/jira/browse/YARN-9323) | FSLeafQueue#computeMaxAMResource does not override zero values for custom resources | Major | . | Szilard Nemeth | Szilard Nemeth | +| [HADOOP-16067](https://issues.apache.org/jira/browse/HADOOP-16067) | Incorrect Format Debug Statement KMSACLs | Trivial | kms | David Mollitor | Charan Hebri | +| [HDFS-14324](https://issues.apache.org/jira/browse/HDFS-14324) | Fix TestDataNodeVolumeFailure | Major | . | Ayush Saxena | Ayush Saxena | +| [HDFS-13997](https://issues.apache.org/jira/browse/HDFS-13997) | Secondary NN Web UI displays nothing, and the console log shows moment is not defined. | Major | namenode | Rui Chen | Ayush Saxena | +| [HDFS-14272](https://issues.apache.org/jira/browse/HDFS-14272) | [SBN read] ObserverReadProxyProvider should sync with active txnID on startup | Major | tools | Wei-Chiu Chuang | Erik Krogen | +| [HDFS-14314](https://issues.apache.org/jira/browse/HDFS-14314) | fullBlockReportLeaseId should be reset after registering to NN | Critical | datanode | star | star | +| [YARN-7266](https://issues.apache.org/jira/browse/YARN-7266) | Timeline Server event handler threads locked | Major | ATSv2, timelineserver | Venkata Puneet Ravuri | Prabhu Joseph | +| [HADOOP-16150](https://issues.apache.org/jira/browse/HADOOP-16150) | checksumFS doesn't wrap concat(): concatenated files don't have checksums | Major | fs | Steve Loughran | Steve Loughran | +| [YARN-8803](https://issues.apache.org/jira/browse/YARN-8803) | [UI2] Show flow runs in the order of recently created time in graph widgets | Major | yarn-ui-v2 | Akhil PB | Akhil PB | +| [HDFS-14111](https://issues.apache.org/jira/browse/HDFS-14111) | hdfsOpenFile on HDFS causes unnecessary IO from file offset 0 | Major | hdfs-client, libhdfs | Todd Lipcon | Sahil Takiar | +| [YARN-9341](https://issues.apache.org/jira/browse/YARN-9341) | Reentrant lock() before try | Minor | yarn | Prabhu Joseph | Prabhu Joseph | +| [HDFS-14317](https://issues.apache.org/jira/browse/HDFS-14317) | Standby does not trigger edit log rolling when in-progress edit log tailing is enabled | Critical | . | Ekanth Sethuramalingam | Ekanth Sethuramalingam | +| [HADOOP-16174](https://issues.apache.org/jira/browse/HADOOP-16174) | Disable wildfly logs to the console | Major | fs/azure | Denes Gerencser | Denes Gerencser | +| [HDFS-14347](https://issues.apache.org/jira/browse/HDFS-14347) | Restore a comment line mistakenly removed in ProtobufRpcEngine | Major | . | Konstantin Shvachko | Fengnan Li | +| [HDFS-14333](https://issues.apache.org/jira/browse/HDFS-14333) | Datanode fails to start if any disk has errors during Namenode registration | Major | datanode | Stephen O'Donnell | Stephen O'Donnell | +| [HADOOP-16192](https://issues.apache.org/jira/browse/HADOOP-16192) | CallQueue backoff bug fixes: doesn't perform backoff when add() is used, and doesn't update backoff when refreshed | Major | ipc | Erik Krogen | Erik Krogen | +| [YARN-9365](https://issues.apache.org/jira/browse/YARN-9365) | fix wrong command in TimelineServiceV2.md | Major | timelineserver | Runlin Zhang | Runlin Zhang | +| [YARN-9357](https://issues.apache.org/jira/browse/YARN-9357) | Modify HBase Liveness monitor log to debug | Minor | ATSv2 | Prabhu Joseph | Prabhu Joseph | +| [HDFS-14359](https://issues.apache.org/jira/browse/HDFS-14359) | Inherited ACL permissions masked when parent directory does not exist (mkdir -p) | Major | . | Stephen O'Donnell | Stephen O'Donnell | +| [HDFS-14037](https://issues.apache.org/jira/browse/HDFS-14037) | Fix SSLFactory truststore reloader thread leak in URLConnectionFactory | Major | hdfs-client, webhdfs | Takanobu Asanuma | Takanobu Asanuma | +| [HDFS-14348](https://issues.apache.org/jira/browse/HDFS-14348) | Fix JNI exception handling issues in libhdfs | Major | hdfs-client, libhdfs, native | Sahil Takiar | Sahil Takiar | +| [YARN-9411](https://issues.apache.org/jira/browse/YARN-9411) | TestYarnNativeServices fails sporadically with bind address in use | Major | test, yarn-native-services | Prabhu Joseph | Prabhu Joseph | +| [HADOOP-15652](https://issues.apache.org/jira/browse/HADOOP-15652) | Fix typos SPENGO into SPNEGO | Trivial | documentation | okumin | okumin | +| [HADOOP-16199](https://issues.apache.org/jira/browse/HADOOP-16199) | KMSLoadBlanceClientProvider does not select token correctly | Major | . | Xiaoyu Yao | Xiaoyu Yao | +| [YARN-9227](https://issues.apache.org/jira/browse/YARN-9227) | DistributedShell RelativePath is not removed at end | Minor | distributed-shell | Prabhu Joseph | Prabhu Joseph | +| [YARN-9431](https://issues.apache.org/jira/browse/YARN-9431) | Fix flaky junit test fair.TestAppRunnability after YARN-8967 | Minor | fairscheduler, test | Wilfred Spiegelenburg | Wilfred Spiegelenburg | +| [HADOOP-16225](https://issues.apache.org/jira/browse/HADOOP-16225) | Fix links to the developer mailing lists in DownstreamDev.md | Minor | documentation | Akira Ajisaka | Wanqiang Ji | +| [HADOOP-16226](https://issues.apache.org/jira/browse/HADOOP-16226) | new Path(String str) does not remove all the trailing slashes of str | Minor | fs | Akira Ajisaka | Akira Ajisaka | +| [HADOOP-16232](https://issues.apache.org/jira/browse/HADOOP-16232) | Fix errors in the checkstyle configration xmls | Major | build | Akira Ajisaka | Wanqiang Ji | +| [YARN-4901](https://issues.apache.org/jira/browse/YARN-4901) | QueueMetrics needs to be cleared before MockRM is initialized | Major | scheduler | Daniel Templeton | Peter Bacsko | +| [HADOOP-16011](https://issues.apache.org/jira/browse/HADOOP-16011) | OsSecureRandom very slow compared to other SecureRandom implementations | Major | security | Todd Lipcon | Siyao Meng | +| [HDFS-14389](https://issues.apache.org/jira/browse/HDFS-14389) | getAclStatus returns incorrect permissions and owner when an iNodeAttributeProvider is configured | Major | namenode | Stephen O'Donnell | Stephen O'Donnell | +| [YARN-9396](https://issues.apache.org/jira/browse/YARN-9396) | YARN\_RM\_CONTAINER\_CREATED published twice to ATS | Major | ATSv2 | Prabhu Joseph | Prabhu Joseph | +| [HDFS-14407](https://issues.apache.org/jira/browse/HDFS-14407) | Fix misuse of SLF4j logging API in DatasetVolumeChecker#checkAllVolumes | Minor | . | Wanqiang Ji | Wanqiang Ji | +| [HDFS-14416](https://issues.apache.org/jira/browse/HDFS-14416) | Fix TestHdfsConfigFields for field dfs.client.failover.resolver.useFQDN | Major | . | Íñigo Goiri | Fengnan Li | +| [YARN-9413](https://issues.apache.org/jira/browse/YARN-9413) | Queue resource leak after app fail for CapacityScheduler | Major | capacityscheduler | Tao Yang | Tao Yang | +| [HADOOP-14635](https://issues.apache.org/jira/browse/HADOOP-14635) | Javadoc correction for AccessControlList#buildACL | Minor | documentation | Bibin Chundatt | Yeliang Cang | +| [HADOOP-12890](https://issues.apache.org/jira/browse/HADOOP-12890) | Fix typo in AbstractService | Trivial | documentation | Mike Drob | Gabor Liptak | +| [HADOOP-16240](https://issues.apache.org/jira/browse/HADOOP-16240) | start-build-env.sh can consume all disk space during image creation | Minor | build | Craig Condit | Craig Condit | +| [HDFS-12245](https://issues.apache.org/jira/browse/HDFS-12245) | Fix INodeId javadoc | Major | documentation, namenode | Wei-Chiu Chuang | Adam Antal | +| [HDFS-14420](https://issues.apache.org/jira/browse/HDFS-14420) | Fix typo in KeyShell console | Minor | kms | hu xiaodong | hu xiaodong | +| [HADOOP-14544](https://issues.apache.org/jira/browse/HADOOP-14544) | DistCp documentation for command line options is misaligned. | Minor | documentation | Chris Nauroth | Masatake Iwasaki | +| [YARN-9481](https://issues.apache.org/jira/browse/YARN-9481) | [JDK 11] Build fails due to hard-coded target version in hadoop-yarn-applications-catalog-webapp | Major | build | Kei Kori | Kei Kori | +| [YARN-9379](https://issues.apache.org/jira/browse/YARN-9379) | Can't specify docker runtime through environment | Minor | nodemanager | caozhiqiang | caozhiqiang | +| [YARN-9336](https://issues.apache.org/jira/browse/YARN-9336) | JobHistoryServer leaks CLOSE\_WAIT tcp connections when using LogAggregationIndexedFileController | Major | log-aggregation | Tarun Parimi | Tarun Parimi | +| [HDFS-10477](https://issues.apache.org/jira/browse/HDFS-10477) | Stop decommission a rack of DataNodes caused NameNode fail over to standby | Major | namenode | yunjiong zhao | yunjiong zhao | +| [HADOOP-15881](https://issues.apache.org/jira/browse/HADOOP-15881) | Remove JUnit from LICENSE.txt | Major | . | Akira Ajisaka | Kei Kori | +| [YARN-9487](https://issues.apache.org/jira/browse/YARN-9487) | NodeManager native build shouldn't link against librt on macOS | Major | nodemanager | Siyao Meng | Siyao Meng | +| [YARN-6695](https://issues.apache.org/jira/browse/YARN-6695) | Race condition in RM for publishing container events vs appFinished events causes NPE | Critical | . | Rohith Sharma K S | Prabhu Joseph | +| [YARN-8622](https://issues.apache.org/jira/browse/YARN-8622) | NodeManager native build fails due to getgrouplist not found on macOS | Major | nodemanager | Ewan Higgs | Siyao Meng | +| [YARN-9495](https://issues.apache.org/jira/browse/YARN-9495) | Fix findbugs warnings in hadoop-yarn-server-resourcemanager module | Minor | . | Tao Yang | Tao Yang | +| [HADOOP-16265](https://issues.apache.org/jira/browse/HADOOP-16265) | Configuration#getTimeDuration is not consistent between default value and manual settings. | Major | . | star | star | +| [HDFS-14445](https://issues.apache.org/jira/browse/HDFS-14445) | TestTrySendErrorReportWhenNNThrowsIOException fails in trunk | Major | . | Ayush Saxena | Ayush Saxena | +| [HDFS-14435](https://issues.apache.org/jira/browse/HDFS-14435) | ObserverReadProxyProvider is unable to properly fetch HAState from Standby NNs | Major | ha, nn | Erik Krogen | Erik Krogen | +| [YARN-9339](https://issues.apache.org/jira/browse/YARN-9339) | Apps pending metric incorrect after moving app to a new queue | Minor | . | Billie Rinaldi | Abhishek Modi | +| [YARN-9491](https://issues.apache.org/jira/browse/YARN-9491) | TestApplicationMasterServiceFair\>ApplicationMasterServiceTestBase.testUpdateTrackingUrl fails intermittent | Minor | . | Prabhu Joseph | Prabhu Joseph | +| [YARN-9501](https://issues.apache.org/jira/browse/YARN-9501) | TestCapacitySchedulerOvercommit#testReducePreemptAndCancel fails intermittent | Minor | capacityscheduler, test | Prabhu Joseph | Prabhu Joseph | +| [YARN-9424](https://issues.apache.org/jira/browse/YARN-9424) | Change getDeclaredMethods to getMethods in FederationClientInterceptor#invokeConcurrent() | Major | federation | Shen Yinjie | Shen Yinjie | +| [MAPREDUCE-7200](https://issues.apache.org/jira/browse/MAPREDUCE-7200) | Remove stale eclipse templates | Minor | . | Akira Ajisaka | Wanqiang Ji | +| [HDFS-13677](https://issues.apache.org/jira/browse/HDFS-13677) | Dynamic refresh Disk configuration results in overwriting VolumeMap | Blocker | . | xuzq | xuzq | +| [YARN-6929](https://issues.apache.org/jira/browse/YARN-6929) | yarn.nodemanager.remote-app-log-dir structure is not scalable | Major | log-aggregation | Prabhu Joseph | Prabhu Joseph | +| [YARN-9285](https://issues.apache.org/jira/browse/YARN-9285) | RM UI progress column is of wrong type | Minor | yarn | Ahmed Hussein | Ahmed Hussein | +| [YARN-9528](https://issues.apache.org/jira/browse/YARN-9528) | Federation RMs starting up at the same time can give duplicate application IDs | Minor | . | Young Chen | Young Chen | +| [HDFS-14438](https://issues.apache.org/jira/browse/HDFS-14438) | Fix typo in OfflineEditsVisitorFactory | Major | hdfs | bianqi | bianqi | +| [HDFS-14372](https://issues.apache.org/jira/browse/HDFS-14372) | NPE while DN is shutting down | Major | . | lujie | lujie | +| [YARN-9524](https://issues.apache.org/jira/browse/YARN-9524) | TestAHSWebServices and TestLogsCLI test case failures | Major | log-aggregation, test | Prabhu Joseph | Prabhu Joseph | +| [YARN-9432](https://issues.apache.org/jira/browse/YARN-9432) | Reserved containers leak after its request has been cancelled or satisfied when multi-nodes enabled | Major | capacityscheduler | Tao Yang | Tao Yang | +| [YARN-9513](https://issues.apache.org/jira/browse/YARN-9513) | [JDK11] Fix TestMetricsInvariantChecker#testManyRuns in case of JDK greater than 8 | Major | test | Siyao Meng | Adam Antal | +| [HADOOP-16293](https://issues.apache.org/jira/browse/HADOOP-16293) | AuthenticationFilterInitializer doc has speudo instead of pseudo | Trivial | auth, documentation | Prabhu Joseph | Prabhu Joseph | +| [YARN-9535](https://issues.apache.org/jira/browse/YARN-9535) | Typos in Docker documentation | Major | . | Szilard Nemeth | Charan Hebri | +| [HADOOP-16299](https://issues.apache.org/jira/browse/HADOOP-16299) | [JDK 11] Build fails without specifying -Djavac.version=11 | Major | . | Akira Ajisaka | Akira Ajisaka | +| [YARN-9483](https://issues.apache.org/jira/browse/YARN-9483) | DistributedShell does not release container when failed to localize at launch | Major | . | Prabhu Joseph | Prabhu Joseph | +| [HADOOP-16278](https://issues.apache.org/jira/browse/HADOOP-16278) | With S3A Filesystem, Long Running services End up Doing lot of GC and eventually die | Major | common, fs/s3, metrics | Rajat Khandelwal | Rajat Khandelwal | +| [YARN-9522](https://issues.apache.org/jira/browse/YARN-9522) | AppBlock ignores full qualified class name of PseudoAuthenticationHandler | Minor | . | Prabhu Joseph | Prabhu Joseph | +| [YARN-9504](https://issues.apache.org/jira/browse/YARN-9504) | [UI2] Fair scheduler queue view page does not show actual capacity | Major | fairscheduler, yarn-ui-v2 | Zoltan Siegl | Zoltan Siegl | +| [YARN-9493](https://issues.apache.org/jira/browse/YARN-9493) | Scheduler Page does not display the right page by query string | Major | capacity scheduler, resourcemanager, webapp | Wanqiang Ji | Wanqiang Ji | +| [HADOOP-16161](https://issues.apache.org/jira/browse/HADOOP-16161) | NetworkTopology#getWeightUsingNetworkLocation return unexpected result | Major | net | Xiaoqiao He | Xiaoqiao He | +| [YARN-9519](https://issues.apache.org/jira/browse/YARN-9519) | TFile log aggregation file format is not working for yarn.log-aggregation.TFile.remote-app-log-dir config | Major | log-aggregation | Adam Antal | Adam Antal | +| [HDFS-14482](https://issues.apache.org/jira/browse/HDFS-14482) | Crash when using libhdfs with bad classpath | Major | . | Todd Lipcon | Sahil Takiar | +| [YARN-9508](https://issues.apache.org/jira/browse/YARN-9508) | YarnConfiguration areNodeLabel enabled is costly in allocation flow | Critical | . | Bibin Chundatt | Bilwa S T | +| [HADOOP-16247](https://issues.apache.org/jira/browse/HADOOP-16247) | NPE in FsUrlConnection | Major | hdfs-client | Karthik Palanisamy | Karthik Palanisamy | +| [YARN-9554](https://issues.apache.org/jira/browse/YARN-9554) | TimelineEntity DAO has java.util.Set interface which JAXB can't handle | Major | timelineservice | Prabhu Joseph | Prabhu Joseph | +| [HDFS-14323](https://issues.apache.org/jira/browse/HDFS-14323) | Distcp fails in Hadoop 3.x when 2.x source webhdfs url has special characters in hdfs file path | Major | webhdfs | Srinivasu Majeti | Srinivasu Majeti | +| [YARN-9575](https://issues.apache.org/jira/browse/YARN-9575) | Fix TestYarnConfigurationFields testcase failing | Major | test, yarn-native-services | Prabhu Joseph | Prabhu Joseph | +| [MAPREDUCE-7205](https://issues.apache.org/jira/browse/MAPREDUCE-7205) | Treat container scheduler kill exit code as a task attempt killing event | Major | applicationmaster, mr-am, mrv2 | Wanqiang Ji | Wanqiang Ji | +| [MAPREDUCE-7198](https://issues.apache.org/jira/browse/MAPREDUCE-7198) | mapreduce.task.timeout=0 configuration used to disable timeout doesn't work | Minor | . | Akira Ajisaka | Akira Ajisaka | +| [HADOOP-12948](https://issues.apache.org/jira/browse/HADOOP-12948) | Remove the defunct startKdc profile from hadoop-common | Major | test | Wei-Chiu Chuang | Wei-Chiu Chuang | +| [YARN-9080](https://issues.apache.org/jira/browse/YARN-9080) | Bucket Directories as part of ATS done accumulates | Major | . | Prabhu Joseph | Prabhu Joseph | +| [YARN-9482](https://issues.apache.org/jira/browse/YARN-9482) | DistributedShell job with localization fails in unsecure cluster | Major | distributed-shell | Prabhu Joseph | Prabhu Joseph | +| [YARN-9558](https://issues.apache.org/jira/browse/YARN-9558) | Log Aggregation testcases failing | Major | log-aggregation, test | Prabhu Joseph | Prabhu Joseph | +| [HDFS-14500](https://issues.apache.org/jira/browse/HDFS-14500) | NameNode StartupProgress continues to report edit log segments after the LOADING\_EDITS phase is finished | Major | namenode | Erik Krogen | Erik Krogen | +| [YARN-9503](https://issues.apache.org/jira/browse/YARN-9503) | Fix JavaDoc error in TestSchedulerOvercommit | Minor | documentation, test | Wanqiang Ji | Wanqiang Ji | +| [YARN-9500](https://issues.apache.org/jira/browse/YARN-9500) | Fix typos in ResourceModel.md | Trivial | documentation | leiqiang | leiqiang | +| [HDFS-14434](https://issues.apache.org/jira/browse/HDFS-14434) | webhdfs that connect secure hdfs should not use user.name parameter | Minor | webhdfs | KWON BYUNGCHANG | KWON BYUNGCHANG | +| [HADOOP-16331](https://issues.apache.org/jira/browse/HADOOP-16331) | Fix ASF License check in pom.xml | Major | . | Wanqiang Ji | Akira Ajisaka | +| [HDFS-14512](https://issues.apache.org/jira/browse/HDFS-14512) | ONE\_SSD policy will be violated while write data with DistributedFileSystem.create(....favoredNodes) | Major | . | Shen Yinjie | Ayush Saxena | +| [HADOOP-16334](https://issues.apache.org/jira/browse/HADOOP-16334) | Fix yetus-wrapper not working when HADOOP\_YETUS\_VERSION \>= 0.9.0 | Major | yetus | Wanqiang Ji | Wanqiang Ji | +| [YARN-9553](https://issues.apache.org/jira/browse/YARN-9553) | Fix NPE in EntityGroupFSTimelineStore#getEntityTimelines | Major | timelineservice | Prabhu Joseph | Prabhu Joseph | +| [HDFS-14521](https://issues.apache.org/jira/browse/HDFS-14521) | Suppress setReplication logging. | Major | . | Kihwal Lee | Kihwal Lee | +| [YARN-9033](https://issues.apache.org/jira/browse/YARN-9033) | ResourceHandlerChain#bootstrap is invoked twice during NM start if LinuxContainerExecutor enabled | Major | yarn | Zhankun Tang | Zhankun Tang | +| [YARN-9027](https://issues.apache.org/jira/browse/YARN-9027) | EntityGroupFSTimelineStore fails to init LevelDBCacheTimelineStore | Major | timelineserver | Prabhu Joseph | Prabhu Joseph | +| [YARN-9507](https://issues.apache.org/jira/browse/YARN-9507) | Fix NPE in NodeManager#serviceStop on startup failure | Minor | . | Bilwa S T | Bilwa S T | +| [YARN-8947](https://issues.apache.org/jira/browse/YARN-8947) | [UI2] Active User info missing from UI2 | Major | yarn-ui-v2 | Akhil PB | Akhil PB | +| [YARN-8906](https://issues.apache.org/jira/browse/YARN-8906) | [UI2] NM hostnames not displayed correctly in Node Heatmap Chart | Major | . | Charan Hebri | Akhil PB | +| [YARN-9580](https://issues.apache.org/jira/browse/YARN-9580) | Fulfilled reservation information in assignment is lost when transferring in ParentQueue#assignContainers | Major | capacityscheduler | Tao Yang | Tao Yang | +| [YARN-9595](https://issues.apache.org/jira/browse/YARN-9595) | FPGA plugin: NullPointerException in FpgaNodeResourceUpdateHandler.updateConfiguredResource() | Major | nodemanager | Peter Bacsko | Peter Bacsko | +| [YARN-8625](https://issues.apache.org/jira/browse/YARN-8625) | Aggregate Resource Allocation for each job is not present in ATS | Major | ATSv2 | Prabhu Joseph | Prabhu Joseph | +| [YARN-9600](https://issues.apache.org/jira/browse/YARN-9600) | Support self-adaption width for columns of containers table on app attempt page | Minor | webapp | Tao Yang | Tao Yang | +| [HDFS-14527](https://issues.apache.org/jira/browse/HDFS-14527) | Stop all DataNodes may result in NN terminate | Major | namenode | Xiaoqiao He | Xiaoqiao He | +| [HDFS-14486](https://issues.apache.org/jira/browse/HDFS-14486) | The exception classes in some throw statements do not accurately describe why they are thrown | Minor | . | Haicheng Chen | Ayush Saxena | +| [HADOOP-16345](https://issues.apache.org/jira/browse/HADOOP-16345) | Potential NPE when instantiating FairCallQueue metrics | Major | ipc | Erik Krogen | Erik Krogen | +| [HDFS-14494](https://issues.apache.org/jira/browse/HDFS-14494) | Move Server logging of StatedId inside receiveRequestState() | Major | . | Konstantin Shvachko | Shweta | +| [YARN-9594](https://issues.apache.org/jira/browse/YARN-9594) | Fix missing break statement in ContainerScheduler#handle | Major | . | lujie | lujie | +| [YARN-9565](https://issues.apache.org/jira/browse/YARN-9565) | RMAppImpl#ranNodes not cleared on FinalTransition | Major | . | Bibin Chundatt | Bilwa S T | +| [YARN-9547](https://issues.apache.org/jira/browse/YARN-9547) | ContainerStatusPBImpl default execution type is not returned | Major | . | Bibin Chundatt | Bilwa S T | +| [HDFS-13231](https://issues.apache.org/jira/browse/HDFS-13231) | Extend visualization for Decommissioning, Maintenance Mode under Datanode tab in the NameNode UI | Major | datanode, namenode | Haibo Yan | Stephen O'Donnell | +| [HADOOP-15960](https://issues.apache.org/jira/browse/HADOOP-15960) | Update guava to 27.0-jre in hadoop-project | Critical | common, security | Gabor Bota | Gabor Bota | +| [HDFS-14549](https://issues.apache.org/jira/browse/HDFS-14549) | EditLogTailer shouldn't output full stack trace when interrupted | Major | . | Íñigo Goiri | Íñigo Goiri | +| [HADOOP-16373](https://issues.apache.org/jira/browse/HADOOP-16373) | Fix typo in FileSystemShell#test documentation | Trivial | documentation | Dinesh Chitlangia | Dinesh Chitlangia | +| [HDFS-14556](https://issues.apache.org/jira/browse/HDFS-14556) | Spelling Mistake "gloablly" | Trivial | hdfs-client | David Mollitor | David Mollitor | +| [HDFS-14535](https://issues.apache.org/jira/browse/HDFS-14535) | The default 8KB buffer in requestFileDescriptors#BufferedOutputStream is causing lots of heap allocation in HBase when using short-circut read | Major | hdfs-client | Zheng Hu | Zheng Hu | +| [HDFS-13730](https://issues.apache.org/jira/browse/HDFS-13730) | BlockReaderRemote.sendReadResult throws NPE | Major | hdfs-client | Wei-Chiu Chuang | Yuanbo Liu | +| [HDFS-12315](https://issues.apache.org/jira/browse/HDFS-12315) | Use Path instead of String in the TestHdfsAdmin.verifyOpenFiles() | Trivial | . | Oleg Danilov | Oleg Danilov | +| [HDFS-12314](https://issues.apache.org/jira/browse/HDFS-12314) | Typo in the TestDataNodeHotSwapVolumes.testAddOneNewVolume() | Trivial | . | Oleg Danilov | Oleg Danilov | +| [YARN-9584](https://issues.apache.org/jira/browse/YARN-9584) | Should put initializeProcessTrees method call before get pid | Critical | nodemanager | Wanqiang Ji | Wanqiang Ji | +| [HDFS-14010](https://issues.apache.org/jira/browse/HDFS-14010) | Pass correct DF usage to ReservedSpaceCalculator builder | Minor | . | Lukas Majercak | Lukas Majercak | +| [HDFS-14078](https://issues.apache.org/jira/browse/HDFS-14078) | Admin helper fails to prettify NullPointerExceptions | Major | . | Marton Elek | Marton Elek | +| [HDFS-14101](https://issues.apache.org/jira/browse/HDFS-14101) | Random failure of testListCorruptFilesCorruptedBlock | Major | test | Kihwal Lee | Zsolt Venczel | +| [HDFS-14537](https://issues.apache.org/jira/browse/HDFS-14537) | Journaled Edits Cache is not cleared when formatting the JN | Major | . | Ranith Sardar | Ranith Sardar | +| [HDFS-14581](https://issues.apache.org/jira/browse/HDFS-14581) | Appending to EC files crashes NameNode | Critical | erasure-coding | Surendra Singh Lilhore | Surendra Singh Lilhore | +| [HDFS-14465](https://issues.apache.org/jira/browse/HDFS-14465) | When the Block expected replications is larger than the number of DataNodes, entering maintenance will never exit. | Major | . | Yicong Cai | Yicong Cai | +| [HDFS-13893](https://issues.apache.org/jira/browse/HDFS-13893) | DiskBalancer: no validations for Disk balancer commands | Major | diskbalancer | Harshakiran Reddy | Lokesh Jain | +| [YARN-9209](https://issues.apache.org/jira/browse/YARN-9209) | When nodePartition is not set in Placement Constraints, containers are allocated only in default partition | Major | capacity scheduler, scheduler | Tarun Parimi | Tarun Parimi | +| [HADOOP-15989](https://issues.apache.org/jira/browse/HADOOP-15989) | Synchronized at CompositeService#removeService is not required | Major | common | Prabhu Joseph | Prabhu Joseph | +| [HDFS-12487](https://issues.apache.org/jira/browse/HDFS-12487) | FsDatasetSpi.isValidBlock() lacks null pointer check inside and neither do the callers | Major | balancer & mover, diskbalancer | liumi | liumi | +| [HDFS-14074](https://issues.apache.org/jira/browse/HDFS-14074) | DataNode runs async disk checks maybe throws NullPointerException, and DataNode failed to register to NameSpace. | Major | hdfs | guangyi lu | guangyi lu | +| [HDFS-14541](https://issues.apache.org/jira/browse/HDFS-14541) | When evictableMmapped or evictable size is zero, do not throw NoSuchElementException | Major | hdfs-client, performance | Zheng Hu | Lisheng Sun | +| [HDFS-13371](https://issues.apache.org/jira/browse/HDFS-13371) | NPE for FsServerDefaults.getKeyProviderUri() for clientProtocol communication between 2.7 and 3.X | Minor | . | Sherwood Zheng | Sherwood Zheng | +| [HDFS-14598](https://issues.apache.org/jira/browse/HDFS-14598) | Findbugs warning caused by HDFS-12487 | Minor | diskbalancer | Wei-Chiu Chuang | Xiaoqiao He | +| [HADOOP-16390](https://issues.apache.org/jira/browse/HADOOP-16390) | Build fails due to bad use of '\>' in javadoc | Critical | build | Kei Kori | Kei Kori | +| [YARN-9639](https://issues.apache.org/jira/browse/YARN-9639) | DecommissioningNodesWatcher cause memory leak | Blocker | . | Bibin Chundatt | Bilwa S T | +| [HDFS-14599](https://issues.apache.org/jira/browse/HDFS-14599) | HDFS-12487 breaks test TestDiskBalancer.testDiskBalancerWithFedClusterWithOneNameServiceEmpty | Major | diskbalancer | Wei-Chiu Chuang | Xiaoqiao He | +| [YARN-9581](https://issues.apache.org/jira/browse/YARN-9581) | Fix WebAppUtils#getRMWebAppURLWithScheme ignores rm2 | Major | client | Prabhu Joseph | Prabhu Joseph | +| [HDFS-14618](https://issues.apache.org/jira/browse/HDFS-14618) | Incorrect synchronization of ArrayList field (ArrayList is thread-unsafe). | Critical | . | Paul Ward | Paul Ward | +| [YARN-9661](https://issues.apache.org/jira/browse/YARN-9661) | Fix typos in LocalityMulticastAMRMProxyPolicy and AbstractConfigurableFederationPolicy | Major | federation, yarn | hunshenshi | hunshenshi | +| [HDFS-14610](https://issues.apache.org/jira/browse/HDFS-14610) | HashMap is not thread safe. Field storageMap is typically synchronized by storageMap. However, in one place, field storageMap is not protected with synchronized. | Critical | . | Paul Ward | Paul Ward | +| [YARN-9327](https://issues.apache.org/jira/browse/YARN-9327) | Improve synchronisation in ProtoUtils#convertToProtoFormat block | Critical | . | Bibin Chundatt | Bibin Chundatt | +| [YARN-9655](https://issues.apache.org/jira/browse/YARN-9655) | AllocateResponse in FederationInterceptor lost applicationPriority | Major | federation | hunshenshi | hunshenshi | +| [HADOOP-16385](https://issues.apache.org/jira/browse/HADOOP-16385) | Namenode crashes with "RedundancyMonitor thread received Runtime exception" | Major | . | krishna reddy | Ayush Saxena | +| [YARN-9658](https://issues.apache.org/jira/browse/YARN-9658) | Fix UT failures in TestLeafQueue | Minor | . | Tao Yang | Tao Yang | +| [YARN-9644](https://issues.apache.org/jira/browse/YARN-9644) | First RMContext object is always leaked during switch over | Blocker | . | Bibin Chundatt | Bibin Chundatt | +| [HDFS-14629](https://issues.apache.org/jira/browse/HDFS-14629) | Property value Hard Coded in DNConf.java | Trivial | . | hemanthboyina | hemanthboyina | +| [HADOOP-16411](https://issues.apache.org/jira/browse/HADOOP-16411) | Fix javadoc warnings in hadoop-dynamometer | Minor | tools | Masatake Iwasaki | Masatake Iwasaki | +| [YARN-9557](https://issues.apache.org/jira/browse/YARN-9557) | Application fails in diskchecker when ReadWriteDiskValidator is configured. | Critical | nodemanager | Anuruddh Nayak | Bilwa S T | +| [HDFS-12703](https://issues.apache.org/jira/browse/HDFS-12703) | Exceptions are fatal to decommissioning monitor | Critical | namenode | Daryn Sharp | Xiaoqiao He | +| [HADOOP-16381](https://issues.apache.org/jira/browse/HADOOP-16381) | The JSON License is included in binary tarball via azure-documentdb:1.16.2 | Blocker | . | Akira Ajisaka | Sushil Ks | +| [HDFS-12748](https://issues.apache.org/jira/browse/HDFS-12748) | NameNode memory leak when accessing webhdfs GETHOMEDIRECTORY | Major | hdfs | Jiandan Yang | Weiwei Yang | +| [HADOOP-16418](https://issues.apache.org/jira/browse/HADOOP-16418) | Fix checkstyle and findbugs warnings in hadoop-dynamometer | Minor | tools | Masatake Iwasaki | Erik Krogen | +| [YARN-9625](https://issues.apache.org/jira/browse/YARN-9625) | UI2 - No link to a queue on the Queues page for Fair Scheduler | Major | . | Charan Hebri | Zoltan Siegl | +| [HDFS-14466](https://issues.apache.org/jira/browse/HDFS-14466) | Add a regression test for HDFS-14323 | Minor | fs, test, webhdfs | Yuya Ebihara | Masatake Iwasaki | +| [HDFS-14499](https://issues.apache.org/jira/browse/HDFS-14499) | Misleading REM\_QUOTA value with snapshot and trash feature enabled for a directory | Major | snapshots | Shashikant Banerjee | Shashikant Banerjee | +| [YARN-9235](https://issues.apache.org/jira/browse/YARN-9235) | If linux container executor is not set for a GPU cluster GpuResourceHandlerImpl is not initialized and NPE is thrown | Major | yarn | Antal Bálint Steinbach | Adam Antal | +| [YARN-9626](https://issues.apache.org/jira/browse/YARN-9626) | UI2 - Fair scheduler queue apps page issues | Major | . | Charan Hebri | Zoltan Siegl | +| [HDFS-14642](https://issues.apache.org/jira/browse/HDFS-14642) | processMisReplicatedBlocks does not return correct processed count | Major | namenode | Stephen O'Donnell | Stephen O'Donnell | +| [YARN-9645](https://issues.apache.org/jira/browse/YARN-9645) | Fix Invalid event FINISHED\_CONTAINERS\_PULLED\_BY\_AM at NEW on NM restart | Major | . | krishna reddy | Bilwa S T | +| [YARN-9646](https://issues.apache.org/jira/browse/YARN-9646) | DistributedShell tests failed to bind to a local host name | Major | test | Ray Yang | Ray Yang | +| [YARN-9682](https://issues.apache.org/jira/browse/YARN-9682) | Wrong log message when finalizing the upgrade | Trivial | . | kyungwan nam | kyungwan nam | +| [HDFS-13647](https://issues.apache.org/jira/browse/HDFS-13647) | Fix the description of storageType option for space quota | Major | documentation, tools | Takanobu Asanuma | Takanobu Asanuma | +| [MAPREDUCE-6521](https://issues.apache.org/jira/browse/MAPREDUCE-6521) | MiniMRYarnCluster should not create /tmp/hadoop-yarn/staging on local filesystem in unit test | Major | test | Masatake Iwasaki | Masatake Iwasaki | +| [YARN-9568](https://issues.apache.org/jira/browse/YARN-9568) | NPE in MiniYarnCluster during FileSystemNodeAttributeStore.recover | Minor | resourcemanager, test | Steve Loughran | Steve Loughran | +| [YARN-6046](https://issues.apache.org/jira/browse/YARN-6046) | Documentation correction in YarnApplicationSecurity | Trivial | . | Bibin Chundatt | Yousef Abu-Salah | +| [HADOOP-16440](https://issues.apache.org/jira/browse/HADOOP-16440) | Distcp can not preserve timestamp with -delete option | Major | . | ludun | ludun | +| [MAPREDUCE-7076](https://issues.apache.org/jira/browse/MAPREDUCE-7076) | TestNNBench#testNNBenchCreateReadAndDelete failing in our internal build | Minor | test | Rushabh Shah | Kevin Su | +| [YARN-9668](https://issues.apache.org/jira/browse/YARN-9668) | UGI conf doesn't read user overridden configurations on RM and NM startup | Major | . | Jonathan Hung | Jonathan Hung | +| [HADOOP-16451](https://issues.apache.org/jira/browse/HADOOP-16451) | Update jackson-databind to 2.9.9.1 | Major | . | Wei-Chiu Chuang | Siyao Meng | +| [HDFS-14647](https://issues.apache.org/jira/browse/HDFS-14647) | NPE during secure namenode startup | Major | hdfs | Fengnan Li | Fengnan Li | +| [HADOOP-9844](https://issues.apache.org/jira/browse/HADOOP-9844) | NPE when trying to create an error message response of SASL RPC | Major | ipc | Steve Loughran | Steve Loughran | +| [HADOOP-16461](https://issues.apache.org/jira/browse/HADOOP-16461) | Regression: FileSystem cache lock parses XML within the lock | Major | fs | Gopal Vijayaraghavan | Gopal Vijayaraghavan | +| [HDFS-14135](https://issues.apache.org/jira/browse/HDFS-14135) | TestWebHdfsTimeouts Fails intermittently in trunk | Major | . | Ayush Saxena | Ayush Saxena | +| [MAPREDUCE-6973](https://issues.apache.org/jira/browse/MAPREDUCE-6973) | Fix comments on creating \_SUCCESS file. | Trivial | documentation | Mehul Garnara (MG) | Mehul Garnara (MG) | +| [HADOOP-16245](https://issues.apache.org/jira/browse/HADOOP-16245) | Enabling SSL within LdapGroupsMapping can break system SSL configs | Major | common, security | Erik Krogen | Erik Krogen | +| [HDFS-14425](https://issues.apache.org/jira/browse/HDFS-14425) | Native build fails on macos due to jlong in hdfs.c | Major | . | hunshenshi | hunshenshi | +| [HDFS-14660](https://issues.apache.org/jira/browse/HDFS-14660) | [SBN Read] ObserverNameNode should throw StandbyException for requests not from ObserverProxyProvider | Major | . | Chao Sun | Chao Sun | +| [HDFS-14429](https://issues.apache.org/jira/browse/HDFS-14429) | Block remain in COMMITTED but not COMPLETE caused by Decommission | Major | . | Yicong Cai | Yicong Cai | +| [HADOOP-16435](https://issues.apache.org/jira/browse/HADOOP-16435) | RpcMetrics should not be retained forever | Critical | rpc-server | Zoltan Haindrich | Zoltan Haindrich | +| [HADOOP-15910](https://issues.apache.org/jira/browse/HADOOP-15910) | Javadoc for LdapAuthenticationHandler#ENABLE\_START\_TLS is wrong | Trivial | . | Ted Yu | Don Jeba | +| [HDFS-14677](https://issues.apache.org/jira/browse/HDFS-14677) | TestDataNodeHotSwapVolumes#testAddVolumesConcurrently fails intermittently in trunk | Major | . | Chen Zhang | Chen Zhang | +| [HADOOP-16460](https://issues.apache.org/jira/browse/HADOOP-16460) | ABFS: fix for Sever Name Indication (SNI) | Major | fs/azure | Thomas Marqardt | Sneha Vijayarajan | +| [HDFS-14569](https://issues.apache.org/jira/browse/HDFS-14569) | Result of crypto -listZones is not formatted properly | Major | . | hemanthboyina | hemanthboyina | +| [YARN-9596](https://issues.apache.org/jira/browse/YARN-9596) | QueueMetrics has incorrect metrics when labelled partitions are involved | Major | capacity scheduler | Muhammad Samir Khan | Muhammad Samir Khan | +| [HDFS-14681](https://issues.apache.org/jira/browse/HDFS-14681) | RBF: TestDisableRouterQuota failed because port 8888 was occupied | Minor | rbf | Wei-Chiu Chuang | Chao Sun | +| [HDFS-14661](https://issues.apache.org/jira/browse/HDFS-14661) | RBF: updateMountTableEntry shouldn't update mountTableEntry if targetPath not exist | Major | rbf | xuzq | xuzq | +| [MAPREDUCE-7225](https://issues.apache.org/jira/browse/MAPREDUCE-7225) | Fix broken current folder expansion during MR job start | Major | mrv2 | Adam Antal | Peter Bacsko | +| [HDFS-13529](https://issues.apache.org/jira/browse/HDFS-13529) | Fix default trash policy emptier trigger time correctly | Major | namenode | Xiaoqiao He | Xiaoqiao He | +| [HADOOP-12282](https://issues.apache.org/jira/browse/HADOOP-12282) | Connection thread's name should be updated after address changing is detected | Major | ipc | zhouyingchao | Lisheng Sun | +| [HADOOP-15410](https://issues.apache.org/jira/browse/HADOOP-15410) | Update scope of log4j in hadoop-auth to provided | Major | . | lqjack | lqjacklee | +| [HDFS-14686](https://issues.apache.org/jira/browse/HDFS-14686) | HttpFS: HttpFSFileSystem#getErasureCodingPolicy always returns null | Major | httpfs | Siyao Meng | Siyao Meng | +| [HADOOP-15681](https://issues.apache.org/jira/browse/HADOOP-15681) | AuthenticationFilter should generate valid date format for Set-Cookie header regardless of default Locale | Minor | security | Cao Manh Dat | Cao Manh Dat | +| [HDFS-13131](https://issues.apache.org/jira/browse/HDFS-13131) | Modifying testcase testEnableAndDisableErasureCodingPolicy | Minor | . | chencan | chencan | +| [HADOOP-15865](https://issues.apache.org/jira/browse/HADOOP-15865) | ConcurrentModificationException in Configuration.overlay() method | Major | . | Oleksandr Shevchenko | Oleksandr Shevchenko | +| [HDFS-14685](https://issues.apache.org/jira/browse/HDFS-14685) | DefaultAuditLogger doesn't print CallerContext | Major | hdfs | xuzq | xuzq | +| [HDFS-14462](https://issues.apache.org/jira/browse/HDFS-14462) | WebHDFS throws "Error writing request body to server" instead of DSQuotaExceededException | Major | webhdfs | Erik Krogen | Simbarashe Dzinamarira | +| [HDFS-12826](https://issues.apache.org/jira/browse/HDFS-12826) | Document Saying the RPC port, But it's required IPC port in HDFS Federation Document. | Minor | balancer & mover, documentation | Harshakiran Reddy | usharani | +| [HDFS-14669](https://issues.apache.org/jira/browse/HDFS-14669) | TestDirectoryScanner#testDirectoryScannerInFederatedCluster fails intermittently in trunk | Minor | datanode | qiang Liu | qiang Liu | +| [HADOOP-16487](https://issues.apache.org/jira/browse/HADOOP-16487) | Update jackson-databind to 2.9.9.2 | Critical | . | Siyao Meng | Siyao Meng | +| [HDFS-14679](https://issues.apache.org/jira/browse/HDFS-14679) | Failed to add erasure code policies with example template | Minor | ec | Yuan Zhou | Yuan Zhou | +| [YARN-9410](https://issues.apache.org/jira/browse/YARN-9410) | Typo in documentation: Using FPGA On YARN | Major | . | Szilard Nemeth | Kevin Su | +| [HDFS-14557](https://issues.apache.org/jira/browse/HDFS-14557) | JournalNode error: Can't scan a pre-transactional edit log | Major | ha | Wei-Chiu Chuang | Stephen O'Donnell | +| [HADOOP-16457](https://issues.apache.org/jira/browse/HADOOP-16457) | Hadoop does not work with Kerberos config in hdfs-site.xml for simple security | Minor | . | Eric Yang | Prabhu Joseph | +| [HDFS-14692](https://issues.apache.org/jira/browse/HDFS-14692) | Upload button should not encode complete url | Major | . | Lokesh Jain | Lokesh Jain | +| [HADOOP-15908](https://issues.apache.org/jira/browse/HADOOP-15908) | hadoop-build-tools jar is downloaded from remote repository instead of using from local | Minor | build | Oleksandr Shevchenko | Oleksandr Shevchenko | +| [HDFS-14631](https://issues.apache.org/jira/browse/HDFS-14631) | The DirectoryScanner doesn't fix the wrongly placed replica. | Major | . | Jinglun | Jinglun | +| [YARN-9601](https://issues.apache.org/jira/browse/YARN-9601) | Potential NPE in ZookeeperFederationStateStore#getPoliciesConfigurations | Major | federation, yarn | hunshenshi | hunshenshi | +| [YARN-9685](https://issues.apache.org/jira/browse/YARN-9685) | NPE when rendering the info table of leaf queue in non-accessible partitions | Major | capacityscheduler | Tao Yang | Tao Yang | +| [HDFS-14459](https://issues.apache.org/jira/browse/HDFS-14459) | ClosedChannelException silently ignored in FsVolumeList.addBlockPool() | Major | datanode | Stephen O'Donnell | Stephen O'Donnell | +| [YARN-9732](https://issues.apache.org/jira/browse/YARN-9732) | yarn.system-metrics-publisher.enabled=false is not honored by RM | Major | resourcemanager, timelineclient | KWON BYUNGCHANG | KWON BYUNGCHANG | +| [YARN-9527](https://issues.apache.org/jira/browse/YARN-9527) | Rogue LocalizerRunner/ContainerLocalizer repeatedly downloading same file | Major | yarn | Jim Brennan | Jim Brennan | +| [HDFS-14623](https://issues.apache.org/jira/browse/HDFS-14623) | In NameNode Web UI, for Head the file (first 32K) old data is showing | Major | . | hemanthboyina | hemanthboyina | +| [HDFS-12125](https://issues.apache.org/jira/browse/HDFS-12125) | Document the missing EC removePolicy command | Major | documentation, erasure-coding | Wenxin He | Siyao Meng | +| [HDFS-13359](https://issues.apache.org/jira/browse/HDFS-13359) | DataXceiver hung due to the lock in FsDatasetImpl#getBlockInputStream | Major | datanode | Yiqun Lin | Yiqun Lin | +| [YARN-9722](https://issues.apache.org/jira/browse/YARN-9722) | PlacementRule logs object ID in place of queue name. | Minor | resourcemanager | Prabhu Joseph | Prabhu Joseph | +| [YARN-9451](https://issues.apache.org/jira/browse/YARN-9451) | AggregatedLogsBlock shows wrong NM http port | Minor | nodemanager | Prabhu Joseph | Prabhu Joseph | +| [YARN-9723](https://issues.apache.org/jira/browse/YARN-9723) | ApplicationPlacementContext is not required for terminated jobs during recovery | Major | resourcemanager | Prabhu Joseph | Prabhu Joseph | +| [HDFS-12914](https://issues.apache.org/jira/browse/HDFS-12914) | Block report leases cause missing blocks until next report | Critical | namenode | Daryn Sharp | Santosh Marella | +| [YARN-9719](https://issues.apache.org/jira/browse/YARN-9719) | Failed to restart yarn-service if it doesn’t exist in RM | Major | yarn-native-services | kyungwan nam | kyungwan nam | +| [HDFS-14148](https://issues.apache.org/jira/browse/HDFS-14148) | HDFS OIV ReverseXML SnapshotSection parser throws exception when there are more than one snapshottable directory | Major | hdfs | Siyao Meng | Siyao Meng | +| [HDFS-14708](https://issues.apache.org/jira/browse/HDFS-14708) | TestLargeBlockReport#testBlockReportSucceedsWithLargerLengthLimit fails in trunk | Minor | . | Lisheng Sun | Lisheng Sun | +| [YARN-9744](https://issues.apache.org/jira/browse/YARN-9744) | RollingLevelDBTimelineStore.getEntityByTime fails with NPE | Major | timelineserver | Prabhu Joseph | Prabhu Joseph | +| [HADOOP-16495](https://issues.apache.org/jira/browse/HADOOP-16495) | Fix invalid metric types in PrometheusMetricsSink | Major | metrics | Akira Ajisaka | Akira Ajisaka | +| [YARN-9747](https://issues.apache.org/jira/browse/YARN-9747) | Reduce additional namenode call by EntityGroupFSTimelineStore#cleanLogs | Major | timelineserver | Prabhu Joseph | Prabhu Joseph | +| [HDFS-14595](https://issues.apache.org/jira/browse/HDFS-14595) | HDFS-11848 breaks API compatibility | Blocker | . | Wei-Chiu Chuang | Siyao Meng | +| [HDFS-14423](https://issues.apache.org/jira/browse/HDFS-14423) | Percent (%) and plus (+) characters no longer work in WebHDFS | Major | webhdfs | Jing Wang | Masatake Iwasaki | +| [HDFS-14719](https://issues.apache.org/jira/browse/HDFS-14719) | Correct the safemode threshold value in BlockManagerSafeMode | Major | namenode | Surendra Singh Lilhore | hemanthboyina | +| [MAPREDUCE-7230](https://issues.apache.org/jira/browse/MAPREDUCE-7230) | TestHSWebApp.testLogsViewSingle fails | Major | jobhistoryserver, test | Prabhu Joseph | Prabhu Joseph | +| [YARN-9749](https://issues.apache.org/jira/browse/YARN-9749) | TestAppLogAggregatorImpl#testDFSQuotaExceeded fails on trunk | Major | log-aggregation, test | Peter Bacsko | Adam Antal | +| [YARN-9461](https://issues.apache.org/jira/browse/YARN-9461) | TestRMWebServicesDelegationTokenAuthentication.testCancelledDelegationToken fails with HTTP 400 | Minor | resourcemanager, test | Peter Bacsko | Peter Bacsko | +| [HADOOP-16391](https://issues.apache.org/jira/browse/HADOOP-16391) | Duplicate values in rpcDetailedMetrics | Major | . | Bilwa S T | Bilwa S T | +| [HDFS-14456](https://issues.apache.org/jira/browse/HDFS-14456) | HAState#prepareToEnterState needn't a lock | Major | hdfs | hunshenshi | hunshenshi | +| [YARN-2599](https://issues.apache.org/jira/browse/YARN-2599) | Standby RM should expose jmx endpoint | Major | resourcemanager | Karthik Kambatla | Rohith Sharma K S | +| [HDFS-12012](https://issues.apache.org/jira/browse/HDFS-12012) | Fix spelling mistakes in BPServiceActor.java. | Major | datanode | chencan | chencan | +| [HDFS-14127](https://issues.apache.org/jira/browse/HDFS-14127) | Add a description about the observer read configuration | Minor | . | xiangheng | xiangheng | +| [HDFS-14687](https://issues.apache.org/jira/browse/HDFS-14687) | Standby Namenode never come out of safemode when EC files are being written. | Critical | ec, namenode | Surendra Singh Lilhore | Surendra Singh Lilhore | +| [HDFS-13101](https://issues.apache.org/jira/browse/HDFS-13101) | Yet another fsimage corruption related to snapshot | Critical | snapshots | Yongjun Zhang | Shashikant Banerjee | +| [YARN-9758](https://issues.apache.org/jira/browse/YARN-9758) | Upgrade JQuery to latest version for YARN UI | Major | yarn | Akhil PB | Akhil PB | +| [HDFS-13201](https://issues.apache.org/jira/browse/HDFS-13201) | Fix prompt message in testPolicyAndStateCantBeNull | Minor | . | chencan | chencan | +| [HDFS-14311](https://issues.apache.org/jira/browse/HDFS-14311) | Multi-threading conflict at layoutVersion when loading block pool storage | Major | rolling upgrades | Yicong Cai | Yicong Cai | +| [HDFS-14582](https://issues.apache.org/jira/browse/HDFS-14582) | Failed to start DN with ArithmeticException when NULL checksum used | Major | datanode | Surendra Singh Lilhore | Surendra Singh Lilhore | +| [HADOOP-16496](https://issues.apache.org/jira/browse/HADOOP-16496) | Apply HDDS-1870 (ConcurrentModification at PrometheusMetricsSink) to Hadoop common | Major | metrics | Akira Ajisaka | Akira Ajisaka | +| [YARN-5857](https://issues.apache.org/jira/browse/YARN-5857) | TestLogAggregationService.testFixedSizeThreadPool fails intermittently on trunk | Minor | . | Varun Saxena | Bilwa S T | +| [YARN-9217](https://issues.apache.org/jira/browse/YARN-9217) | Nodemanager will fail to start if GPU is misconfigured on the node or GPU drivers missing | Major | yarn | Antal Bálint Steinbach | Peter Bacsko | +| [HDFS-14759](https://issues.apache.org/jira/browse/HDFS-14759) | HDFS cat logs an info message | Major | . | Eric Badger | Eric Badger | +| [HADOOP-16494](https://issues.apache.org/jira/browse/HADOOP-16494) | Add SHA-256 or SHA-512 checksum to release artifacts to comply with the release distribution policy | Blocker | build | Akira Ajisaka | Akira Ajisaka | +| [HDFS-14741](https://issues.apache.org/jira/browse/HDFS-14741) | RBF: RecoverLease should be return false when the file is open in multiple destination | Major | rbf | xuzq | xuzq | +| [HDFS-14583](https://issues.apache.org/jira/browse/HDFS-14583) | FileStatus#toString() will throw IllegalArgumentException | Major | . | xuzq | xuzq | +| [YARN-9774](https://issues.apache.org/jira/browse/YARN-9774) | Fix order of arguments for assertEquals in TestSLSUtils | Minor | test | Nikhil Navadiya | Nikhil Navadiya | +| [HDFS-13596](https://issues.apache.org/jira/browse/HDFS-13596) | NN restart fails after RollingUpgrade from 2.x to 3.x | Blocker | hdfs | Hanisha Koneru | Fei Hui | +| [HDFS-14396](https://issues.apache.org/jira/browse/HDFS-14396) | Failed to load image from FSImageFile when downgrade from 3.x to 2.x | Blocker | rolling upgrades | Fei Hui | Fei Hui | +| [HDFS-14747](https://issues.apache.org/jira/browse/HDFS-14747) | RBF: IsFileClosed should be return false when the file is open in multiple destination | Major | rbf | xuzq | xuzq | +| [HDFS-14761](https://issues.apache.org/jira/browse/HDFS-14761) | RBF: MountTableResolver cannot invalidate cache correctly | Major | rbf | Yuxuan Wang | Yuxuan Wang | +| [HDFS-14722](https://issues.apache.org/jira/browse/HDFS-14722) | RBF: GetMountPointStatus should return mountTable information when getFileInfoAll throw IOException | Major | rbf | xuzq | xuzq | +| [YARN-8917](https://issues.apache.org/jira/browse/YARN-8917) | Absolute (maximum) capacity of level3+ queues is wrongly calculated for absolute resource | Critical | capacityscheduler | Tao Yang | Tao Yang | +| [YARN-9771](https://issues.apache.org/jira/browse/YARN-9771) | Add GPU in the container-executor.cfg example | Trivial | nodemanager, yarn | Adam Antal | Kinga Marton | +| [YARN-9642](https://issues.apache.org/jira/browse/YARN-9642) | Fix Memory Leak in AbstractYarnScheduler caused by timer | Blocker | resourcemanager | Bibin Chundatt | Bibin Chundatt | +| [HDFS-13977](https://issues.apache.org/jira/browse/HDFS-13977) | NameNode can kill itself if it tries to send too many txns to a QJM simultaneously | Major | namenode, qjm | Erik Krogen | Erik Krogen | +| [HDFS-2470](https://issues.apache.org/jira/browse/HDFS-2470) | NN should automatically set permissions on dfs.namenode.\*.dir | Major | namenode | Aaron Myers | Siddharth Wagle | +| [HADOOP-15958](https://issues.apache.org/jira/browse/HADOOP-15958) | Revisiting LICENSE and NOTICE files | Blocker | . | Akira Ajisaka | Akira Ajisaka | +| [HADOOP-16533](https://issues.apache.org/jira/browse/HADOOP-16533) | Update jackson-databind to 2.9.9.3 | Major | . | Akira Ajisaka | Akira Ajisaka | +| [YARN-9438](https://issues.apache.org/jira/browse/YARN-9438) | launchTime not written to state store for running applications | Major | . | Jonathan Hung | Jonathan Hung | +| [YARN-9640](https://issues.apache.org/jira/browse/YARN-9640) | Slow event processing could cause too many attempt unregister events | Critical | . | Bibin Chundatt | Bibin Chundatt | +| [HDFS-14721](https://issues.apache.org/jira/browse/HDFS-14721) | RBF: ProxyOpComplete is not accurate in FederationRPCPerformanceMonitor | Major | rbf | xuzq | xuzq | +| [HDFS-11246](https://issues.apache.org/jira/browse/HDFS-11246) | FSNameSystem#logAuditEvent should be called outside the read or write locks | Major | . | Kuhu Shukla | Xiaoqiao He | +| [HDFS-12212](https://issues.apache.org/jira/browse/HDFS-12212) | Options.Rename.To\_TRASH is considered even when Options.Rename.NONE is specified | Major | namenode | Vinayakumar B | Vinayakumar B | +| [YARN-9714](https://issues.apache.org/jira/browse/YARN-9714) | ZooKeeper connection in ZKRMStateStore leaks after RM transitioned to standby | Major | resourcemanager | Tao Yang | Tao Yang | +| [HDFS-14796](https://issues.apache.org/jira/browse/HDFS-14796) | Define LOG instead of BlockManager.LOG in ErasureCodingWork/ReplicationWork | Major | . | Fei Hui | Fei Hui | +| [YARN-9540](https://issues.apache.org/jira/browse/YARN-9540) | TestRMAppTransitions fails intermittently | Minor | resourcemanager, test | Prabhu Joseph | Tao Yang | +| [HDFS-8178](https://issues.apache.org/jira/browse/HDFS-8178) | QJM doesn't move aside stale inprogress edits files | Major | qjm | Zhe Zhang | Istvan Fajth | +| [YARN-9798](https://issues.apache.org/jira/browse/YARN-9798) | ApplicationMasterServiceTestBase#testRepeatedFinishApplicationMaster fails intermittently | Minor | test | Tao Yang | Tao Yang | +| [YARN-9800](https://issues.apache.org/jira/browse/YARN-9800) | TestRMDelegationTokens can fail in testRemoveExpiredMasterKeyInRMStateStore | Major | test, yarn | Adam Antal | Adam Antal | +| [YARN-9793](https://issues.apache.org/jira/browse/YARN-9793) | Remove duplicate sentence from TimelineServiceV2.md | Major | ATSv2, docs | Kinga Marton | Kinga Marton | +| [YARN-8174](https://issues.apache.org/jira/browse/YARN-8174) | Add containerId to ResourceLocalizationService fetch failure log statement | Minor | nodemanager | Prabhu Joseph | Prabhu Joseph | +| [HDFS-14706](https://issues.apache.org/jira/browse/HDFS-14706) | Checksums are not checked if block meta file is less than 7 bytes | Major | . | Stephen O'Donnell | Stephen O'Donnell | +| [HDFS-14630](https://issues.apache.org/jira/browse/HDFS-14630) | Configuration.getTimeDurationHelper() should not log time unit warning in info log. | Minor | hdfs | Surendra Singh Lilhore | hemanthboyina | +| [YARN-9797](https://issues.apache.org/jira/browse/YARN-9797) | LeafQueue#activateApplications should use resourceCalculator#fitsIn | Blocker | . | Bibin Chundatt | Bilwa S T | +| [HADOOP-16534](https://issues.apache.org/jira/browse/HADOOP-16534) | Exclude submarine from hadoop source build | Major | . | Nanda kumar | Nanda kumar | +| [HDFS-14807](https://issues.apache.org/jira/browse/HDFS-14807) | SetTimes updates all negative values apart from -1 | Major | . | Harshakiran Reddy | Ayush Saxena | +| [YARN-9785](https://issues.apache.org/jira/browse/YARN-9785) | Fix DominantResourceCalculator when one resource is zero | Blocker | . | Bilwa S T | Bilwa S T | +| [HDFS-14777](https://issues.apache.org/jira/browse/HDFS-14777) | RBF: Set ReadOnly is failing for mount Table but actually readonly succeed to set | Major | . | Ranith Sardar | Ranith Sardar | +| [YARN-9718](https://issues.apache.org/jira/browse/YARN-9718) | Yarn REST API, services endpoint remote command ejection | Major | . | Eric Yang | Eric Yang | +| [HDFS-14826](https://issues.apache.org/jira/browse/HDFS-14826) | dfs.ha.zkfc.port property duplicated in hdfs-default.xml | Major | hdfs | Renukaprasad C | Renukaprasad C | +| [YARN-9817](https://issues.apache.org/jira/browse/YARN-9817) | Fix failing testcases due to not initialized AsyncDispatcher - ArithmeticException: / by zero | Major | test | Prabhu Joseph | Prabhu Joseph | +| [YARN-9813](https://issues.apache.org/jira/browse/YARN-9813) | RM does not start on JDK11 when UIv2 is enabled | Critical | resourcemanager, yarn | Adam Antal | Adam Antal | +| [YARN-9812](https://issues.apache.org/jira/browse/YARN-9812) | mvn javadoc:javadoc fails in hadoop-sls | Major | documentation | Akira Ajisaka | Abhishek Modi | +| [YARN-9784](https://issues.apache.org/jira/browse/YARN-9784) | org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.TestLeafQueue is flaky | Major | test | Kinga Marton | Kinga Marton | +| [YARN-9820](https://issues.apache.org/jira/browse/YARN-9820) | RM logs InvalidStateTransitionException when app is submitted | Critical | . | Rohith Sharma K S | Prabhu Joseph | +| [HADOOP-16554](https://issues.apache.org/jira/browse/HADOOP-16554) | mvn javadoc:javadoc fails in hadoop-aws | Major | documentation | Akira Ajisaka | Xieming Li | +| [YARN-9728](https://issues.apache.org/jira/browse/YARN-9728) |  ResourceManager REST API can produce an illegal xml response | Major | api, resourcemanager | Thomas | Prabhu Joseph | +| [HDFS-14835](https://issues.apache.org/jira/browse/HDFS-14835) | RBF: Secured Router should not run when it can't initialize DelegationTokenSecretManager | Major | . | Takanobu Asanuma | Takanobu Asanuma | +| [HDFS-14838](https://issues.apache.org/jira/browse/HDFS-14838) | RBF: Display RPC (instead of HTTP) Port Number in RBF web UI | Minor | rbf, ui | Xieming Li | Xieming Li | +| [YARN-9816](https://issues.apache.org/jira/browse/YARN-9816) | EntityGroupFSTimelineStore#scanActiveLogs fails when undesired files are present under /ats/active. | Major | timelineserver | Prabhu Joseph | Prabhu Joseph | +| [HDFS-14699](https://issues.apache.org/jira/browse/HDFS-14699) | Erasure Coding: Storage not considered in live replica when replication streams hard limit reached to threshold | Critical | ec | Zhao Yi Ming | Zhao Yi Ming | +| [HDFS-14798](https://issues.apache.org/jira/browse/HDFS-14798) | Synchronize invalidateBlocks in DatanodeDescriptor | Minor | namenode | David Mollitor | hemanthboyina | +| [HDFS-14821](https://issues.apache.org/jira/browse/HDFS-14821) | Make HDFS-14617 (fsimage sub-sections) off by default | Blocker | namenode | Stephen O'Donnell | Stephen O'Donnell | +| [HDFS-14303](https://issues.apache.org/jira/browse/HDFS-14303) | check block directory logic not correct when there is only meta file, print no meaning warn log | Minor | datanode, hdfs | qiang Liu | qiang Liu | +| [YARN-9833](https://issues.apache.org/jira/browse/YARN-9833) | Race condition when DirectoryCollection.checkDirs() runs during container launch | Major | . | Peter Bacsko | Peter Bacsko | +| [YARN-9837](https://issues.apache.org/jira/browse/YARN-9837) | YARN Service fails to fetch status for Stopped apps with bigger spec files | Major | yarn-native-services | Tarun Parimi | Tarun Parimi | +| [YARN-2255](https://issues.apache.org/jira/browse/YARN-2255) | YARN Audit logging not added to log4j.properties | Major | . | Varun Saxena | Aihua Xu | +| [YARN-9814](https://issues.apache.org/jira/browse/YARN-9814) | JobHistoryServer can't delete aggregated files, if remote app root directory is created by NodeManager | Minor | log-aggregation, yarn | Adam Antal | Adam Antal | +| [HDFS-14836](https://issues.apache.org/jira/browse/HDFS-14836) | FileIoProvider should not increase FileIoErrors metric in datanode volume metric | Minor | . | Aiphago | Aiphago | +| [HDFS-14846](https://issues.apache.org/jira/browse/HDFS-14846) | libhdfs tests are failing on trunk due to jni usage bugs | Major | libhdfs, native | Sahil Takiar | Sahil Takiar | +| [HADOOP-16582](https://issues.apache.org/jira/browse/HADOOP-16582) | LocalFileSystem's mkdirs() does not work as expected under viewfs. | Major | . | Kihwal Lee | Kihwal Lee | +| [HDFS-14609](https://issues.apache.org/jira/browse/HDFS-14609) | RBF: Security should use common AuthenticationFilter | Major | . | CR Hota | Chen Zhang | +| [HADOOP-16581](https://issues.apache.org/jira/browse/HADOOP-16581) | ValueQueue does not trigger an async refill when number of values falls below watermark | Major | common, kms | Yuval Degani | Yuval Degani | +| [HDFS-14853](https://issues.apache.org/jira/browse/HDFS-14853) | NPE in DFSNetworkTopology#chooseRandomWithStorageType() when the excludedNode is not present | Major | . | Ranith Sardar | Ranith Sardar | +| [HDFS-13660](https://issues.apache.org/jira/browse/HDFS-13660) | DistCp job fails when new data is appended in the file while the distCp copy job is running | Critical | distcp | Mukund Thakur | Mukund Thakur | +| [HDFS-14868](https://issues.apache.org/jira/browse/HDFS-14868) | RBF: Fix typo in TestRouterQuota | Trivial | . | Jinglun | Jinglun | +| [HDFS-14808](https://issues.apache.org/jira/browse/HDFS-14808) | EC: Improper size values for corrupt ec block in LOG | Major | ec | Harshakiran Reddy | Ayush Saxena | +| [HDFS-14845](https://issues.apache.org/jira/browse/HDFS-14845) | Ignore AuthenticationFilterInitializer for HttpFSServerWebServer and honor hadoop.http.authentication configs | Critical | httpfs | Akira Ajisaka | Prabhu Joseph | +| [HADOOP-16602](https://issues.apache.org/jira/browse/HADOOP-16602) | mvn package fails in hadoop-aws | Major | documentation | Xieming Li | Xieming Li | +| [HDFS-14874](https://issues.apache.org/jira/browse/HDFS-14874) | Fix TestHDFSCLI and TestDFSShell test break because of logging change in mkdir | Major | hdfs | Gabor Bota | Gabor Bota | +| [HDFS-14873](https://issues.apache.org/jira/browse/HDFS-14873) | Fix dfsadmin doc for triggerBlockReport | Major | documentation | Fei Hui | Fei Hui | +| [HDFS-14849](https://issues.apache.org/jira/browse/HDFS-14849) | Erasure Coding: the internal block is replicated many times when datanode is decommissioning | Major | ec, erasure-coding | HuangTao | HuangTao | +| [HDFS-14876](https://issues.apache.org/jira/browse/HDFS-14876) | Remove unused imports from TestBlockMissingException.java and TestClose.java | Minor | test | Lisheng Sun | Lisheng Sun | +| [YARN-9858](https://issues.apache.org/jira/browse/YARN-9858) | Optimize RMContext getExclusiveEnforcedPartitions | Major | . | Jonathan Hung | Jonathan Hung | +| [HDFS-14492](https://issues.apache.org/jira/browse/HDFS-14492) | Snapshot memory leak | Major | snapshots | Wei-Chiu Chuang | Wei-Chiu Chuang | +| [HDFS-14418](https://issues.apache.org/jira/browse/HDFS-14418) | Remove redundant super user priveledge checks from namenode. | Major | . | Ayush Saxena | Ayush Saxena | +| [YARN-9862](https://issues.apache.org/jira/browse/YARN-9862) | yarn-services-core test timeout | Major | . | Prabhu Joseph | Prabhu Joseph | +| [HDFS-14885](https://issues.apache.org/jira/browse/HDFS-14885) | UI: Fix a typo on WebUI of DataNode. | Trivial | datanode, ui | Xieming Li | Xieming Li | +| [HADOOP-16619](https://issues.apache.org/jira/browse/HADOOP-16619) | Upgrade jackson and jackson-databind to 2.9.10 | Major | . | Siyao Meng | Siyao Meng | +| [HDFS-14216](https://issues.apache.org/jira/browse/HDFS-14216) | NullPointerException happens in NamenodeWebHdfs | Critical | . | lujie | lujie | +| [HADOOP-16605](https://issues.apache.org/jira/browse/HADOOP-16605) | NPE in TestAdlSdkConfiguration failing in yetus | Major | fs/adl | Steve Loughran | Sneha Vijayarajan | +| [HDFS-14881](https://issues.apache.org/jira/browse/HDFS-14881) | Safemode 'forceExit' option, doesn’t shown in help message | Major | . | Renukaprasad C | Renukaprasad C | +| [HDFS-14637](https://issues.apache.org/jira/browse/HDFS-14637) | Namenode may not replicate blocks to meet the policy after enabling upgradeDomain | Major | namenode | Stephen O'Donnell | Stephen O'Donnell | +| [HDFS-14879](https://issues.apache.org/jira/browse/HDFS-14879) | Header was wrong in Snapshot web UI | Major | . | hemanthboyina | hemanthboyina | +| [HDFS-14655](https://issues.apache.org/jira/browse/HDFS-14655) | [SBN Read] Namenode crashes if one of The JN is down | Critical | . | Harshakiran Reddy | Ayush Saxena | +| [HDFS-14859](https://issues.apache.org/jira/browse/HDFS-14859) | Prevent unnecessary evaluation of costly operation getNumLiveDataNodes when dfs.namenode.safemode.min.datanodes is not zero | Major | hdfs | Srinivasu Majeti | Srinivasu Majeti | +| [YARN-6715](https://issues.apache.org/jira/browse/YARN-6715) | Fix documentation about NodeHealthScriptRunner | Major | documentation, nodemanager | Peter Bacsko | Peter Bacsko | +| [YARN-9552](https://issues.apache.org/jira/browse/YARN-9552) | FairScheduler: NODE\_UPDATE can cause NoSuchElementException | Major | fairscheduler | Peter Bacsko | Peter Bacsko | +| [HDFS-14754](https://issues.apache.org/jira/browse/HDFS-14754) | Erasure Coding : The number of Under-Replicated Blocks never reduced | Critical | ec | hemanthboyina | hemanthboyina | +| [HDFS-14900](https://issues.apache.org/jira/browse/HDFS-14900) | Fix build failure of hadoop-hdfs-native-client | Major | . | Masatake Iwasaki | Masatake Iwasaki | +| [HDFS-14245](https://issues.apache.org/jira/browse/HDFS-14245) | Class cast error in GetGroups with ObserverReadProxyProvider | Major | . | Shen Yinjie | Erik Krogen | +| [HDFS-14373](https://issues.apache.org/jira/browse/HDFS-14373) | EC : Decoding is failing when block group last incomplete cell fall in to AlignedStripe | Critical | ec, hdfs-client | Surendra Singh Lilhore | Surendra Singh Lilhore | +| [HDFS-14509](https://issues.apache.org/jira/browse/HDFS-14509) | DN throws InvalidToken due to inequality of password when upgrade NN 2.x to 3.x | Blocker | . | Yuxuan Wang | Yuxuan Wang | +| [HADOOP-13907](https://issues.apache.org/jira/browse/HADOOP-13907) | Fix TestWebDelegationToken#testKerberosDelegationTokenAuthenticator on Windows | Major | security | Xiaoyu Yao | Kitti Nanasi | +| [HDFS-14886](https://issues.apache.org/jira/browse/HDFS-14886) | In NameNode Web UI's Startup Progress page, Loading edits always shows 0 sec | Major | . | hemanthboyina | hemanthboyina | +| [YARN-8453](https://issues.apache.org/jira/browse/YARN-8453) | Additional Unit tests to verify queue limit and max-limit with multiple resource types | Major | capacity scheduler | Sunil G | Adam Antal | +| [HDFS-14890](https://issues.apache.org/jira/browse/HDFS-14890) | Setting permissions on name directory fails on non posix compliant filesystems | Blocker | . | hirik | Siddharth Wagle | +| [HADOOP-15169](https://issues.apache.org/jira/browse/HADOOP-15169) | "hadoop.ssl.enabled.protocols" should be considered in httpserver2 | Major | security | Brahma Reddy Battula | Brahma Reddy Battula | +| [HADOOP-16580](https://issues.apache.org/jira/browse/HADOOP-16580) | Disable retry of FailoverOnNetworkExceptionRetry in case of AccessControlException | Major | common | Adam Antal | Adam Antal | +| [HDFS-14739](https://issues.apache.org/jira/browse/HDFS-14739) | RBF: LS command for mount point shows wrong owner and permission information. | Major | . | xuzq | Jinglun | +| [HDFS-14909](https://issues.apache.org/jira/browse/HDFS-14909) | DFSNetworkTopology#chooseRandomWithStorageType() should not decrease storage count for excluded node which is already part of excluded scope | Major | namenode | Surendra Singh Lilhore | Surendra Singh Lilhore | +| [HADOOP-16662](https://issues.apache.org/jira/browse/HADOOP-16662) | Remove unnecessary InnerNode check in NetworkTopology#add() | Minor | . | Lisheng Sun | Lisheng Sun | +| [HDFS-14887](https://issues.apache.org/jira/browse/HDFS-14887) | RBF: In Router Web UI, Observer Namenode Information displaying as Unavailable | Major | . | hemanthboyina | hemanthboyina | +| [HDFS-14847](https://issues.apache.org/jira/browse/HDFS-14847) | Erasure Coding: Blocks are over-replicated while EC decommissioning | Critical | ec | Fei Hui | Fei Hui | +| [HDFS-14916](https://issues.apache.org/jira/browse/HDFS-14916) | RBF: line breaks are missing from the output of 'hdfs dfsrouteradmin -ls' | Minor | rbf, ui | Xieming Li | Xieming Li | +| [HDFS-14913](https://issues.apache.org/jira/browse/HDFS-14913) | Correct the value of available count in DFSNetworkTopology#chooseRandomWithStorageType() | Major | . | Ayush Saxena | Ayush Saxena | +| [YARN-9915](https://issues.apache.org/jira/browse/YARN-9915) | Fix FindBug issue in QueueMetrics | Minor | . | Prabhu Joseph | Prabhu Joseph | +| [HDFS-12749](https://issues.apache.org/jira/browse/HDFS-12749) | DN may not send block report to NN after NN restart | Major | datanode | TanYuxin | Xiaoqiao He | +| [HDFS-13901](https://issues.apache.org/jira/browse/HDFS-13901) | INode access time is ignored because of race between open and rename | Major | . | Jinglun | Jinglun | +| [HADOOP-16658](https://issues.apache.org/jira/browse/HADOOP-16658) | S3A connector does not support including the token renewer in the token identifier | Major | fs/s3 | Philip Zampino | Philip Zampino | +| [YARN-9921](https://issues.apache.org/jira/browse/YARN-9921) | Issue in PlacementConstraint when YARN Service AM retries allocation on component failure. | Major | . | Tarun Parimi | Tarun Parimi | +| [HADOOP-16614](https://issues.apache.org/jira/browse/HADOOP-16614) | Missing leveldbjni package of aarch64 platform | Major | . | liusheng | | +| [HDFS-14910](https://issues.apache.org/jira/browse/HDFS-14910) | Rename Snapshot with Pre Descendants Fail With IllegalArgumentException. | Blocker | . | Íñigo Goiri | Wei-Chiu Chuang | +| [HDFS-14933](https://issues.apache.org/jira/browse/HDFS-14933) | Fixing a typo in documentation of Observer NameNode | Trivial | documentation | Xieming Li | Xieming Li | +| [HDFS-14308](https://issues.apache.org/jira/browse/HDFS-14308) | DFSStripedInputStream curStripeBuf is not freed by unbuffer() | Major | ec | Joe McDonnell | Zhao Yi Ming | +| [HDFS-14931](https://issues.apache.org/jira/browse/HDFS-14931) | hdfs crypto commands limit column width | Major | . | Eric Badger | Eric Badger | +| [HDFS-14730](https://issues.apache.org/jira/browse/HDFS-14730) | Remove unused configuration dfs.web.authentication.filter | Major | . | Chen Zhang | Chen Zhang | +| [HDFS-14920](https://issues.apache.org/jira/browse/HDFS-14920) | Erasure Coding: Decommission may hang If one or more datanodes are out of service during decommission | Major | ec | Fei Hui | Fei Hui | +| [HDFS-14768](https://issues.apache.org/jira/browse/HDFS-14768) | EC : Busy DN replica should be consider in live replica check. | Major | datanode, erasure-coding, hdfs, namenode | guojh | guojh | +| [HDFS-13736](https://issues.apache.org/jira/browse/HDFS-13736) | BlockPlacementPolicyDefault can not choose favored nodes when 'dfs.namenode.block-placement-policy.default.prefer-local-node' set to false | Major | . | hu xiaodong | hu xiaodong | +| [HDFS-14925](https://issues.apache.org/jira/browse/HDFS-14925) | rename operation should check nest snapshot | Major | namenode | Junwang Zhao | Junwang Zhao | +| [YARN-9949](https://issues.apache.org/jira/browse/YARN-9949) | Add missing queue configs for root queue in RMWebService#CapacitySchedulerInfo | Minor | capacity scheduler | Prabhu Joseph | Prabhu Joseph | +| [HDFS-14945](https://issues.apache.org/jira/browse/HDFS-14945) | Revise PacketResponder's log. | Minor | datanode | Xudong Cao | Xudong Cao | +| [HDFS-14946](https://issues.apache.org/jira/browse/HDFS-14946) | Erasure Coding: Block recovery failed during decommissioning | Major | . | Fei Hui | Fei Hui | +| [HDFS-14938](https://issues.apache.org/jira/browse/HDFS-14938) | Add check if excludedNodes contain scope in DFSNetworkTopology#chooseRandomWithStorageType() | Major | . | Lisheng Sun | Lisheng Sun | +| [HADOOP-16681](https://issues.apache.org/jira/browse/HADOOP-16681) | mvn javadoc:javadoc fails in hadoop-aws | Major | documentation | Xieming Li | Xieming Li | +| [HDFS-14384](https://issues.apache.org/jira/browse/HDFS-14384) | When lastLocatedBlock token expire, it will take 1~3s second to refetch it. | Major | hdfs-client | Surendra Singh Lilhore | Surendra Singh Lilhore | +| [HDFS-14880](https://issues.apache.org/jira/browse/HDFS-14880) | Correct the sequence of statistics & exit message in balencer | Major | balancer & mover | Renukaprasad C | Renukaprasad C | +| [HDFS-14806](https://issues.apache.org/jira/browse/HDFS-14806) | Bootstrap standby may fail if used in-progress tailing | Major | namenode | Chen Liang | Chen Liang | +| [HDFS-14941](https://issues.apache.org/jira/browse/HDFS-14941) | Potential editlog race condition can cause corrupted file | Major | namenode | Chen Liang | Chen Liang | +| [HDFS-14958](https://issues.apache.org/jira/browse/HDFS-14958) | TestBalancerWithNodeGroup is not using NetworkTopologyWithNodeGroup | Minor | hdfs | Jim Brennan | Jim Brennan | +| [HDFS-14720](https://issues.apache.org/jira/browse/HDFS-14720) | DataNode shouldn't report block as bad block if the block length is Long.MAX\_VALUE. | Major | datanode | Surendra Singh Lilhore | hemanthboyina | +| [HDFS-14962](https://issues.apache.org/jira/browse/HDFS-14962) | RBF: ConnectionPool#newConnection() error log wrong protocol class | Minor | rbf | Yuxuan Wang | Yuxuan Wang | +| [HADOOP-16701](https://issues.apache.org/jira/browse/HADOOP-16701) | Fix broken links in site index | Minor | documentation | Masatake Iwasaki | Masatake Iwasaki | +| [HADOOP-16702](https://issues.apache.org/jira/browse/HADOOP-16702) | Move documentation of hadoop-cos to under src directory. | Major | tools | Masatake Iwasaki | Masatake Iwasaki | +| [HADOOP-16665](https://issues.apache.org/jira/browse/HADOOP-16665) | Filesystems to be closed if they failed during initialize() | Major | fs, fs/s3 | Steve Loughran | Steve Loughran | +| [HDFS-14922](https://issues.apache.org/jira/browse/HDFS-14922) | Prevent snapshot modification time got change on startup | Major | . | hemanthboyina | hemanthboyina | +| [HADOOP-16677](https://issues.apache.org/jira/browse/HADOOP-16677) | Recalculate the remaining timeout millis correctly while throwing an InterupptedException in SocketIOWithTimeout. | Minor | common | Xudong Cao | Xudong Cao | +| [HADOOP-16585](https://issues.apache.org/jira/browse/HADOOP-16585) | [Tool:NNloadGeneratorMR] Multiple threads are using same id for creating file LoadGenerator#write | Major | . | Ranith Sardar | Ranith Sardar | +| [HDFS-14884](https://issues.apache.org/jira/browse/HDFS-14884) | Add sanity check that zone key equals feinfo key while setting Xattrs | Major | encryption, hdfs | Mukul Kumar Singh | Mukul Kumar Singh | +| [HADOOP-15097](https://issues.apache.org/jira/browse/HADOOP-15097) | AbstractContractDeleteTest::testDeleteNonEmptyDirRecursive with misleading path | Minor | fs, test | zhoutai.zt | Xieming Li | +| [HDFS-14802](https://issues.apache.org/jira/browse/HDFS-14802) | The feature of protect directories should be used in RenameOp | Major | hdfs | Fei Hui | Fei Hui | +| [YARN-9982](https://issues.apache.org/jira/browse/YARN-9982) | Fix Container API example link in NodeManager REST API doc | Trivial | documentation | Charan Hebri | Charan Hebri | +| [HDFS-14967](https://issues.apache.org/jira/browse/HDFS-14967) | TestWebHDFS fails in Windows | Major | . | Renukaprasad C | Renukaprasad C | +| [YARN-9965](https://issues.apache.org/jira/browse/YARN-9965) | Fix NodeManager failing to start on subsequent times when Hdfs Auxillary Jar is set | Major | auxservices, nodemanager | Prabhu Joseph | Prabhu Joseph | +| [YARN-9984](https://issues.apache.org/jira/browse/YARN-9984) | FSPreemptionThread can cause NullPointerException while app is unregistered with containers running on a node | Major | fairscheduler | Wilfred Spiegelenburg | Wilfred Spiegelenburg | +| [YARN-9986](https://issues.apache.org/jira/browse/YARN-9986) | signalToContainer REST API does not work even if requested by the app owner | Major | restapi | kyungwan nam | kyungwan nam | +| [HDFS-14992](https://issues.apache.org/jira/browse/HDFS-14992) | TestOfflineEditsViewer is failing in Trunk | Major | . | hemanthboyina | hemanthboyina | +| [YARN-9983](https://issues.apache.org/jira/browse/YARN-9983) | Typo in YARN Service overview documentation | Trivial | documentation | Denes Gerencser | Denes Gerencser | +| [HADOOP-16719](https://issues.apache.org/jira/browse/HADOOP-16719) | Remove the disallowed element config within maven-checkstyle-plugin | Major | . | Wanqiang Ji | Wanqiang Ji | +| [HADOOP-16700](https://issues.apache.org/jira/browse/HADOOP-16700) | RpcQueueTime may be negative when the response has to be sent later | Minor | . | xuzq | xuzq | +| [HADOOP-15686](https://issues.apache.org/jira/browse/HADOOP-15686) | Supress bogus AbstractWadlGeneratorGrammarGenerator in KMS stderr | Major | kms | Wei-Chiu Chuang | Wei-Chiu Chuang | +| [HADOOP-16276](https://issues.apache.org/jira/browse/HADOOP-16276) | Fix jsvc startup command in hadoop-functions.sh due to jsvc \>= 1.0.11 changed default current working directory | Major | scripts | Siyao Meng | Siyao Meng | +| [HDFS-14996](https://issues.apache.org/jira/browse/HDFS-14996) | RBF: GetFileStatus fails for directory with EC policy set in case of multiple destinations | Major | ec, rbf | Ayush Saxena | Ayush Saxena | +| [HDFS-14940](https://issues.apache.org/jira/browse/HDFS-14940) | HDFS Balancer : Do not allow to set balancer maximum network bandwidth more than 1TB | Minor | balancer & mover | Souryakanta Dwivedy | hemanthboyina | +| [HDFS-14924](https://issues.apache.org/jira/browse/HDFS-14924) | RenameSnapshot not updating new modification time | Major | . | hemanthboyina | hemanthboyina | +| [HDFS-13842](https://issues.apache.org/jira/browse/HDFS-13842) | RBF: Exceptions are conflicting when creating the same mount entry twice | Major | federation | Soumyapn | Ranith Sardar | +| [YARN-9838](https://issues.apache.org/jira/browse/YARN-9838) | Fix resource inconsistency for queues when moving app with reserved container to another queue | Critical | capacity scheduler | jiulongzhu | jiulongzhu | +| [YARN-9968](https://issues.apache.org/jira/browse/YARN-9968) | Public Localizer is exiting in NodeManager due to NullPointerException | Major | nodemanager | Tarun Parimi | Tarun Parimi | +| [YARN-9011](https://issues.apache.org/jira/browse/YARN-9011) | Race condition during decommissioning | Major | nodemanager | Peter Bacsko | Peter Bacsko | +| [HDFS-14973](https://issues.apache.org/jira/browse/HDFS-14973) | Balancer getBlocks RPC dispersal does not function properly | Major | balancer & mover | Erik Krogen | Erik Krogen | +| [HADOOP-16685](https://issues.apache.org/jira/browse/HADOOP-16685) | FileSystem#listStatusIterator does not check if given path exists | Major | fs | Sahil Takiar | Sahil Takiar | +| [YARN-9290](https://issues.apache.org/jira/browse/YARN-9290) | Invalid SchedulingRequest not rejected in Scheduler PlacementConstraintsHandler | Major | . | Prabhu Joseph | Prabhu Joseph | +| [MAPREDUCE-7240](https://issues.apache.org/jira/browse/MAPREDUCE-7240) | Exception ' Invalid event: TA\_TOO\_MANY\_FETCH\_FAILURE at SUCCESS\_FINISHING\_CONTAINER' cause job error | Critical | . | luhuachao | luhuachao | +| [HDFS-14986](https://issues.apache.org/jira/browse/HDFS-14986) | ReplicaCachingGetSpaceUsed throws ConcurrentModificationException | Major | datanode, performance | Ryan Wu | Aiphago | +| [MAPREDUCE-7249](https://issues.apache.org/jira/browse/MAPREDUCE-7249) | Invalid event TA\_TOO\_MANY\_FETCH\_FAILURE at SUCCESS\_CONTAINER\_CLEANUP causes job failure | Critical | applicationmaster, mrv2 | Wilfred Spiegelenburg | Wilfred Spiegelenburg | +| [HDFS-14961](https://issues.apache.org/jira/browse/HDFS-14961) | [SBN read] Prevent ZKFC changing Observer Namenode state | Major | . | Íñigo Goiri | Ayush Saxena | +| [HDFS-15010](https://issues.apache.org/jira/browse/HDFS-15010) | BlockPoolSlice#addReplicaThreadPool static pool should be initialized by static method | Major | datanode | Surendra Singh Lilhore | Surendra Singh Lilhore | +| [YARN-9990](https://issues.apache.org/jira/browse/YARN-9990) | Testcase fails with "Insufficient configured threads: required=16 \< max=10" | Major | . | Prabhu Joseph | Prabhu Joseph | +| [HDFS-15009](https://issues.apache.org/jira/browse/HDFS-15009) | FSCK "-list-corruptfileblocks" return Invalid Entries | Major | . | hemanthboyina | hemanthboyina | +| [HDFS-9695](https://issues.apache.org/jira/browse/HDFS-9695) | HTTPFS - CHECKACCESS operation missing | Major | . | Bert Hekman | hemanthboyina | +| [HDFS-15026](https://issues.apache.org/jira/browse/HDFS-15026) | TestPendingReconstruction#testPendingReconstruction() fail in trunk | Minor | . | Lisheng Sun | Lisheng Sun | +| [HADOOP-16441](https://issues.apache.org/jira/browse/HADOOP-16441) | if use -Dbundle.openssl=true, bundled with unnecessary libk5crypto.\* | Major | build | KWON BYUNGCHANG | KWON BYUNGCHANG | +| [YARN-9969](https://issues.apache.org/jira/browse/YARN-9969) | Improve yarn.scheduler.capacity.queue-mappings documentation | Major | . | Manikandan R | Manikandan R | +| [YARN-9938](https://issues.apache.org/jira/browse/YARN-9938) | Validate Parent Queue for QueueMapping contains dynamic group as parent queue | Major | . | Manikandan R | Manikandan R | +| [HADOOP-16744](https://issues.apache.org/jira/browse/HADOOP-16744) | Fix building instruction to enable zstd | Minor | documentation | Masatake Iwasaki | Masatake Iwasaki | +| [YARN-10006](https://issues.apache.org/jira/browse/YARN-10006) | IOException used in place of YARNException in CapacitySheduler | Minor | capacity scheduler | Prabhu Joseph | Adam Antal | +| [HDFS-14751](https://issues.apache.org/jira/browse/HDFS-14751) | Synchronize on diffs in DirectoryScanner | Minor | . | Lisheng Sun | Lisheng Sun | +| [YARN-9985](https://issues.apache.org/jira/browse/YARN-9985) | Unsupported "transitionToObserver" option displaying for rmadmin command | Minor | RM, yarn | Souryakanta Dwivedy | Ayush Saxena | +| [HDFS-15040](https://issues.apache.org/jira/browse/HDFS-15040) | RBF: Secured Router should not run when SecretManager is not running | Major | . | Takanobu Asanuma | Takanobu Asanuma | +| [YARN-7721](https://issues.apache.org/jira/browse/YARN-7721) | TestContinuousScheduling fails sporadically with NPE | Major | fairscheduler | Jason Darrell Lowe | Wilfred Spiegelenburg | +| [HDFS-15045](https://issues.apache.org/jira/browse/HDFS-15045) | DataStreamer#createBlockOutputStream() should log exception in warn. | Major | dfsclient | Surendra Singh Lilhore | Ravuri Sushma sree | +| [HADOOP-16754](https://issues.apache.org/jira/browse/HADOOP-16754) | Fix docker failed to build yetus/hadoop | Blocker | build | Kevin Su | Kevin Su | +| [HDFS-15032](https://issues.apache.org/jira/browse/HDFS-15032) | Balancer crashes when it fails to contact an unavailable NN via ObserverReadProxyProvider | Major | balancer & mover | Erik Krogen | Erik Krogen | +| [HDFS-15036](https://issues.apache.org/jira/browse/HDFS-15036) | Active NameNode should not silently fail the image transfer | Major | namenode | Konstantin Shvachko | Chen Liang | +| [HADOOP-16755](https://issues.apache.org/jira/browse/HADOOP-16755) | Fix broken link in single node cluster setup documentation | Trivial | documentation | Denes Gerencser | Denes Gerencser | +| [HDFS-15048](https://issues.apache.org/jira/browse/HDFS-15048) | Fix findbug in DirectoryScanner | Major | . | Takanobu Asanuma | Masatake Iwasaki | +| [HADOOP-16765](https://issues.apache.org/jira/browse/HADOOP-16765) | Fix curator dependencies for gradle projects using hadoop-minicluster | Major | . | Mate Szalay-Beko | Mate Szalay-Beko | +| [HDFS-14908](https://issues.apache.org/jira/browse/HDFS-14908) | LeaseManager should check parent-child relationship when filter open files. | Minor | . | Jinglun | Jinglun | +| [HDFS-14519](https://issues.apache.org/jira/browse/HDFS-14519) | NameQuota is not update after concat operation, so namequota is wrong | Major | . | Ranith Sardar | Ranith Sardar | +| [YARN-10020](https://issues.apache.org/jira/browse/YARN-10020) | Fix build instruction of hadoop-yarn-ui | Minor | yarn-ui-v2 | Masatake Iwasaki | Masatake Iwasaki | +| [HDFS-15012](https://issues.apache.org/jira/browse/HDFS-15012) | NN fails to parse Edit logs after applying HDFS-13101 | Blocker | nn | Eric Lin | Shashikant Banerjee | +| [YARN-10037](https://issues.apache.org/jira/browse/YARN-10037) | Upgrade build tools for YARN Web UI v2 | Major | build, security, yarn-ui-v2 | Akira Ajisaka | Masatake Iwasaki | +| [YARN-10042](https://issues.apache.org/jira/browse/YARN-10042) | Uupgrade grpc-xxx depdencies to 1.26.0 | Major | . | liusheng | liusheng | +| [HADOOP-16774](https://issues.apache.org/jira/browse/HADOOP-16774) | TestDiskChecker and TestReadWriteDiskValidator fails when run with -Pparallel-tests | Major | . | Vinayakumar B | Vinayakumar B | +| [YARN-10054](https://issues.apache.org/jira/browse/YARN-10054) | Upgrade yarn to the latest version in Dockerfile | Major | build, yarn-ui-v2 | Akira Ajisaka | Akira Ajisaka | +| [YARN-10055](https://issues.apache.org/jira/browse/YARN-10055) | bower install fails | Blocker | build, yarn-ui-v2 | Akira Ajisaka | Akira Ajisaka | +| [HDFS-15076](https://issues.apache.org/jira/browse/HDFS-15076) | Fix tests that hold FSDirectory lock, without holding FSNamesystem lock. | Major | test | Konstantin Shvachko | Konstantin Shvachko | +| [HDFS-15073](https://issues.apache.org/jira/browse/HDFS-15073) | Replace curator-shaded guava import with the standard one | Minor | hdfs-client | Akira Ajisaka | Chandra Sanivarapu | +| [YARN-10057](https://issues.apache.org/jira/browse/YARN-10057) | Upgrade the dependencies managed by yarnpkg | Major | build, yarn-ui-v2 | Akira Ajisaka | Akira Ajisaka | +| [HDFS-15054](https://issues.apache.org/jira/browse/HDFS-15054) | Delete Snapshot not updating new modification time | Major | . | hemanthboyina | hemanthboyina | +| [HADOOP-16042](https://issues.apache.org/jira/browse/HADOOP-16042) | Update the link to HadoopJavaVersion | Minor | documentation | Akira Ajisaka | Chandra Sanivarapu | +| [YARN-10041](https://issues.apache.org/jira/browse/YARN-10041) | Should not use AbstractPath to create unix domain socket | Major | test | zhao bo | liusheng | +| [HDFS-14934](https://issues.apache.org/jira/browse/HDFS-14934) | [SBN Read] Standby NN throws many InterruptedExceptions when dfs.ha.tail-edits.period is 0 | Major | . | Takanobu Asanuma | Ayush Saxena | +| [HDFS-15063](https://issues.apache.org/jira/browse/HDFS-15063) | HttpFS : getFileStatus doesn't return ecPolicy | Major | . | hemanthboyina | hemanthboyina | +| [YARN-10053](https://issues.apache.org/jira/browse/YARN-10053) | Placement rules do not use correct group service init | Major | yarn | Wilfred Spiegelenburg | Wilfred Spiegelenburg | +| [HDFS-15068](https://issues.apache.org/jira/browse/HDFS-15068) | DataNode could meet deadlock if invoke refreshVolumes when register | Major | datanode | Xiaoqiao He | Aiphago | +| [HDFS-15089](https://issues.apache.org/jira/browse/HDFS-15089) | RBF: SmallFix for RBFMetrics in doc | Trivial | documentation, rbf | luhuachao | luhuachao | +| [MAPREDUCE-7255](https://issues.apache.org/jira/browse/MAPREDUCE-7255) | Fix typo in MapReduce documentaion example | Trivial | documentation | Sergey Pogorelov | Sergey Pogorelov | +| [YARN-9956](https://issues.apache.org/jira/browse/YARN-9956) | Improve connection error message for YARN ApiServerClient | Major | . | Eric Yang | Prabhu Joseph | +| [HADOOP-16773](https://issues.apache.org/jira/browse/HADOOP-16773) | Fix duplicate assertj-core dependency in hadoop-common module. | Minor | build | Akira Ajisaka | Xieming Li | +| [HDFS-15072](https://issues.apache.org/jira/browse/HDFS-15072) | HDFS MiniCluster fails to start when run in directory path with a % | Minor | . | Geoffrey Jacoby | Masatake Iwasaki | +| [HDFS-15077](https://issues.apache.org/jira/browse/HDFS-15077) | Fix intermittent failure of TestDFSClientRetries#testLeaseRenewSocketTimeout | Minor | test | Masatake Iwasaki | Masatake Iwasaki | +| [HDFS-15080](https://issues.apache.org/jira/browse/HDFS-15080) | Fix the issue in reading persistent memory cached data with an offset | Major | caching, datanode | Feilong He | Feilong He | +| [YARN-7387](https://issues.apache.org/jira/browse/YARN-7387) | org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.TestIncreaseAllocationExpirer fails intermittently | Major | . | Miklos Szegedi | Jim Brennan | +| [YARN-8672](https://issues.apache.org/jira/browse/YARN-8672) | TestContainerManager#testLocalingResourceWhileContainerRunning occasionally times out | Major | nodemanager | Jason Darrell Lowe | Chandni Singh | +| [HDFS-14957](https://issues.apache.org/jira/browse/HDFS-14957) | INodeReference Space Consumed was not same in QuotaUsage and ContentSummary | Major | namenode | hemanthboyina | hemanthboyina | +| [MAPREDUCE-7252](https://issues.apache.org/jira/browse/MAPREDUCE-7252) | Handling 0 progress in SimpleExponential task runtime estimator | Minor | . | Ahmed Hussein | Ahmed Hussein | +| [HDFS-15102](https://issues.apache.org/jira/browse/HDFS-15102) | HttpFS: put requests are not supported for path "/" | Major | . | hemanthboyina | hemanthboyina | +| [HDFS-15110](https://issues.apache.org/jira/browse/HDFS-15110) | HttpFS: POST requests are not supported for path "/" | Major | . | hemanthboyina | hemanthboyina | +| [HADOOP-16749](https://issues.apache.org/jira/browse/HADOOP-16749) | Configuration parsing of CDATA values are blank | Major | conf | Jonathan Turner Eagles | Daryn Sharp | +| [HDFS-15095](https://issues.apache.org/jira/browse/HDFS-15095) | Fix accidental comment in flaky test TestDecommissioningStatus | Major | hdfs | Ahmed Hussein | Ahmed Hussein | +| [HADOOP-16590](https://issues.apache.org/jira/browse/HADOOP-16590) | IBM Java has deprecated OS login module classes and OS principal classes. | Major | security | Nicholas Marion | | +| [YARN-10019](https://issues.apache.org/jira/browse/YARN-10019) | container-executor: misc improvements in child processes and exec() calls | Minor | nodemanager | Peter Bacsko | Peter Bacsko | +| [HDFS-15099](https://issues.apache.org/jira/browse/HDFS-15099) | [SBN Read] checkOperation(WRITE) should throw ObserverRetryOnActiveException on ObserverNode | Major | namenode | Konstantin Shvachko | Chen Liang | +| [HDFS-15108](https://issues.apache.org/jira/browse/HDFS-15108) | RBF: MembershipNamenodeResolver should invalidate cache incase of active namenode update | Major | . | Ayush Saxena | Ayush Saxena | +| [HDFS-14578](https://issues.apache.org/jira/browse/HDFS-14578) | AvailableSpaceBlockPlacementPolicy always prefers local node | Major | block placement | Wei-Chiu Chuang | Ayush Saxena | +| [YARN-9866](https://issues.apache.org/jira/browse/YARN-9866) | u:user2:%primary\_group is not working as expected | Major | . | Manikandan R | Manikandan R | +| [HADOOP-16683](https://issues.apache.org/jira/browse/HADOOP-16683) | Disable retry of FailoverOnNetworkExceptionRetry in case of wrapped AccessControlException | Major | common | Adam Antal | Adam Antal | +| [MAPREDUCE-7256](https://issues.apache.org/jira/browse/MAPREDUCE-7256) | Fix javadoc error in SimpleExponentialSmoothing | Minor | documentation | Masatake Iwasaki | Masatake Iwasaki | +| [YARN-8373](https://issues.apache.org/jira/browse/YARN-8373) | RM Received RMFatalEvent of type CRITICAL\_THREAD\_CRASH | Major | fairscheduler, resourcemanager | Girish Bhat | Wilfred Spiegelenburg | +| [YARN-9512](https://issues.apache.org/jira/browse/YARN-9512) | [JDK11] TestAuxServices#testCustomizedAuxServiceClassPath fails because of ClassCastException. | Major | test | Siyao Meng | Akira Ajisaka | +| [MAPREDUCE-7247](https://issues.apache.org/jira/browse/MAPREDUCE-7247) | Modify HistoryServerRest.html content,change The job attempt id‘s datatype from string to int | Major | documentation | zhaoshengjie | | +| [YARN-10070](https://issues.apache.org/jira/browse/YARN-10070) | NPE if no rule is defined and application-tag-based-placement is enabled | Major | . | Kinga Marton | Kinga Marton | +| [YARN-9970](https://issues.apache.org/jira/browse/YARN-9970) | Refactor TestUserGroupMappingPlacementRule#verifyQueueMapping | Major | . | Manikandan R | Manikandan R | +| [YARN-8148](https://issues.apache.org/jira/browse/YARN-8148) | Update decimal values for queue capacities shown on queue status CLI | Major | client | Prabhu Joseph | Prabhu Joseph | +| [YARN-10081](https://issues.apache.org/jira/browse/YARN-10081) | Exception message from ClientRMProxy#getRMAddress is misleading | Trivial | yarn | Adam Antal | Ravuri Sushma sree | +| [HADOOP-16808](https://issues.apache.org/jira/browse/HADOOP-16808) | Use forkCount and reuseForks parameters instead of forkMode in the config of maven surefire plugin | Minor | build | Akira Ajisaka | Xieming Li | +| [HADOOP-16793](https://issues.apache.org/jira/browse/HADOOP-16793) | Remove WARN log when ipc connection interrupted in Client#handleSaslConnectionFailure() | Minor | . | Lisheng Sun | Lisheng Sun | +| [HDFS-15126](https://issues.apache.org/jira/browse/HDFS-15126) | TestDatanodeRegistration#testForcedRegistration fails intermittently | Major | . | Ahmed Hussein | Ahmed Hussein | +| [YARN-9462](https://issues.apache.org/jira/browse/YARN-9462) | TestResourceTrackerService.testNodeRemovalGracefully fails sporadically | Minor | resourcemanager, test | Prabhu Joseph | Prabhu Joseph | +| [YARN-9790](https://issues.apache.org/jira/browse/YARN-9790) | Failed to set default-application-lifetime if maximum-application-lifetime is less than or equal to zero | Major | . | kyungwan nam | kyungwan nam | +| [HDFS-15128](https://issues.apache.org/jira/browse/HDFS-15128) | Unit test failing to clean testing data and crashed future Maven test run due to failure in TestDataNodeVolumeFailureToleration | Critical | hdfs, test | Ctest | Ctest | +| [HDFS-15143](https://issues.apache.org/jira/browse/HDFS-15143) | LocatedStripedBlock returns wrong block type | Major | . | Ayush Saxena | Ayush Saxena | +| [HDFS-14993](https://issues.apache.org/jira/browse/HDFS-14993) | checkDiskError doesn't work during datanode startup | Major | datanode | Yang Yun | Yang Yun | +| [HDFS-15145](https://issues.apache.org/jira/browse/HDFS-15145) | HttpFS: getAclStatus() returns permission as null | Major | . | hemanthboyina | hemanthboyina | +| [HDFS-13179](https://issues.apache.org/jira/browse/HDFS-13179) | TestLazyPersistReplicaRecovery#testDnRestartWithSavedReplicas fails intermittently | Critical | fs | Gabor Bota | Ahmed Hussein | +| [MAPREDUCE-7259](https://issues.apache.org/jira/browse/MAPREDUCE-7259) | testSpeculateSuccessfulWithUpdateEvents fails Intermittently | Minor | . | Ahmed Hussein | Ahmed Hussein | +| [YARN-10107](https://issues.apache.org/jira/browse/YARN-10107) | Invoking NMWebServices#getNMResourceInfo tries to execute gpu discovery binary even if auto discovery is turned off | Major | . | Szilard Nemeth | Szilard Nemeth | +| [MAPREDUCE-7079](https://issues.apache.org/jira/browse/MAPREDUCE-7079) | JobHistory#ServiceStop implementation is incorrect | Major | . | Jason Darrell Lowe | Ahmed Hussein | +| [HDFS-15118](https://issues.apache.org/jira/browse/HDFS-15118) | [SBN Read] Slow clients when Observer reads are enabled but there are no Observers on the cluster. | Major | hdfs-client | Konstantin Shvachko | Chen Liang | +| [YARN-9743](https://issues.apache.org/jira/browse/YARN-9743) | [JDK11] TestTimelineWebServices.testContextFactory fails | Major | timelineservice | Adam Antal | Akira Ajisaka | +| [HDFS-7175](https://issues.apache.org/jira/browse/HDFS-7175) | Client-side SocketTimeoutException during Fsck | Major | namenode | Carl Steinbach | Stephen O'Donnell | +| [HADOOP-16834](https://issues.apache.org/jira/browse/HADOOP-16834) | Replace com.sun.istack.Nullable with javax.annotation.Nullable in DNS.java | Minor | build | Akira Ajisaka | Xieming Li | +| [HDFS-15136](https://issues.apache.org/jira/browse/HDFS-15136) | LOG flooding in secure mode when Cookies are not set in request header | Major | . | Renukaprasad C | Renukaprasad C | +| [HDFS-15115](https://issues.apache.org/jira/browse/HDFS-15115) | Namenode crash caused by NPE in BlockPlacementPolicyDefault when dynamically change logger to debug | Major | . | wangzhixiang | wangzhixiang | +| [HDFS-15158](https://issues.apache.org/jira/browse/HDFS-15158) | The number of failed volumes mismatch with volumeFailures of Datanode metrics | Minor | datanode | Yang Yun | Yang Yun | +| [HADOOP-16851](https://issues.apache.org/jira/browse/HADOOP-16851) | unused import in Configuration class | Trivial | conf | runzhou wu | Jan Hentschel | +| [HADOOP-16849](https://issues.apache.org/jira/browse/HADOOP-16849) | start-build-env.sh behaves incorrectly when username is numeric only | Minor | build | Jihyun Cho | Jihyun Cho | +| [HADOOP-16856](https://issues.apache.org/jira/browse/HADOOP-16856) | cmake is missing in the CentOS 8 section of BUILDING.txt | Minor | build, documentation | Akira Ajisaka | Masatake Iwasaki | +| [HDFS-15161](https://issues.apache.org/jira/browse/HDFS-15161) | When evictableMmapped or evictable size is zero, do not throw NoSuchElementException in ShortCircuitCache#close() | Major | . | Lisheng Sun | Lisheng Sun | +| [YARN-9521](https://issues.apache.org/jira/browse/YARN-9521) | RM failed to start due to system services | Major | . | kyungwan nam | kyungwan nam | +| [YARN-10136](https://issues.apache.org/jira/browse/YARN-10136) | [Router] : Application metrics are hardcode as N/A in UI. | Major | federation, yarn | Sushanta Sen | Bilwa S T | +| [HDFS-15164](https://issues.apache.org/jira/browse/HDFS-15164) | Fix TestDelegationTokensWithHA | Major | . | Ayush Saxena | Ayush Saxena | +| [HADOOP-16868](https://issues.apache.org/jira/browse/HADOOP-16868) | ipc.Server readAndProcess threw NullPointerException | Major | rpc-server | Tsz-wo Sze | Tsz-wo Sze | +| [HDFS-15165](https://issues.apache.org/jira/browse/HDFS-15165) | In Du missed calling getAttributesProvider | Major | . | Bharat Viswanadham | Bharat Viswanadham | +| [YARN-10119](https://issues.apache.org/jira/browse/YARN-10119) | Cannot reset the AM failure count for YARN Service | Major | . | kyungwan nam | kyungwan nam | +| [HADOOP-16869](https://issues.apache.org/jira/browse/HADOOP-16869) | Upgrade findbugs-maven-plugin to 3.0.5 to fix mvn findbugs:findbugs failure | Major | build | Akira Ajisaka | Akira Ajisaka | +| [HDFS-15052](https://issues.apache.org/jira/browse/HDFS-15052) | WebHDFS getTrashRoot leads to OOM due to FileSystem object creation | Major | webhdfs | Wei-Chiu Chuang | Masatake Iwasaki | +| [YARN-10147](https://issues.apache.org/jira/browse/YARN-10147) | FPGA plugin can't find the localized aocx file | Major | nodemanager | Peter Bacsko | Peter Bacsko | +| [HDFS-15185](https://issues.apache.org/jira/browse/HDFS-15185) | StartupProgress reports edits segments until the entire startup completes | Major | namenode | Konstantin Shvachko | Konstantin Shvachko | +| [HDFS-15182](https://issues.apache.org/jira/browse/HDFS-15182) | TestBlockManager#testOneOfTwoRacksDecommissioned() fail in trunk | Minor | . | Lisheng Sun | Lisheng Sun | +| [HDFS-15166](https://issues.apache.org/jira/browse/HDFS-15166) | Remove redundant field fStream in ByteStringLog | Major | . | Konstantin Shvachko | Xieming Li | +| [HDFS-15187](https://issues.apache.org/jira/browse/HDFS-15187) | CORRUPT replica mismatch between namenodes after failover | Critical | . | Ayush Saxena | Ayush Saxena | +| [YARN-10143](https://issues.apache.org/jira/browse/YARN-10143) | YARN-10101 broke Yarn logs CLI | Blocker | yarn | Adam Antal | Adam Antal | +| [HADOOP-16841](https://issues.apache.org/jira/browse/HADOOP-16841) | The description of hadoop.http.authentication.signature.secret.file contains outdated information | Minor | documentation | Akira Ajisaka | Xieming Li | +| [YARN-8767](https://issues.apache.org/jira/browse/YARN-8767) | TestStreamingStatus fails | Major | . | Andras Bokor | Andras Bokor | +| [YARN-10156](https://issues.apache.org/jira/browse/YARN-10156) | Fix typo 'complaint' which means quite different in Federation.md | Minor | documentation, federation | Sungpeo Kook | Sungpeo Kook | +| [YARN-9593](https://issues.apache.org/jira/browse/YARN-9593) | Updating scheduler conf with comma in config value fails | Major | . | Anthony Hsu | Tanu Ajmera | +| [YARN-10141](https://issues.apache.org/jira/browse/YARN-10141) | Interceptor in FederationInterceptorREST doesnt update on RM switchover | Major | federation, restapi | D M Murali Krishna Reddy | D M Murali Krishna Reddy | +| [YARN-10152](https://issues.apache.org/jira/browse/YARN-10152) | Fix findbugs warnings in hadoop-yarn-applications-mawo-core module | Major | . | Akira Ajisaka | Akira Ajisaka | +| [HDFS-15167](https://issues.apache.org/jira/browse/HDFS-15167) | Block Report Interval shouldn't be reset apart from first Block Report | Major | . | Ayush Saxena | Ayush Saxena | +| [HDFS-15147](https://issues.apache.org/jira/browse/HDFS-15147) | LazyPersistTestCase wait logic is error-prone | Minor | . | Ahmed Hussein | Ahmed Hussein | +| [HDFS-14668](https://issues.apache.org/jira/browse/HDFS-14668) | Support Fuse with Users from multiple Security Realms | Critical | fuse-dfs | Sailesh Patel | Istvan Fajth | +| [HDFS-15124](https://issues.apache.org/jira/browse/HDFS-15124) | Crashing bugs in NameNode when using a valid configuration for \`dfs.namenode.audit.loggers\` | Critical | namenode | Ctest | Ctest | +| [YARN-10155](https://issues.apache.org/jira/browse/YARN-10155) | TestDelegationTokenRenewer.testTokenThreadTimeout fails in trunk | Major | yarn | Adam Antal | Manikandan R | +| [HDFS-15111](https://issues.apache.org/jira/browse/HDFS-15111) | stopStandbyServices() should log which service state it is transitioning from. | Major | hdfs, logging | Konstantin Shvachko | Xieming Li | +| [HDFS-15199](https://issues.apache.org/jira/browse/HDFS-15199) | NPE in BlockSender | Major | . | Ayush Saxena | Ayush Saxena | +| [YARN-9452](https://issues.apache.org/jira/browse/YARN-9452) | Fix TestDistributedShell and TestTimelineAuthFilterForV2 failures | Major | ATSv2, distributed-shell, test | Prabhu Joseph | Prabhu Joseph | +| [HADOOP-16891](https://issues.apache.org/jira/browse/HADOOP-16891) | Upgrade jackson-databind to 2.9.10.3 | Blocker | . | Siyao Meng | Siyao Meng | +| [HDFS-15149](https://issues.apache.org/jira/browse/HDFS-15149) | TestDeadNodeDetection test cases time-out | Major | datanode | Ahmed Hussein | Lisheng Sun | +| [HADOOP-16897](https://issues.apache.org/jira/browse/HADOOP-16897) | Sort fields in ReflectionUtils.java | Minor | test, util | cpugputpu | cpugputpu | +| [HADOOP-16437](https://issues.apache.org/jira/browse/HADOOP-16437) | Documentation typos: fs.s3a.experimental.fadvise -\> fs.s3a.experimental.input.fadvise | Major | documentation, fs/s3 | Josh Rosen | Josh Rosen | +| [HDFS-15204](https://issues.apache.org/jira/browse/HDFS-15204) | TestRetryCacheWithHA testRemoveCacheDescriptor fails intermittently | Major | hdfs | Ahmed Hussein | Ahmed Hussein | +| [HDFS-14977](https://issues.apache.org/jira/browse/HDFS-14977) | Quota Usage and Content summary are not same in Truncate with Snapshot | Major | . | hemanthboyina | hemanthboyina | +| [YARN-10173](https://issues.apache.org/jira/browse/YARN-10173) | Make pid file generation timeout configurable in case of reacquired container | Minor | yarn | Adam Antal | Adam Antal | +| [HDFS-15212](https://issues.apache.org/jira/browse/HDFS-15212) | TestEncryptionZones.testVersionAndSuiteNegotiation fails in trunk | Major | . | Ayush Saxena | Ayush Saxena | +| [HADOOP-16840](https://issues.apache.org/jira/browse/HADOOP-16840) | AliyunOSS: getFileStatus throws FileNotFoundException in versioning bucket | Major | fs/oss | wujinhu | wujinhu | +| [YARN-9419](https://issues.apache.org/jira/browse/YARN-9419) | Log a warning if GPU isolation is enabled but LinuxContainerExecutor is disabled | Major | . | Szilard Nemeth | Andras Gyori | +| [YARN-9427](https://issues.apache.org/jira/browse/YARN-9427) | TestContainerSchedulerQueuing.testKillOnlyRequiredOpportunisticContainers fails sporadically | Major | scheduler, test | Prabhu Joseph | Ahmed Hussein | +| [HDFS-15135](https://issues.apache.org/jira/browse/HDFS-15135) | EC : ArrayIndexOutOfBoundsException in BlockRecoveryWorker#RecoveryTaskStriped. | Major | erasure-coding | Surendra Singh Lilhore | Ravuri Sushma sree | +| [HDFS-14442](https://issues.apache.org/jira/browse/HDFS-14442) | Disagreement between HAUtil.getAddressOfActive and RpcInvocationHandler.getConnectionId | Major | . | Erik Krogen | Ravuri Sushma sree | +| [HDFS-14612](https://issues.apache.org/jira/browse/HDFS-14612) | SlowDiskReport won't update when SlowDisks is always empty in heartbeat | Major | . | Haibin Huang | Haibin Huang | +| [HDFS-15155](https://issues.apache.org/jira/browse/HDFS-15155) | writeIoRate of DataNodeVolumeMetrics is never used | Major | hdfs | Haibin Huang | Haibin Huang | +| [HDFS-15216](https://issues.apache.org/jira/browse/HDFS-15216) | Wrong Use Case of -showprogress in fsck | Major | . | Ravuri Sushma sree | Ravuri Sushma sree | +| [HADOOP-16885](https://issues.apache.org/jira/browse/HADOOP-16885) | Encryption zone file copy failure leaks temp file .\_COPYING\_ and wrapped stream | Major | . | Xiaoyu Yao | Xiaoyu Yao | +| [HDFS-11396](https://issues.apache.org/jira/browse/HDFS-11396) | TestNameNodeMetadataConsistency#testGenerationStampInFuture timed out | Minor | namenode, test | John Zhuge | Ayush Saxena | +| [YARN-10195](https://issues.apache.org/jira/browse/YARN-10195) | Dependency divergence building Timeline Service on HBase 2.2.0 and above | Major | build, timelineservice | Wei-Chiu Chuang | Wei-Chiu Chuang | +| [HDFS-14820](https://issues.apache.org/jira/browse/HDFS-14820) | The default 8KB buffer of BlockReaderRemote#newBlockReader#BufferedOutputStream is too big | Major | . | Lisheng Sun | Lisheng Sun | +| [HDFS-15211](https://issues.apache.org/jira/browse/HDFS-15211) | EC: File write hangs during close in case of Exception during updatePipeline | Critical | . | Ayush Saxena | Ayush Saxena | +| [HDFS-14919](https://issues.apache.org/jira/browse/HDFS-14919) | Provide Non DFS Used per DataNode in DataNode UI | Major | . | Lisheng Sun | Lisheng Sun | +| [HDFS-15208](https://issues.apache.org/jira/browse/HDFS-15208) | Suppress bogus AbstractWadlGeneratorGrammarGenerator in KMS stderr in hdfs | Trivial | . | Wei-Chiu Chuang | Wei-Chiu Chuang | +| [YARN-10034](https://issues.apache.org/jira/browse/YARN-10034) | Allocation tags are not removed when node decommission | Major | . | kyungwan nam | kyungwan nam | +| [HDFS-15223](https://issues.apache.org/jira/browse/HDFS-15223) | FSCK fails if one namenode is not available | Major | . | Ayush Saxena | Ayush Saxena | +| [HDFS-15200](https://issues.apache.org/jira/browse/HDFS-15200) | Delete Corrupt Replica Immediately Irrespective of Replicas On Stale Storage | Critical | . | Ayush Saxena | Ayush Saxena | +| [HDFS-15214](https://issues.apache.org/jira/browse/HDFS-15214) | WebHDFS: Add snapshot counts to Content Summary | Major | . | hemanthboyina | hemanthboyina | +| [HDFS-15227](https://issues.apache.org/jira/browse/HDFS-15227) | NPE if the last block changes from COMMITTED to COMPLETE during FSCK | Major | . | krishna reddy | Ayush Saxena | +| [YARN-10198](https://issues.apache.org/jira/browse/YARN-10198) | [managedParent].%primary\_group mapping rule doesn't work after YARN-9868 | Major | capacity scheduler | Peter Bacsko | Peter Bacsko | +| [HDFS-15113](https://issues.apache.org/jira/browse/HDFS-15113) | Missing IBR when NameNode restart if open processCommand async feature | Blocker | datanode | Xiaoqiao He | Xiaoqiao He | +| [HDFS-15232](https://issues.apache.org/jira/browse/HDFS-15232) | Fix libhdfspp test failures with GCC 7 | Major | native, test | Akira Ajisaka | Akira Ajisaka | +| [HDFS-15201](https://issues.apache.org/jira/browse/HDFS-15201) | SnapshotCounter hits MaxSnapshotID limit | Major | snapshots | Karthik Palanisamy | Karthik Palanisamy | +| [HDFS-15219](https://issues.apache.org/jira/browse/HDFS-15219) | DFS Client will stuck when ResponseProcessor.run throw Error | Major | hdfs-client | zhengchenyu | zhengchenyu | +| [HDFS-15215](https://issues.apache.org/jira/browse/HDFS-15215) | The Timestamp for longest write/read lock held log is wrong | Major | . | Toshihiro Suzuki | Toshihiro Suzuki | +| [HDFS-15234](https://issues.apache.org/jira/browse/HDFS-15234) | Add a default method body for the INodeAttributeProvider#checkPermissionWithContext API | Blocker | namenode | Wei-Chiu Chuang | Wei-Chiu Chuang | +| [HDFS-15193](https://issues.apache.org/jira/browse/HDFS-15193) | Improving the error message for missing \`dfs.namenode.rpc-address.$NAMESERVICE\` | Minor | hdfs | Ctest | Ctest | +| [YARN-10202](https://issues.apache.org/jira/browse/YARN-10202) | Fix documentation about NodeAttributes. | Minor | documentation | Sen Zhao | Sen Zhao | +| [MAPREDUCE-7268](https://issues.apache.org/jira/browse/MAPREDUCE-7268) | Fix TestMapreduceConfigFields | Major | mrv2 | Ayush Saxena | Wanqiang Ji | +| [HADOOP-16949](https://issues.apache.org/jira/browse/HADOOP-16949) | pylint fails in the build environment | Critical | build | Akira Ajisaka | Akira Ajisaka | +| [MAPREDUCE-7269](https://issues.apache.org/jira/browse/MAPREDUCE-7269) | TestNetworkedJob fails | Major | test | Akira Ajisaka | Akira Ajisaka | +| [HADOOP-14836](https://issues.apache.org/jira/browse/HADOOP-14836) | Upgrade maven-clean-plugin to 3.1.0 | Major | build | Allen Wittenauer | Akira Ajisaka | +| [YARN-10207](https://issues.apache.org/jira/browse/YARN-10207) | CLOSE\_WAIT socket connection leaks during rendering of (corrupted) aggregated logs on the JobHistoryServer Web UI | Major | yarn | Siddharth Ahuja | Siddharth Ahuja | +| [YARN-10226](https://issues.apache.org/jira/browse/YARN-10226) | NPE in Capacity Scheduler while using %primary\_group queue mapping | Critical | capacity scheduler | Peter Bacsko | Peter Bacsko | +| [HDFS-15269](https://issues.apache.org/jira/browse/HDFS-15269) | NameNode should check the authorization API version only once during initialization | Blocker | namenode | Wei-Chiu Chuang | Wei-Chiu Chuang | +| [HADOOP-16932](https://issues.apache.org/jira/browse/HADOOP-16932) | distcp copy calls getFileStatus() needlessly and can fail against S3 | Minor | fs/s3, tools/distcp | Thangamani Murugasamy | Steve Loughran | +| [YARN-2710](https://issues.apache.org/jira/browse/YARN-2710) | RM HA tests failed intermittently on trunk | Major | client | Wangda Tan | Ahmed Hussein | +| [HDFS-12862](https://issues.apache.org/jira/browse/HDFS-12862) | CacheDirective becomes invalid when NN restart or failover | Major | caching, hdfs | Wang XL | Wang XL | +| [MAPREDUCE-7272](https://issues.apache.org/jira/browse/MAPREDUCE-7272) | TaskAttemptListenerImpl excessive log messages | Major | . | Ahmed Hussein | Ahmed Hussein | +| [YARN-10219](https://issues.apache.org/jira/browse/YARN-10219) | YARN service placement constraints is broken | Blocker | . | Eric Yang | Eric Yang | +| [YARN-10233](https://issues.apache.org/jira/browse/YARN-10233) | [YARN UI2] No Logs were found in "YARN Daemon Logs" page | Blocker | yarn-ui-v2 | Akhil PB | Akhil PB | +| [HADOOP-16985](https://issues.apache.org/jira/browse/HADOOP-16985) | Handle release package related issues | Major | . | Vinayakumar B | Vinayakumar B | +| [HDFS-15283](https://issues.apache.org/jira/browse/HDFS-15283) | Cache pool MAXTTL is not persisted and restored on cluster restart | Major | namenode | Stephen O'Donnell | Stephen O'Donnell | +| [HDFS-15218](https://issues.apache.org/jira/browse/HDFS-15218) | RBF: MountTableRefresherService failed to refresh other router MountTableEntries in secure mode. | Major | rbf | Surendra Singh Lilhore | Surendra Singh Lilhore | +| [HADOOP-16944](https://issues.apache.org/jira/browse/HADOOP-16944) | Use Yetus 0.12.0 in GitHub PR | Major | build | Akira Ajisaka | Akira Ajisaka | +| [HDFS-15276](https://issues.apache.org/jira/browse/HDFS-15276) | Concat on INodeRefernce fails with illegal state exception | Critical | . | hemanthboyina | hemanthboyina | +| [HADOOP-16341](https://issues.apache.org/jira/browse/HADOOP-16341) | ShutDownHookManager: Regressed performance on Hook removals after HADOOP-15679 | Major | common | Gopal Vijayaraghavan | Gopal Vijayaraghavan | +| [YARN-10223](https://issues.apache.org/jira/browse/YARN-10223) | Duplicate jersey-test-framework-core dependency in yarn-server-common | Minor | build | Akira Ajisaka | Akira Ajisaka | +| [HADOOP-17007](https://issues.apache.org/jira/browse/HADOOP-17007) | hadoop-cos fails to build | Major | fs/cos | Wei-Chiu Chuang | Yang Yu | +| [YARN-9848](https://issues.apache.org/jira/browse/YARN-9848) | revert YARN-4946 | Blocker | log-aggregation, resourcemanager | Steven Rand | Steven Rand | +| [HDFS-15286](https://issues.apache.org/jira/browse/HDFS-15286) | Concat on a same files deleting the file | Critical | . | hemanthboyina | hemanthboyina | +| [HDFS-15301](https://issues.apache.org/jira/browse/HDFS-15301) | statfs function in hdfs-fuse is not working | Major | fuse-dfs, libhdfs | Aryan Gupta | Aryan Gupta | +| [HDFS-15334](https://issues.apache.org/jira/browse/HDFS-15334) | INodeAttributeProvider's new API checkPermissionWithContext not getting called in for authorization | Major | . | Wei-Chiu Chuang | Wei-Chiu Chuang | +| [HDFS-15323](https://issues.apache.org/jira/browse/HDFS-15323) | StandbyNode fails transition to active due to insufficient transaction tailing | Major | namenode, qjm | Konstantin Shvachko | Konstantin Shvachko | +| [HDFS-15343](https://issues.apache.org/jira/browse/HDFS-15343) | TestConfiguredFailoverProxyProvider is failing | Major | test | Akira Ajisaka | Akira Ajisaka | +| [HADOOP-15565](https://issues.apache.org/jira/browse/HADOOP-15565) | ViewFileSystem.close doesn't close child filesystems and causes FileSystem objects leak. | Major | . | Jinglun | Jinglun | +| [YARN-9898](https://issues.apache.org/jira/browse/YARN-9898) | Dependency netty-all-4.1.27.Final doesn't support ARM platform | Major | . | liusheng | liusheng | +| [YARN-10265](https://issues.apache.org/jira/browse/YARN-10265) | Upgrade Netty-all dependency to latest version 4.1.50 to fix ARM support issue | Major | . | liusheng | liusheng | +| [YARN-9444](https://issues.apache.org/jira/browse/YARN-9444) | YARN API ResourceUtils's getRequestedResourcesFromConfig doesn't recognize yarn.io/gpu as a valid resource | Minor | api | Gergely Pollak | Gergely Pollak | +| [HDFS-15038](https://issues.apache.org/jira/browse/HDFS-15038) | TestFsck testFsckListCorruptSnapshotFiles is failing in trunk | Major | . | hemanthboyina | hemanthboyina | +| [HDFS-15293](https://issues.apache.org/jira/browse/HDFS-15293) | Relax the condition for accepting a fsimage when receiving a checkpoint | Critical | namenode | Chen Liang | Chen Liang | +| [YARN-10228](https://issues.apache.org/jira/browse/YARN-10228) | Yarn Service fails if am java opts contains ZK authentication file path | Major | . | Bilwa S T | Bilwa S T | +| [MAPREDUCE-7278](https://issues.apache.org/jira/browse/MAPREDUCE-7278) | Speculative execution behavior is observed even when mapreduce.map.speculative and mapreduce.reduce.speculative are false | Major | task | Tarun Parimi | Tarun Parimi | +| [YARN-10314](https://issues.apache.org/jira/browse/YARN-10314) | YarnClient throws NoClassDefFoundError for WebSocketException with only shaded client jars | Blocker | yarn | Vinayakumar B | Vinayakumar B | +| [HDFS-15421](https://issues.apache.org/jira/browse/HDFS-15421) | IBR leak causes standby NN to be stuck in safe mode | Blocker | namenode | Kihwal Lee | Akira Ajisaka | + + +### TESTS: + +| JIRA | Summary | Priority | Component | Reporter | Contributor | +|:---- |:---- | :--- |:---- |:---- |:---- | +| [YARN-8907](https://issues.apache.org/jira/browse/YARN-8907) | Modify a logging message in TestCapacityScheduler | Trivial | . | Zhankun Tang | Zhankun Tang | +| [YARN-8904](https://issues.apache.org/jira/browse/YARN-8904) | TestRMDelegationTokens can fail in testRMDTMasterKeyStateOnRollingMasterKey | Minor | test | Wilfred Spiegelenburg | Wilfred Spiegelenburg | +| [YARN-8944](https://issues.apache.org/jira/browse/YARN-8944) | TestContainerAllocation.testUserLimitAllocationMultipleContainers failure after YARN-8896 | Minor | capacity scheduler | Wilfred Spiegelenburg | Wilfred Spiegelenburg | +| [YARN-9263](https://issues.apache.org/jira/browse/YARN-9263) | TestConfigurationNodeAttributesProvider fails after Mockito updated | Minor | . | Weiwei Yang | Weiwei Yang | +| [YARN-9315](https://issues.apache.org/jira/browse/YARN-9315) | TestCapacitySchedulerMetrics fails intermittently | Minor | capacity scheduler | Prabhu Joseph | Prabhu Joseph | +| [YARN-9316](https://issues.apache.org/jira/browse/YARN-9316) | TestPlacementConstraintsUtil#testInterAppConstraintsByAppID fails intermittently | Minor | capacity scheduler | Prabhu Joseph | Prabhu Joseph | +| [YARN-9324](https://issues.apache.org/jira/browse/YARN-9324) | TestSchedulingRequestContainerAllocation(Async) fails with junit-4.11 | Major | capacity scheduler | Prabhu Joseph | Prabhu Joseph | +| [YARN-9338](https://issues.apache.org/jira/browse/YARN-9338) | Timeline related testcases are failing | Major | . | Prabhu Joseph | Abhishek Modi | +| [YARN-9299](https://issues.apache.org/jira/browse/YARN-9299) | TestTimelineReaderWhitelistAuthorizationFilter ignores Http Errors | Major | . | Prabhu Joseph | Prabhu Joseph | +| [YARN-9404](https://issues.apache.org/jira/browse/YARN-9404) | TestApplicationLifetimeMonitor#testApplicationLifetimeMonitor fails intermittent | Major | resourcemanager | Prabhu Joseph | Prabhu Joseph | +| [YARN-9405](https://issues.apache.org/jira/browse/YARN-9405) | TestYarnNativeServices#testExpressUpgrade fails intermittent | Major | yarn-native-services | Prabhu Joseph | Prabhu Joseph | +| [YARN-9325](https://issues.apache.org/jira/browse/YARN-9325) | TestQueueManagementDynamicEditPolicy fails intermittent | Minor | capacity scheduler | Prabhu Joseph | Prabhu Joseph | +| [HDFS-11950](https://issues.apache.org/jira/browse/HDFS-11950) | Disable libhdfs zerocopy test on Mac | Minor | libhdfs | John Zhuge | Akira Ajisaka | +| [HDFS-11949](https://issues.apache.org/jira/browse/HDFS-11949) | Add testcase for ensuring that FsShell cann't move file to the target directory that file exists | Minor | test | legend | legend | +| [YARN-10072](https://issues.apache.org/jira/browse/YARN-10072) | TestCSAllocateCustomResource failures | Major | yarn | Jim Brennan | Jim Brennan | +| [HDFS-15092](https://issues.apache.org/jira/browse/HDFS-15092) | TestRedudantBlocks#testProcessOverReplicatedAndRedudantBlock sometimes fails | Minor | test | Fei Hui | Fei Hui | +| [YARN-10161](https://issues.apache.org/jira/browse/YARN-10161) | TestRouterWebServicesREST is corrupting STDOUT | Minor | yarn | Jim Brennan | Jim Brennan | +| [HADOOP-14206](https://issues.apache.org/jira/browse/HADOOP-14206) | TestSFTPFileSystem#testFileExists failure: Invalid encoding for signature | Major | fs, test | John Zhuge | Jim Brennan | + + +### SUB-TASKS: + +| JIRA | Summary | Priority | Component | Reporter | Contributor | +|:---- |:---- | :--- |:---- |:---- |:---- | +| [HDFS-12975](https://issues.apache.org/jira/browse/HDFS-12975) | Changes to the NameNode to support reads from standby | Major | namenode | Konstantin Shvachko | Chao Sun | +| [HDFS-12977](https://issues.apache.org/jira/browse/HDFS-12977) | Add stateId to RPC headers. | Major | ipc, namenode | Konstantin Shvachko | Plamen Jeliazkov | +| [HDFS-13331](https://issues.apache.org/jira/browse/HDFS-13331) | Add lastSeenStateId to RpcRequestHeader. | Major | . | Plamen Jeliazkov | Plamen Jeliazkov | +| [HDFS-11807](https://issues.apache.org/jira/browse/HDFS-11807) | libhdfs++: Get minidfscluster tests running under valgrind | Major | hdfs-client | James Clampffer | Anatoli Shein | +| [HDFS-13286](https://issues.apache.org/jira/browse/HDFS-13286) | Add haadmin commands to transition between standby and observer | Major | ha, hdfs, namenode | Chao Sun | Chao Sun | +| [HDFS-13578](https://issues.apache.org/jira/browse/HDFS-13578) | Add ReadOnly annotation to methods in ClientProtocol | Major | . | Chao Sun | Chao Sun | +| [HDFS-13399](https://issues.apache.org/jira/browse/HDFS-13399) | Make Client field AlignmentContext non-static. | Major | . | Plamen Jeliazkov | Plamen Jeliazkov | +| [HDFS-13607](https://issues.apache.org/jira/browse/HDFS-13607) | [Edit Tail Fast Path Pt 1] Enhance JournalNode with an in-memory cache of recent edit transactions | Major | ha, journal-node | Erik Krogen | Erik Krogen | +| [HDFS-13608](https://issues.apache.org/jira/browse/HDFS-13608) | [Edit Tail Fast Path Pt 2] Add ability for JournalNode to serve edits via RPC | Major | . | Erik Krogen | Erik Krogen | +| [HDFS-13609](https://issues.apache.org/jira/browse/HDFS-13609) | [Edit Tail Fast Path Pt 3] NameNode-side changes to support tailing edits via RPC | Major | ha, namenode | Erik Krogen | Erik Krogen | +| [HDFS-13706](https://issues.apache.org/jira/browse/HDFS-13706) | ClientGCIContext should be correctly named ClientGSIContext | Major | . | Konstantin Shvachko | Konstantin Shvachko | +| [HDFS-12976](https://issues.apache.org/jira/browse/HDFS-12976) | Introduce ObserverReadProxyProvider | Major | hdfs-client | Konstantin Shvachko | Chao Sun | +| [HDFS-13665](https://issues.apache.org/jira/browse/HDFS-13665) | Move RPC response serialization into Server.doResponse | Major | . | Plamen Jeliazkov | Plamen Jeliazkov | +| [HADOOP-15349](https://issues.apache.org/jira/browse/HADOOP-15349) | S3Guard DDB retryBackoff to be more informative on limits exceeded | Major | fs/s3 | Steve Loughran | Gabor Bota | +| [HDFS-13610](https://issues.apache.org/jira/browse/HDFS-13610) | [Edit Tail Fast Path Pt 4] Cleanup: integration test, documentation, remove unnecessary dummy sync | Major | ha, journal-node, namenode | Erik Krogen | Erik Krogen | +| [HDFS-13150](https://issues.apache.org/jira/browse/HDFS-13150) | [Edit Tail Fast Path] Allow SbNN to tail in-progress edits from JN via RPC | Major | ha, hdfs, journal-node, namenode | Erik Krogen | Erik Krogen | +| [HDFS-13688](https://issues.apache.org/jira/browse/HDFS-13688) | Introduce msync API call | Major | . | Chen Liang | Chen Liang | +| [HDFS-13789](https://issues.apache.org/jira/browse/HDFS-13789) | Reduce logging frequency of QuorumJournalManager#selectInputStreams | Trivial | namenode, qjm | Erik Krogen | Erik Krogen | +| [HDFS-13767](https://issues.apache.org/jira/browse/HDFS-13767) | Add msync server implementation. | Major | namenode | Chen Liang | Chen Liang | +| [HDFS-13851](https://issues.apache.org/jira/browse/HDFS-13851) | Remove AlignmentContext from AbstractNNFailoverProxyProvider | Major | . | Konstantin Shvachko | Konstantin Shvachko | +| [HDFS-13782](https://issues.apache.org/jira/browse/HDFS-13782) | ObserverReadProxyProvider should work with IPFailoverProxyProvider | Major | test | Konstantin Shvachko | Konstantin Shvachko | +| [HDFS-13779](https://issues.apache.org/jira/browse/HDFS-13779) | Implement performFailover logic for ObserverReadProxyProvider. | Major | hdfs-client | Konstantin Shvachko | Erik Krogen | +| [HADOOP-15635](https://issues.apache.org/jira/browse/HADOOP-15635) | s3guard set-capacity command to fail fast if bucket is unguarded | Minor | fs/s3 | Steve Loughran | Gabor Bota | +| [HADOOP-15750](https://issues.apache.org/jira/browse/HADOOP-15750) | remove obsolete S3A test ITestS3ACredentialsInURL | Major | fs/s3 | Steve Loughran | Steve Loughran | +| [HDFS-13880](https://issues.apache.org/jira/browse/HDFS-13880) | Add mechanism to allow certain RPC calls to bypass sync | Major | namenode | Chen Liang | Chen Liang | +| [HDFS-13778](https://issues.apache.org/jira/browse/HDFS-13778) | TestStateAlignmentContextWithHA should use real ObserverReadProxyProvider instead of AlignmentContextProxyProvider. | Major | test | Konstantin Shvachko | Plamen Jeliazkov | +| [HDFS-13749](https://issues.apache.org/jira/browse/HDFS-13749) | Use getServiceStatus to discover observer namenodes | Major | . | Chao Sun | Chao Sun | +| [HDFS-13898](https://issues.apache.org/jira/browse/HDFS-13898) | Throw retriable exception for getBlockLocations when ObserverNameNode is in safemode | Major | . | Chao Sun | Chao Sun | +| [HDFS-13791](https://issues.apache.org/jira/browse/HDFS-13791) | Limit logging frequency of edit tail related statements | Major | hdfs, qjm | Erik Krogen | Erik Krogen | +| [HADOOP-15767](https://issues.apache.org/jira/browse/HADOOP-15767) | [JDK10] Building native package on JDK10 fails due to missing javah | Major | native | Takanobu Asanuma | Takanobu Asanuma | +| [HDFS-13944](https://issues.apache.org/jira/browse/HDFS-13944) | [JDK10] Fix javadoc errors in hadoop-hdfs-rbf module | Major | documentation | Akira Ajisaka | Íñigo Goiri | +| [HADOOP-15621](https://issues.apache.org/jira/browse/HADOOP-15621) | S3Guard: Implement time-based (TTL) expiry for Authoritative Directory Listing | Major | fs/s3 | Aaron Fabbri | Gabor Bota | +| [HDFS-13877](https://issues.apache.org/jira/browse/HDFS-13877) | HttpFS: Implement GETSNAPSHOTDIFF | Major | httpfs | Siyao Meng | Siyao Meng | +| [HDFS-13961](https://issues.apache.org/jira/browse/HDFS-13961) | TestObserverNode refactoring | Major | test | Konstantin Shvachko | Konstantin Shvachko | +| [YARN-8763](https://issues.apache.org/jira/browse/YARN-8763) | Add WebSocket logic to the Node Manager web server to establish servlet | Major | . | Zian Chen | Zian Chen | +| [HADOOP-15775](https://issues.apache.org/jira/browse/HADOOP-15775) | [JDK9] Add missing javax.activation-api dependency | Critical | test | Akira Ajisaka | Akira Ajisaka | +| [HADOOP-15827](https://issues.apache.org/jira/browse/HADOOP-15827) | NPE in DynamoDBMetadataStore.lambda$listChildren for root + auth S3Guard | Blocker | fs/s3 | Steve Loughran | Gabor Bota | +| [YARN-7652](https://issues.apache.org/jira/browse/YARN-7652) | Handle AM register requests asynchronously in FederationInterceptor | Major | amrmproxy, federation | Subramaniam Krishnan | Botong Huang | +| [HADOOP-15825](https://issues.apache.org/jira/browse/HADOOP-15825) | ABFS: Enable some tests for namespace not enabled account using OAuth | Major | fs/azure | Da Zhou | Da Zhou | +| [HADOOP-15785](https://issues.apache.org/jira/browse/HADOOP-15785) | [JDK10] Javadoc build fails on JDK 10 in hadoop-common | Major | build, documentation | Takanobu Asanuma | Dinesh Chitlangia | +| [YARN-8777](https://issues.apache.org/jira/browse/YARN-8777) | Container Executor C binary change to execute interactive docker command | Major | . | Zian Chen | Eric Yang | +| [YARN-5742](https://issues.apache.org/jira/browse/YARN-5742) | Serve aggregated logs of historical apps from timeline service | Critical | timelineserver | Varun Saxena | Rohith Sharma K S | +| [YARN-8834](https://issues.apache.org/jira/browse/YARN-8834) | Provide Java client for fetching Yarn specific entities from TimelineReader | Critical | timelinereader | Rohith Sharma K S | Abhishek Modi | +| [YARN-3879](https://issues.apache.org/jira/browse/YARN-3879) | [Storage implementation] Create HDFS backing storage implementation for ATS reads | Major | timelineserver | Tsuyoshi Ozawa | Abhishek Modi | +| [HDFS-13523](https://issues.apache.org/jira/browse/HDFS-13523) | Support observer nodes in MiniDFSCluster | Major | namenode, test | Erik Krogen | Konstantin Shvachko | +| [HDFS-13906](https://issues.apache.org/jira/browse/HDFS-13906) | RBF: Add multiple paths for dfsrouteradmin "rm" and "clrquota" commands | Major | federation | Soumyapn | Ayush Saxena | +| [HADOOP-15826](https://issues.apache.org/jira/browse/HADOOP-15826) | @Retries annotation of putObject() call & uses wrong | Minor | fs/s3 | Steve Loughran | Steve Loughran | +| [YARN-8448](https://issues.apache.org/jira/browse/YARN-8448) | AM HTTPS Support for AM communication with RMWeb proxy | Major | . | Robert Kanter | Robert Kanter | +| [HDFS-13925](https://issues.apache.org/jira/browse/HDFS-13925) | Unit Test for transitioning between different states | Major | test | Sherwood Zheng | Sherwood Zheng | +| [YARN-8582](https://issues.apache.org/jira/browse/YARN-8582) | Document YARN support for HTTPS in AM Web server | Major | docs | Robert Kanter | Robert Kanter | +| [YARN-8449](https://issues.apache.org/jira/browse/YARN-8449) | RM HA for AM web server HTTPS Support | Major | . | Robert Kanter | Robert Kanter | +| [YARN-8873](https://issues.apache.org/jira/browse/YARN-8873) | [YARN-8811] Add CSI java-based client library | Major | . | Weiwei Yang | Weiwei Yang | +| [HDFS-14011](https://issues.apache.org/jira/browse/HDFS-14011) | RBF: Add more information to HdfsFileStatus for a mount point | Major | . | Akira Ajisaka | Akira Ajisaka | +| [HDFS-13566](https://issues.apache.org/jira/browse/HDFS-13566) | Add configurable additional RPC listener to NameNode | Major | ipc | Chen Liang | Chen Liang | +| [HDFS-13924](https://issues.apache.org/jira/browse/HDFS-13924) | Handle BlockMissingException when reading from observer | Major | . | Chao Sun | Chao Sun | +| [HADOOP-14775](https://issues.apache.org/jira/browse/HADOOP-14775) | Change junit dependency in parent pom file to junit 5 while maintaining backward compatibility to junit4. | Major | build | Ajay Kumar | Akira Ajisaka | +| [HADOOP-15823](https://issues.apache.org/jira/browse/HADOOP-15823) | ABFS: Stop requiring client ID and tenant ID for MSI | Major | . | Sean Mackrory | Da Zhou | +| [HADOOP-15868](https://issues.apache.org/jira/browse/HADOOP-15868) | AliyunOSS: update document for properties of multiple part download, multiple part upload and directory copy | Major | fs/oss | wujinhu | wujinhu | +| [YARN-8569](https://issues.apache.org/jira/browse/YARN-8569) | Create an interface to provide cluster information to application | Major | . | Eric Yang | Eric Yang | +| [HDFS-13845](https://issues.apache.org/jira/browse/HDFS-13845) | RBF: The default MountTableResolver should fail resolving multi-destination paths | Major | federation, hdfs | yanghuafeng | yanghuafeng | +| [YARN-8871](https://issues.apache.org/jira/browse/YARN-8871) | Document behavior of YARN-5742 | Major | . | Vrushali C | Suma Shivaprasad | +| [YARN-7754](https://issues.apache.org/jira/browse/YARN-7754) | [Atsv2] Update document for running v1 and v2 TS | Major | . | Rohith Sharma K S | Suma Shivaprasad | +| [HDFS-13942](https://issues.apache.org/jira/browse/HDFS-13942) | [JDK10] Fix javadoc errors in hadoop-hdfs module | Major | documentation | Akira Ajisaka | Dinesh Chitlangia | +| [HDFS-14033](https://issues.apache.org/jira/browse/HDFS-14033) | [libhdfs++] Disable libhdfs++ build on systems that do not support thread\_local | Major | . | Anatoli Shein | Anatoli Shein | +| [HDFS-14016](https://issues.apache.org/jira/browse/HDFS-14016) | ObserverReadProxyProvider should enable observer read by default | Major | . | Chen Liang | Chen Liang | +| [HDFS-12026](https://issues.apache.org/jira/browse/HDFS-12026) | libhdfs++: Fix compilation errors and warnings when compiling with Clang | Blocker | hdfs-client | Anatoli Shein | Anatoli Shein | +| [HADOOP-15895](https://issues.apache.org/jira/browse/HADOOP-15895) | [JDK9+] Add missing javax.annotation-api dependency to hadoop-yarn-csi | Major | build | Takanobu Asanuma | Takanobu Asanuma | +| [HDFS-14024](https://issues.apache.org/jira/browse/HDFS-14024) | RBF: ProvidedCapacityTotal json exception in NamenodeHeartbeatService | Major | . | CR Hota | CR Hota | +| [YARN-8893](https://issues.apache.org/jira/browse/YARN-8893) | [AMRMProxy] Fix thread leak in AMRMClientRelayer and UAM client | Major | amrmproxy, federation | Botong Huang | Botong Huang | +| [HDFS-13834](https://issues.apache.org/jira/browse/HDFS-13834) | RBF: Connection creator thread should catch Throwable | Critical | . | CR Hota | CR Hota | +| [YARN-8905](https://issues.apache.org/jira/browse/YARN-8905) | [Router] Add JvmMetricsInfo and pause monitor | Minor | . | Bibin Chundatt | Bilwa S T | +| [HADOOP-15904](https://issues.apache.org/jira/browse/HADOOP-15904) | [JDK 11] Javadoc build failed due to "bad use of '\>'" in hadoop-hdfs | Major | build | Takanobu Asanuma | Takanobu Asanuma | +| [HADOOP-15902](https://issues.apache.org/jira/browse/HADOOP-15902) | [JDK 11] Specify the HTML version of Javadoc to 4.01 | Major | . | Takanobu Asanuma | Takanobu Asanuma | +| [HDFS-14047](https://issues.apache.org/jira/browse/HDFS-14047) | [libhdfs++] Fix hdfsGetLastExceptionRootCause bug in test\_libhdfs\_threaded.c | Major | libhdfs, native | Anatoli Shein | Anatoli Shein | +| [HDFS-12284](https://issues.apache.org/jira/browse/HDFS-12284) | RBF: Support for Kerberos authentication | Major | security | Zhe Zhang | Sherwood Zheng | +| [YARN-8880](https://issues.apache.org/jira/browse/YARN-8880) | Phase 1 - Add configurations for pluggable plugin framework | Major | . | Zhankun Tang | Zhankun Tang | +| [YARN-8988](https://issues.apache.org/jira/browse/YARN-8988) | Reduce the verbose log on RM heartbeat path when distributed node-attributes is enabled | Major | . | Weiwei Yang | Tao Yang | +| [YARN-8902](https://issues.apache.org/jira/browse/YARN-8902) | [CSI] Add volume manager that manages CSI volume lifecycle | Major | . | Weiwei Yang | Weiwei Yang | +| [YARN-8987](https://issues.apache.org/jira/browse/YARN-8987) | Usability improvements node-attributes CLI | Critical | . | Weiwei Yang | Bibin Chundatt | +| [YARN-8877](https://issues.apache.org/jira/browse/YARN-8877) | [CSI] Extend service spec to allow setting resource attributes | Major | . | Weiwei Yang | Weiwei Yang | +| [YARN-8776](https://issues.apache.org/jira/browse/YARN-8776) | Container Executor change to create stdin/stdout pipeline | Major | . | Zian Chen | Eric Yang | +| [HDFS-13852](https://issues.apache.org/jira/browse/HDFS-13852) | RBF: The DN\_REPORT\_TIME\_OUT and DN\_REPORT\_CACHE\_EXPIRE should be configured in RBFConfigKeys. | Major | federation, hdfs | yanghuafeng | yanghuafeng | +| [HADOOP-15917](https://issues.apache.org/jira/browse/HADOOP-15917) | AliyunOSS: fix incorrect ReadOps and WriteOps in statistics | Major | fs/oss | wujinhu | wujinhu | +| [HDFS-14035](https://issues.apache.org/jira/browse/HDFS-14035) | NN status discovery does not leverage delegation token | Major | . | Chen Liang | Chen Liang | +| [HADOOP-15936](https://issues.apache.org/jira/browse/HADOOP-15936) | [JDK 11] MiniDFSClusterManager & MiniHadoopClusterManager compilation fails due to the usage of '\_' as identifier | Major | build | Devaraj Kavali | Zsolt Venczel | +| [YARN-8303](https://issues.apache.org/jira/browse/YARN-8303) | YarnClient should contact TimelineReader for application/attempt/container report | Critical | . | Rohith Sharma K S | Abhishek Modi | +| [HDFS-14017](https://issues.apache.org/jira/browse/HDFS-14017) | ObserverReadProxyProviderWithIPFailover should work with HA configuration | Major | . | Chen Liang | Chen Liang | +| [YARN-8881](https://issues.apache.org/jira/browse/YARN-8881) | [YARN-8851] Add basic pluggable device plugin framework | Major | . | Zhankun Tang | Zhankun Tang | +| [YARN-8778](https://issues.apache.org/jira/browse/YARN-8778) | Add Command Line interface to invoke interactive docker shell | Major | . | Zian Chen | Eric Yang | +| [YARN-8953](https://issues.apache.org/jira/browse/YARN-8953) | [CSI] CSI driver adaptor module support in NodeManager | Major | . | Weiwei Yang | Weiwei Yang | +| [YARN-8838](https://issues.apache.org/jira/browse/YARN-8838) | Add security check for container user is same as websocket user | Major | nodemanager | Eric Yang | Eric Yang | +| [HDFS-13776](https://issues.apache.org/jira/browse/HDFS-13776) | RBF: Add Storage policies related ClientProtocol APIs | Major | . | Dibyendu Karmakar | Dibyendu Karmakar | +| [HDFS-14089](https://issues.apache.org/jira/browse/HDFS-14089) | RBF: Failed to specify server's Kerberos pricipal name in NamenodeHeartbeatService | Minor | . | Ranith Sardar | Ranith Sardar | +| [HDFS-14067](https://issues.apache.org/jira/browse/HDFS-14067) | Allow manual failover between standby and observer | Major | . | Chao Sun | Chao Sun | +| [HDFS-14094](https://issues.apache.org/jira/browse/HDFS-14094) | Fix the order of logging arguments in ObserverReadProxyProvider. | Major | logging | Konstantin Shvachko | Ayush Saxena | +| [YARN-9054](https://issues.apache.org/jira/browse/YARN-9054) | Fix FederationStateStoreFacade#buildGetSubClustersCacheRequest | Major | federation | Bibin Chundatt | Bibin Chundatt | +| [YARN-9058](https://issues.apache.org/jira/browse/YARN-9058) | [CSI] YARN service fail to launch due to CSI changes | Blocker | . | Eric Yang | Weiwei Yang | +| [YARN-8986](https://issues.apache.org/jira/browse/YARN-8986) | publish all exposed ports to random ports when using bridge network | Minor | yarn | dockerzhang | dockerzhang | +| [YARN-9034](https://issues.apache.org/jira/browse/YARN-9034) | ApplicationCLI should have option to take clusterId | Major | . | Rohith Sharma K S | Rohith Sharma K S | +| [HADOOP-15798](https://issues.apache.org/jira/browse/HADOOP-15798) | LocalMetadataStore put() does not retain isDeleted in parent listing | Major | fs/s3 | Gabor Bota | Gabor Bota | +| [HADOOP-15370](https://issues.apache.org/jira/browse/HADOOP-15370) | S3A log message on rm s3a://bucket/ not intuitive | Trivial | fs/s3 | Steve Loughran | Gabor Bota | +| [YARN-8882](https://issues.apache.org/jira/browse/YARN-8882) | [YARN-8851] Add a shared device mapping manager (scheduler) for device plugins | Major | . | Zhankun Tang | Zhankun Tang | +| [YARN-8989](https://issues.apache.org/jira/browse/YARN-8989) | Move DockerCommandPlugin volume related APIs' invocation from DockerLinuxContainerRuntime#prepareContainer to #launchContainer | Major | . | Zhankun Tang | Zhankun Tang | +| [HDFS-13713](https://issues.apache.org/jira/browse/HDFS-13713) | Add specification of Multipart Upload API to FS specification, with contract tests | Blocker | fs, test | Steve Loughran | Ewan Higgs | +| [HADOOP-14927](https://issues.apache.org/jira/browse/HADOOP-14927) | ITestS3GuardTool failures in testDestroyNoBucket() | Minor | fs/s3 | Aaron Fabbri | Gabor Bota | +| [HDFS-13547](https://issues.apache.org/jira/browse/HDFS-13547) | Add ingress port based sasl resolver | Major | security | Chen Liang | Chen Liang | +| [HDFS-14120](https://issues.apache.org/jira/browse/HDFS-14120) | ORFPP should also clone DT for the virtual IP | Major | . | Chen Liang | Chen Liang | +| [HDFS-14085](https://issues.apache.org/jira/browse/HDFS-14085) | RBF: LS command for root shows wrong owner and permission information. | Major | . | Ayush Saxena | Ayush Saxena | +| [HDFS-14114](https://issues.apache.org/jira/browse/HDFS-14114) | RBF: MIN\_ACTIVE\_RATIO should be configurable | Major | . | Fei Hui | Fei Hui | +| [YARN-9057](https://issues.apache.org/jira/browse/YARN-9057) | [CSI] CSI jar file should not bundle third party dependencies | Blocker | build | Eric Yang | Weiwei Yang | +| [YARN-8914](https://issues.apache.org/jira/browse/YARN-8914) | Add xtermjs to YARN UI2 | Major | yarn-ui-v2 | Eric Yang | Eric Yang | +| [HDFS-14131](https://issues.apache.org/jira/browse/HDFS-14131) | Create user guide for "Consistent reads from Observer" feature. | Major | documentation | Konstantin Shvachko | Chao Sun | +| [HADOOP-15428](https://issues.apache.org/jira/browse/HADOOP-15428) | s3guard bucket-info will create s3guard table if FS is set to do this automatically | Major | fs/s3 | Steve Loughran | Gabor Bota | +| [HADOOP-15988](https://issues.apache.org/jira/browse/HADOOP-15988) | Should be able to set empty directory flag to TRUE in DynamoDBMetadataStore#innerGet when using authoritative directory listings | Major | fs/s3 | Gabor Bota | Gabor Bota | +| [HDFS-14142](https://issues.apache.org/jira/browse/HDFS-14142) | Move ipfailover config key out of HdfsClientConfigKeys | Minor | . | Chen Liang | Chen Liang | +| [YARN-8885](https://issues.apache.org/jira/browse/YARN-8885) | [DevicePlugin] Support NM APIs to query device resource allocation | Major | . | Zhankun Tang | Zhankun Tang | +| [YARN-9015](https://issues.apache.org/jira/browse/YARN-9015) | [DevicePlugin] Add an interface for device plugin to provide customized scheduler | Major | . | Zhankun Tang | Zhankun Tang | +| [YARN-8962](https://issues.apache.org/jira/browse/YARN-8962) | Add ability to use interactive shell with normal yarn container | Major | . | Eric Yang | Eric Yang | +| [HDFS-13873](https://issues.apache.org/jira/browse/HDFS-13873) | ObserverNode should reject read requests when it is too far behind. | Major | hdfs-client, namenode | Konstantin Shvachko | Konstantin Shvachko | +| [HDFS-14138](https://issues.apache.org/jira/browse/HDFS-14138) | Description errors in the comparison logic of transaction ID | Minor | . | xiangheng | xiangheng | +| [HDFS-14146](https://issues.apache.org/jira/browse/HDFS-14146) | Handle exception from internalQueueCall | Critical | ipc | Chao Sun | Chao Sun | +| [YARN-9032](https://issues.apache.org/jira/browse/YARN-9032) | Support sh shell for interactive container shell at command line | Blocker | yarn | Eric Yang | Eric Yang | +| [YARN-9089](https://issues.apache.org/jira/browse/YARN-9089) | Add Terminal Link to Service component instance page for UI2 | Major | yarn-ui-v2 | Eric Yang | Eric Yang | +| [YARN-8963](https://issues.apache.org/jira/browse/YARN-8963) | Add flag to disable interactive shell | Major | . | Eric Yang | Eric Yang | +| [YARN-9091](https://issues.apache.org/jira/browse/YARN-9091) | Improve terminal message when connection is refused | Major | . | Eric Yang | Eric Yang | +| [HDFS-14152](https://issues.apache.org/jira/browse/HDFS-14152) | RBF: Fix a typo in RouterAdmin usage | Major | . | Takanobu Asanuma | Ayush Saxena | +| [HDFS-13869](https://issues.apache.org/jira/browse/HDFS-13869) | RBF: Handle NPE for NamenodeBeanMetrics#getFederationMetrics | Major | namenode | Surendra Singh Lilhore | Ranith Sardar | +| [HDFS-14096](https://issues.apache.org/jira/browse/HDFS-14096) | [SPS] : Add Support for Storage Policy Satisfier in ViewFs | Major | federation | Ayush Saxena | Ayush Saxena | +| [HADOOP-15969](https://issues.apache.org/jira/browse/HADOOP-15969) | ABFS: getNamespaceEnabled can fail blocking user access thru ACLs | Major | fs/azure | Da Zhou | Da Zhou | +| [HADOOP-15972](https://issues.apache.org/jira/browse/HADOOP-15972) | ABFS: reduce list page size to to 500 | Major | fs/azure | Da Zhou | Da Zhou | +| [HADOOP-16004](https://issues.apache.org/jira/browse/HADOOP-16004) | ABFS: Convert 404 error response in AbfsInputStream and AbfsOutPutStream to FileNotFoundException | Major | fs/azure | Da Zhou | Da Zhou | +| [YARN-9125](https://issues.apache.org/jira/browse/YARN-9125) | Carriage Return character in launch command cause node manager to become unhealthy | Major | . | Eric Yang | Billie Rinaldi | +| [HDFS-14116](https://issues.apache.org/jira/browse/HDFS-14116) | Fix class cast error in NNThroughputBenchmark with ObserverReadProxyProvider. | Major | hdfs-client | Chen Liang | Chao Sun | +| [HDFS-14149](https://issues.apache.org/jira/browse/HDFS-14149) | Adjust annotations on new interfaces/classes for SBN reads. | Major | . | Konstantin Shvachko | Chao Sun | +| [HDFS-14151](https://issues.apache.org/jira/browse/HDFS-14151) | RBF: Make the read-only column of Mount Table clearly understandable | Major | . | Takanobu Asanuma | Takanobu Asanuma | +| [YARN-9072](https://issues.apache.org/jira/browse/YARN-9072) | Web browser close without proper exit can leak shell process | Major | . | Eric Yang | Eric Yang | +| [YARN-9117](https://issues.apache.org/jira/browse/YARN-9117) | Container shell does not work when using yarn.nodemanager.linux-container-executor.nonsecure-mode.local-user is set | Major | . | Eric Yang | Eric Yang | +| [YARN-9075](https://issues.apache.org/jira/browse/YARN-9075) | Dynamically add or remove auxiliary services | Major | nodemanager | Billie Rinaldi | Billie Rinaldi | +| [HDFS-13443](https://issues.apache.org/jira/browse/HDFS-13443) | RBF: Update mount table cache immediately after changing (add/update/remove) mount table entries. | Major | fs | Mohammad Arshad | Mohammad Arshad | +| [YARN-9126](https://issues.apache.org/jira/browse/YARN-9126) | Container reinit always fails in branch-3.2 and trunk | Major | . | Eric Yang | Chandni Singh | +| [HDFS-14160](https://issues.apache.org/jira/browse/HDFS-14160) | ObserverReadInvocationHandler should implement RpcInvocationHandler | Major | . | Konstantin Shvachko | Konstantin Shvachko | +| [YARN-9129](https://issues.apache.org/jira/browse/YARN-9129) | Ensure flush after printing to log plus additional cleanup | Major | . | Billie Rinaldi | Eric Yang | +| [YARN-8523](https://issues.apache.org/jira/browse/YARN-8523) | Interactive docker shell | Major | . | Eric Yang | Zian Chen | +| [HADOOP-15935](https://issues.apache.org/jira/browse/HADOOP-15935) | [JDK 11] Update maven.plugin-tools.version to 3.6.0 | Major | build | Devaraj Kavali | Dinesh Chitlangia | +| [HDFS-14154](https://issues.apache.org/jira/browse/HDFS-14154) | Document dfs.ha.tail-edits.period in user guide. | Major | documentation | Chao Sun | Chao Sun | +| [HADOOP-16015](https://issues.apache.org/jira/browse/HADOOP-16015) | Add bouncycastle jars to hadoop-aws as test dependencies | Major | fs/s3, test | Steve Loughran | Steve Loughran | +| [YARN-9131](https://issues.apache.org/jira/browse/YARN-9131) | Document usage of Dynamic auxiliary services | Major | . | Eric Yang | Billie Rinaldi | +| [YARN-9152](https://issues.apache.org/jira/browse/YARN-9152) | Auxiliary service REST API query does not return running services | Major | . | Eric Yang | Billie Rinaldi | +| [YARN-8925](https://issues.apache.org/jira/browse/YARN-8925) | Updating distributed node attributes only when necessary | Major | resourcemanager | Tao Yang | Tao Yang | +| [YARN-5168](https://issues.apache.org/jira/browse/YARN-5168) | Add port mapping handling when docker container use bridge network | Major | . | Jun Gong | Xun Liu | +| [HDFS-14170](https://issues.apache.org/jira/browse/HDFS-14170) | Fix white spaces related to SBN reads. | Major | . | Konstantin Shvachko | Konstantin Shvachko | +| [HADOOP-16009](https://issues.apache.org/jira/browse/HADOOP-16009) | Replace the url of the repository in Apache Hadoop source code | Major | documentation | Akira Ajisaka | Akira Ajisaka | +| [HADOOP-15860](https://issues.apache.org/jira/browse/HADOOP-15860) | ABFS: Throw IllegalArgumentException when Directory/File name ends with a period(.) | Major | fs/azure | Sean Mackrory | Shweta | +| [HDFS-14167](https://issues.apache.org/jira/browse/HDFS-14167) | RBF: Add stale nodes to federation metrics | Major | . | Íñigo Goiri | Íñigo Goiri | +| [HDFS-14161](https://issues.apache.org/jira/browse/HDFS-14161) | RBF: Throw StandbyException instead of IOException so that client can retry when can not get connection | Major | . | Fei Hui | Fei Hui | +| [HADOOP-15323](https://issues.apache.org/jira/browse/HADOOP-15323) | AliyunOSS: Improve copy file performance for AliyunOSSFileSystemStore | Major | fs/oss | wujinhu | wujinhu | +| [YARN-9147](https://issues.apache.org/jira/browse/YARN-9147) | Auxiliary manifest file deleted from HDFS does not trigger service to be removed | Major | . | Eric Yang | Billie Rinaldi | +| [YARN-9038](https://issues.apache.org/jira/browse/YARN-9038) | [CSI] Add ability to publish/unpublish volumes on node managers | Major | . | Weiwei Yang | Weiwei Yang | +| [YARN-6149](https://issues.apache.org/jira/browse/YARN-6149) | Allow port range to be specified while starting NM Timeline collector manager. | Major | timelineserver | Varun Saxena | Abhishek Modi | +| [YARN-9166](https://issues.apache.org/jira/browse/YARN-9166) | Fix logging for preemption of Opportunistic containers for Guaranteed containers. | Minor | . | Abhishek Modi | Abhishek Modi | +| [HADOOP-15937](https://issues.apache.org/jira/browse/HADOOP-15937) | [JDK 11] Update maven-shade-plugin.version to 3.2.1 | Major | build | Devaraj Kavali | Dinesh Chitlangia | +| [YARN-9169](https://issues.apache.org/jira/browse/YARN-9169) | Add metrics for queued opportunistic and guaranteed containers. | Major | . | Abhishek Modi | Abhishek Modi | +| [YARN-8822](https://issues.apache.org/jira/browse/YARN-8822) | Nvidia-docker v2 support for YARN GPU feature | Critical | . | Zhankun Tang | dockerzhang | +| [YARN-9037](https://issues.apache.org/jira/browse/YARN-9037) | [CSI] Ignore volume resource in resource calculators based on tags | Major | . | Weiwei Yang | Sunil G | +| [HDFS-14150](https://issues.apache.org/jira/browse/HDFS-14150) | RBF: Quotas of the sub-cluster should be removed when removing the mount point | Major | . | Takanobu Asanuma | Takanobu Asanuma | +| [HDFS-14191](https://issues.apache.org/jira/browse/HDFS-14191) | RBF: Remove hard coded router status from FederationMetrics. | Major | . | Ranith Sardar | Ranith Sardar | +| [HADOOP-16027](https://issues.apache.org/jira/browse/HADOOP-16027) | [DOC] Effective use of FS instances during S3A integration tests | Major | fs/s3 | Gabor Bota | Gabor Bota | +| [HDFS-13856](https://issues.apache.org/jira/browse/HDFS-13856) | RBF: RouterAdmin should support dfsrouteradmin -refreshRouterArgs command | Major | federation, hdfs | yanghuafeng | yanghuafeng | +| [HADOOP-16045](https://issues.apache.org/jira/browse/HADOOP-16045) | Don't run TestDU on Windows | Trivial | common, test | Lukas Majercak | Lukas Majercak | +| [HADOOP-14556](https://issues.apache.org/jira/browse/HADOOP-14556) | S3A to support Delegation Tokens | Major | fs/s3 | Steve Loughran | Steve Loughran | +| [HDFS-14206](https://issues.apache.org/jira/browse/HDFS-14206) | RBF: Cleanup quota modules | Major | . | Íñigo Goiri | Íñigo Goiri | +| [HDFS-14129](https://issues.apache.org/jira/browse/HDFS-14129) | RBF: Create new policy provider for router | Major | namenode | Surendra Singh Lilhore | Ranith Sardar | +| [HADOOP-15941](https://issues.apache.org/jira/browse/HADOOP-15941) | [JDK 11] Compilation failure: package com.sun.jndi.ldap is not visible | Major | common | Uma Maheswara Rao G | Takanobu Asanuma | +| [HDFS-14193](https://issues.apache.org/jira/browse/HDFS-14193) | RBF: Inconsistency with the Default Namespace | Major | . | Ayush Saxena | Ayush Saxena | +| [HDFS-14156](https://issues.apache.org/jira/browse/HDFS-14156) | RBF: rollEdit() command fails with Router | Major | . | Harshakiran Reddy | Shubham Dewan | +| [HADOOP-16046](https://issues.apache.org/jira/browse/HADOOP-16046) | [JDK 11] Correct the compiler exclusion of org/apache/hadoop/yarn/webapp/hamlet/\*\* classes for \>= Java 9 | Major | build | Devaraj Kavali | Devaraj Kavali | +| [HADOOP-15787](https://issues.apache.org/jira/browse/HADOOP-15787) | [JDK11] TestIPC.testRTEDuringConnectionSetup fails | Major | . | Akira Ajisaka | Zsolt Venczel | +| [YARN-9146](https://issues.apache.org/jira/browse/YARN-9146) | REST API to trigger storing auxiliary manifest file and publish to NMs | Major | . | Eric Yang | Billie Rinaldi | +| [YARN-8101](https://issues.apache.org/jira/browse/YARN-8101) | Add UT to verify node-attributes in RM nodes rest API | Minor | resourcemanager, restapi | Weiwei Yang | Prabhu Joseph | +| [HDFS-14209](https://issues.apache.org/jira/browse/HDFS-14209) | RBF: setQuota() through router is working for only the mount Points under the Source column in MountTable | Major | . | Shubham Dewan | Shubham Dewan | +| [YARN-9116](https://issues.apache.org/jira/browse/YARN-9116) | Capacity Scheduler: implements queue level maximum-allocation inheritance | Major | capacity scheduler | Aihua Xu | Aihua Xu | +| [YARN-8867](https://issues.apache.org/jira/browse/YARN-8867) | Retrieve the status of resource localization | Major | yarn | Chandni Singh | Chandni Singh | +| [HDFS-14223](https://issues.apache.org/jira/browse/HDFS-14223) | RBF: Add configuration documents for using multiple sub-clusters | Major | documentation | Takanobu Asanuma | Takanobu Asanuma | +| [YARN-9221](https://issues.apache.org/jira/browse/YARN-9221) | Add a flag to enable dynamic auxiliary service feature | Major | . | Eric Yang | Billie Rinaldi | +| [HDFS-14224](https://issues.apache.org/jira/browse/HDFS-14224) | RBF: NPE in getContentSummary() for getEcPolicy() in case of multiple destinations | Major | . | Ayush Saxena | Ayush Saxena | +| [HDFS-14215](https://issues.apache.org/jira/browse/HDFS-14215) | RBF: Remove dependency on availability of default namespace | Major | . | Ayush Saxena | Ayush Saxena | +| [YARN-9074](https://issues.apache.org/jira/browse/YARN-9074) | Docker container rm command should be executed after stop | Major | . | Zhaohui Xin | Zhaohui Xin | +| [YARN-9086](https://issues.apache.org/jira/browse/YARN-9086) | [CSI] Run csi-driver-adaptor as aux service | Major | . | Weiwei Yang | Weiwei Yang | +| [HADOOP-14178](https://issues.apache.org/jira/browse/HADOOP-14178) | Move Mockito up to version 2.23.4 | Major | test | Steve Loughran | Akira Ajisaka | +| [HADOOP-16041](https://issues.apache.org/jira/browse/HADOOP-16041) | UserAgent string for ABFS | Major | fs/azure | Shweta | Shweta | +| [HADOOP-15938](https://issues.apache.org/jira/browse/HADOOP-15938) | [JDK 11] Remove animal-sniffer-maven-plugin to fix build | Major | build | Devaraj Kavali | Dinesh Chitlangia | +| [HDFS-14225](https://issues.apache.org/jira/browse/HDFS-14225) | RBF : MiniRouterDFSCluster should configure the failover proxy provider for namespace | Minor | federation | Surendra Singh Lilhore | Ranith Sardar | +| [YARN-9275](https://issues.apache.org/jira/browse/YARN-9275) | Add link to NodeAttributes doc in PlacementConstraints document | Minor | documentation | Weiwei Yang | Masatake Iwasaki | +| [YARN-6735](https://issues.apache.org/jira/browse/YARN-6735) | Have a way to turn off container metrics from NMs | Major | timelineserver | Vrushali C | Abhishek Modi | +| [HDFS-14252](https://issues.apache.org/jira/browse/HDFS-14252) | RBF : Exceptions are exposing the actual sub cluster path | Major | . | Ayush Saxena | Ayush Saxena | +| [HADOOP-15954](https://issues.apache.org/jira/browse/HADOOP-15954) | ABFS: Enable owner and group conversion for MSI and login user using OAuth | Major | fs/azure | junhua gu | Da Zhou | +| [YARN-9253](https://issues.apache.org/jira/browse/YARN-9253) | Add UT to verify Placement Constraint in Distributed Shell | Major | . | Prabhu Joseph | Prabhu Joseph | +| [YARN-9252](https://issues.apache.org/jira/browse/YARN-9252) | Allocation Tag Namespace support in Distributed Shell | Major | distributed-shell | Prabhu Joseph | Prabhu Joseph | +| [YARN-8555](https://issues.apache.org/jira/browse/YARN-8555) | Parameterize TestSchedulingRequestContainerAllocation(Async) to cover both PC handler options | Minor | . | Weiwei Yang | Prabhu Joseph | +| [YARN-996](https://issues.apache.org/jira/browse/YARN-996) | REST API support for node resource configuration | Major | graceful, nodemanager, scheduler | Junping Du | Íñigo Goiri | +| [YARN-9229](https://issues.apache.org/jira/browse/YARN-9229) | Document docker registry deployment with NFS Gateway | Major | . | Eric Yang | Eric Yang | +| [HADOOP-15364](https://issues.apache.org/jira/browse/HADOOP-15364) | Add support for S3 Select to S3A | Major | fs/s3 | Steve Loughran | Steve Loughran | +| [HDFS-14230](https://issues.apache.org/jira/browse/HDFS-14230) | RBF: Throw RetriableException instead of IOException when no namenodes available | Major | . | Fei Hui | Fei Hui | +| [YARN-9184](https://issues.apache.org/jira/browse/YARN-9184) | Docker run doesn't pull down latest image if the image exists locally | Major | nodemanager | Zhaohui Xin | Zhaohui Xin | +| [HDFS-13617](https://issues.apache.org/jira/browse/HDFS-13617) | Allow wrapping NN QOP into token in encrypted message | Major | . | Chen Liang | Chen Liang | +| [HDFS-13358](https://issues.apache.org/jira/browse/HDFS-13358) | RBF: Support for Delegation Token (RPC) | Major | . | Sherwood Zheng | CR Hota | +| [HDFS-14262](https://issues.apache.org/jira/browse/HDFS-14262) | [SBN read] Unclear Log.WARN message in GlobalStateIdContext | Major | hdfs | Shweta | Shweta | +| [YARN-9293](https://issues.apache.org/jira/browse/YARN-9293) | Optimize MockAMLauncher event handling | Major | . | Bibin Chundatt | Bibin Chundatt | +| [HDFS-14226](https://issues.apache.org/jira/browse/HDFS-14226) | RBF: Setting attributes should set on all subclusters' directories. | Major | . | Takanobu Asanuma | Ayush Saxena | +| [HDFS-14268](https://issues.apache.org/jira/browse/HDFS-14268) | RBF: Fix the location of the DNs in getDatanodeReport() | Major | . | Íñigo Goiri | Íñigo Goiri | +| [YARN-9060](https://issues.apache.org/jira/browse/YARN-9060) | [YARN-8851] Phase 1 - Support device isolation and use the Nvidia GPU plugin as an example | Major | . | Zhankun Tang | Zhankun Tang | +| [HADOOP-15843](https://issues.apache.org/jira/browse/HADOOP-15843) | s3guard bucket-info command to not print a stack trace on bucket-not-found | Minor | fs/s3 | Steve Loughran | Adam Antal | +| [HADOOP-16104](https://issues.apache.org/jira/browse/HADOOP-16104) | Wasb tests to downgrade to skip when test a/c is namespace enabled | Major | fs/azure, test | Steve Loughran | Masatake Iwasaki | +| [HDFS-14249](https://issues.apache.org/jira/browse/HDFS-14249) | RBF: Tooling to identify the subcluster location of a file | Major | . | Íñigo Goiri | Íñigo Goiri | +| [YARN-9258](https://issues.apache.org/jira/browse/YARN-9258) | Support to specify allocation tags without constraint in distributed shell CLI | Major | distributed-shell | Prabhu Joseph | Prabhu Joseph | +| [YARN-9156](https://issues.apache.org/jira/browse/YARN-9156) | [YARN-8851] Improve debug message in device plugin method compatibility check of ResourcePluginManager | Trivial | . | Zhankun Tang | Zhankun Tang | +| [YARN-8891](https://issues.apache.org/jira/browse/YARN-8891) | Documentation of the pluggable device framework | Major | documentation | Zhankun Tang | Zhankun Tang | +| [YARN-9244](https://issues.apache.org/jira/browse/YARN-9244) | Document docker registry deployment with direct S3 driver | Major | . | Eric Yang | Suma Shivaprasad | +| [YARN-8821](https://issues.apache.org/jira/browse/YARN-8821) | [YARN-8851] GPU hierarchy/topology scheduling support based on pluggable device framework | Major | . | Zhankun Tang | Zhankun Tang | +| [HDFS-14130](https://issues.apache.org/jira/browse/HDFS-14130) | Make ZKFC ObserverNode aware | Major | ha | Konstantin Shvachko | xiangheng | +| [YARN-9331](https://issues.apache.org/jira/browse/YARN-9331) | [YARN-8851] Fix a bug that lacking cgroup initialization when bootstrap DeviceResourceHandlerImpl | Major | . | Zhankun Tang | Zhankun Tang | +| [HADOOP-16093](https://issues.apache.org/jira/browse/HADOOP-16093) | Move DurationInfo from hadoop-aws to hadoop-common org.apache.hadoop.util | Minor | fs/s3, util | Steve Loughran | Abhishek Modi | +| [YARN-8783](https://issues.apache.org/jira/browse/YARN-8783) | Improve the documentation for the docker.trusted.registries configuration | Major | . | Simon Prewo | Eric Yang | +| [HADOOP-16136](https://issues.apache.org/jira/browse/HADOOP-16136) | ABFS: Should only transform username to short name | Major | . | Da Zhou | Da Zhou | +| [YARN-9245](https://issues.apache.org/jira/browse/YARN-9245) | Add support for Docker Images command | Major | yarn | Chandni Singh | Chandni Singh | +| [YARN-5336](https://issues.apache.org/jira/browse/YARN-5336) | Limit the flow name size & consider cleanup for hex chars | Major | timelineserver | Vrushali C | Sushil Ks | +| [YARN-3841](https://issues.apache.org/jira/browse/YARN-3841) | [Storage implementation] Adding retry semantics to HDFS backing storage | Major | timelineserver | Tsuyoshi Ozawa | Abhishek Modi | +| [HADOOP-16068](https://issues.apache.org/jira/browse/HADOOP-16068) | ABFS Authentication and Delegation Token plugins to optionally be bound to specific URI of the store | Major | fs/azure | Steve Loughran | Steve Loughran | +| [YARN-7904](https://issues.apache.org/jira/browse/YARN-7904) | Privileged, trusted containers should be supported only in ENTRYPOINT mode | Major | . | Eric Badger | Eric Yang | +| [HDFS-14259](https://issues.apache.org/jira/browse/HDFS-14259) | RBF: Fix safemode message for Router | Major | . | Íñigo Goiri | Ranith Sardar | +| [HDFS-14329](https://issues.apache.org/jira/browse/HDFS-14329) | RBF: Add maintenance nodes to federation metrics | Major | . | Ayush Saxena | Ayush Saxena | +| [YARN-7477](https://issues.apache.org/jira/browse/YARN-7477) | Moving logging APIs over to slf4j in hadoop-yarn-common | Major | . | Yeliang Cang | Prabhu Joseph | +| [HDFS-14331](https://issues.apache.org/jira/browse/HDFS-14331) | RBF: IOE While Removing Mount Entry | Major | rbf | Surendra Singh Lilhore | Ayush Saxena | +| [HDFS-14335](https://issues.apache.org/jira/browse/HDFS-14335) | RBF: Fix heartbeat typos in the Router. | Trivial | . | CR Hota | CR Hota | +| [YARN-7243](https://issues.apache.org/jira/browse/YARN-7243) | Moving logging APIs over to slf4j in hadoop-yarn-server-resourcemanager | Major | . | Yeliang Cang | Prabhu Joseph | +| [HDFS-7663](https://issues.apache.org/jira/browse/HDFS-7663) | Erasure Coding: Append on striped file | Major | . | Jing Zhao | Ayush Saxena | +| [HADOOP-16163](https://issues.apache.org/jira/browse/HADOOP-16163) | NPE in setup/teardown of ITestAbfsDelegationTokens | Major | fs/azure, test | Steve Loughran | Steve Loughran | +| [HDFS-14334](https://issues.apache.org/jira/browse/HDFS-14334) | RBF: Use human readable format for long numbers in the Router UI | Major | . | Íñigo Goiri | Íñigo Goiri | +| [YARN-9239](https://issues.apache.org/jira/browse/YARN-9239) | Document docker registry deployment with Ozone CSI driver | Major | . | Eric Yang | Eric Yang | +| [YARN-8549](https://issues.apache.org/jira/browse/YARN-8549) | Adding a NoOp timeline writer and reader plugin classes for ATSv2 | Minor | ATSv2, timelineclient, timelineserver | Prabha Manepalli | Prabha Manepalli | +| [YARN-9265](https://issues.apache.org/jira/browse/YARN-9265) | FPGA plugin fails to recognize Intel Processing Accelerator Card | Critical | . | Peter Bacsko | Peter Bacsko | +| [YARN-8643](https://issues.apache.org/jira/browse/YARN-8643) | Docker image life cycle management on HDFS | Major | yarn | Eric Yang | Eric Yang | +| [HDFS-14343](https://issues.apache.org/jira/browse/HDFS-14343) | RBF: Fix renaming folders spread across multiple subclusters | Major | . | Íñigo Goiri | Ayush Saxena | +| [YARN-8805](https://issues.apache.org/jira/browse/YARN-8805) | Automatically convert the launch command to the exec form when using entrypoint support | Major | . | Shane Kumpf | Eric Yang | +| [HDFS-14270](https://issues.apache.org/jira/browse/HDFS-14270) | [SBN Read] StateId and TrasactionId not present in Trace level logging | Trivial | namenode | Shweta | Shweta | +| [YARN-9266](https://issues.apache.org/jira/browse/YARN-9266) | General improvements in IntelFpgaOpenclPlugin | Major | . | Peter Bacsko | Peter Bacsko | +| [HADOOP-16109](https://issues.apache.org/jira/browse/HADOOP-16109) | Parquet reading S3AFileSystem causes EOF | Blocker | fs/s3 | Dave Christianson | Steve Loughran | +| [YARN-8376](https://issues.apache.org/jira/browse/YARN-8376) | Separate white list for docker.trusted.registries and docker.privileged-container.registries | Major | . | Eric Yang | Eric Yang | +| [HADOOP-15625](https://issues.apache.org/jira/browse/HADOOP-15625) | S3A input stream to use etags/version number to detect changed source files | Major | fs/s3 | Brahma Reddy Battula | Ben Roling | +| [HDFS-14354](https://issues.apache.org/jira/browse/HDFS-14354) | Refactor MappableBlock to align with the implementation of SCM cache | Major | caching, datanode | Feilong He | Feilong He | +| [YARN-9343](https://issues.apache.org/jira/browse/YARN-9343) | Replace isDebugEnabled with SLF4J parameterized log messages | Major | . | Prabhu Joseph | Prabhu Joseph | +| [HADOOP-16182](https://issues.apache.org/jira/browse/HADOOP-16182) | Update abfs storage back-end with "close" flag when application is done writing to a file | Major | fs/azure | Vishwajeet Dusane | Vishwajeet Dusane | +| [YARN-9363](https://issues.apache.org/jira/browse/YARN-9363) | Replace isDebugEnabled with SLF4J parameterized log messages for remaining code | Minor | yarn | Prabhu Joseph | Prabhu Joseph | +| [HADOOP-16124](https://issues.apache.org/jira/browse/HADOOP-16124) | Extend documentation in testing.md about endpoint constants | Trivial | fs/s3 | Adam Antal | Adam Antal | +| [YARN-9364](https://issues.apache.org/jira/browse/YARN-9364) | Remove commons-logging dependency from remaining hadoop-yarn | Major | . | Prabhu Joseph | Prabhu Joseph | +| [HADOOP-16191](https://issues.apache.org/jira/browse/HADOOP-16191) | AliyunOSS: improvements for copyFile/copyDirectory and logging | Major | fs/oss | wujinhu | wujinhu | +| [HDFS-14351](https://issues.apache.org/jira/browse/HDFS-14351) | RBF: Optimize configuration item resolving for monitor namenode | Major | rbf | Xiaoqiao He | Xiaoqiao He | +| [YARN-9387](https://issues.apache.org/jira/browse/YARN-9387) | Update document for ATS HBase Custom tablenames (-entityTableName) | Critical | ATSv2 | Prabhu Joseph | Prabhu Joseph | +| [YARN-9389](https://issues.apache.org/jira/browse/YARN-9389) | FlowActivity and FlowRun table prefix is wrong | Minor | ATSv2 | Prabhu Joseph | Prabhu Joseph | +| [YARN-9398](https://issues.apache.org/jira/browse/YARN-9398) | Javadoc error on FPGA related java files | Major | . | Eric Yang | Peter Bacsko | +| [YARN-9267](https://issues.apache.org/jira/browse/YARN-9267) | General improvements in FpgaResourceHandlerImpl | Major | . | Peter Bacsko | Peter Bacsko | +| [YARN-9402](https://issues.apache.org/jira/browse/YARN-9402) | Opportunistic containers should not be scheduled on Decommissioning nodes. | Major | . | Abhishek Modi | Abhishek Modi | +| [HADOOP-16201](https://issues.apache.org/jira/browse/HADOOP-16201) | S3AFileSystem#innerMkdirs builds needless lists | Trivial | fs/s3 | Lokesh Jain | Lokesh Jain | +| [HDFS-14388](https://issues.apache.org/jira/browse/HDFS-14388) | RBF: Prevent loading metric system when disabled | Major | . | Íñigo Goiri | Íñigo Goiri | +| [YARN-9391](https://issues.apache.org/jira/browse/YARN-9391) | Disable PATH variable to be passed to Docker container | Major | . | Eric Yang | Jim Brennan | +| [YARN-9268](https://issues.apache.org/jira/browse/YARN-9268) | General improvements in FpgaDevice | Major | . | Peter Bacsko | Peter Bacsko | +| [YARN-9269](https://issues.apache.org/jira/browse/YARN-9269) | Minor cleanup in FpgaResourceAllocator | Minor | . | Peter Bacsko | Peter Bacsko | +| [HADOOP-16186](https://issues.apache.org/jira/browse/HADOOP-16186) | S3Guard: NPE in DynamoDBMetadataStore.lambda$listChildren | Major | fs/s3 | Steve Loughran | Gabor Bota | +| [HADOOP-15999](https://issues.apache.org/jira/browse/HADOOP-15999) | S3Guard: Better support for out-of-band operations | Major | fs/s3 | Sean Mackrory | Gabor Bota | +| [HDFS-14393](https://issues.apache.org/jira/browse/HDFS-14393) | Refactor FsDatasetCache for SCM cache implementation | Major | . | Rakesh Radhakrishnan | Rakesh Radhakrishnan | +| [HADOOP-16058](https://issues.apache.org/jira/browse/HADOOP-16058) | S3A tests to include Terasort | Major | fs/s3, test | Steve Loughran | Steve Loughran | +| [YARN-9270](https://issues.apache.org/jira/browse/YARN-9270) | Minor cleanup in TestFpgaDiscoverer | Minor | . | Peter Bacsko | Peter Bacsko | +| [YARN-7129](https://issues.apache.org/jira/browse/YARN-7129) | Application Catalog initial project setup and source | Major | applications | Eric Yang | Eric Yang | +| [YARN-9348](https://issues.apache.org/jira/browse/YARN-9348) | Build issues on hadoop-yarn-application-catalog-webapp | Major | . | Eric Yang | Eric Yang | +| [HDFS-14316](https://issues.apache.org/jira/browse/HDFS-14316) | RBF: Support unavailable subclusters for mount points with multiple destinations | Major | . | Íñigo Goiri | Íñigo Goiri | +| [HDFS-14355](https://issues.apache.org/jira/browse/HDFS-14355) | Implement HDFS cache on SCM by using pure java mapped byte buffer | Major | caching, datanode | Feilong He | Feilong He | +| [HADOOP-16220](https://issues.apache.org/jira/browse/HADOOP-16220) | Add findbugs ignores for unjustified issues during update to guava to 27.0-jre in hadoop-project | Major | . | Gabor Bota | Gabor Bota | +| [YARN-9255](https://issues.apache.org/jira/browse/YARN-9255) | Improve recommend applications order | Major | . | Eric Yang | Eric Yang | +| [YARN-9418](https://issues.apache.org/jira/browse/YARN-9418) | ATSV2 /apps/appId/entities/YARN\_CONTAINER rest api does not show metrics | Critical | ATSv2 | Prabhu Joseph | Prabhu Joseph | +| [HADOOP-16218](https://issues.apache.org/jira/browse/HADOOP-16218) | findbugs warning of null param to non-nullable method in Configuration with Guava update | Minor | build | Steve Loughran | Steve Loughran | +| [YARN-9303](https://issues.apache.org/jira/browse/YARN-9303) | Username splits won't help timelineservice.app\_flow table | Major | ATSv2 | Prabhu Joseph | Prabhu Joseph | +| [HADOOP-16210](https://issues.apache.org/jira/browse/HADOOP-16210) | Update guava to 27.0-jre in hadoop-project trunk | Critical | build | Gabor Bota | Gabor Bota | +| [YARN-9441](https://issues.apache.org/jira/browse/YARN-9441) | Component name should start with Apache Hadoop for consistency | Minor | . | Weiwei Yang | Weiwei Yang | +| [HADOOP-16197](https://issues.apache.org/jira/browse/HADOOP-16197) | S3AUtils.translateException to map CredentialInitializationException to AccessDeniedException | Major | . | Steve Loughran | Steve Loughran | +| [HDFS-13853](https://issues.apache.org/jira/browse/HDFS-13853) | RBF: RouterAdmin update cmd is overwriting the entry not updating the existing | Major | . | Dibyendu Karmakar | Ayush Saxena | +| [YARN-9382](https://issues.apache.org/jira/browse/YARN-9382) | Publish container killed, paused and resumed events to ATSv2. | Major | . | Abhishek Modi | Abhishek Modi | +| [YARN-9335](https://issues.apache.org/jira/browse/YARN-9335) | [atsv2] Restrict the number of elements held in timeline collector when backend is unreachable for async calls | Major | . | Vrushali C | Abhishek Modi | +| [YARN-9313](https://issues.apache.org/jira/browse/YARN-9313) | Support asynchronized scheduling mode and multi-node lookup mechanism for scheduler activities | Major | . | Tao Yang | Tao Yang | +| [HDFS-14369](https://issues.apache.org/jira/browse/HDFS-14369) | RBF: Fix trailing "/" for webhdfs | Major | . | CR Hota | Akira Ajisaka | +| [YARN-999](https://issues.apache.org/jira/browse/YARN-999) | In case of long running tasks, reduce node resource should balloon out resource quickly by calling preemption API and suspending running task. | Major | graceful, nodemanager, scheduler | Junping Du | Íñigo Goiri | +| [YARN-9435](https://issues.apache.org/jira/browse/YARN-9435) | Add Opportunistic Scheduler metrics in ResourceManager. | Major | . | Abhishek Modi | Abhishek Modi | +| [HADOOP-16195](https://issues.apache.org/jira/browse/HADOOP-16195) | S3A MarshalledCredentials.toString() doesn't print full date/time of expiry | Minor | fs/s3 | Steve Loughran | Steve Loughran | +| [HDFS-13699](https://issues.apache.org/jira/browse/HDFS-13699) | Add DFSClient sending handshake token to DataNode, and allow DataNode overwrite downstream QOP | Major | . | Chen Liang | Chen Liang | +| [HADOOP-14747](https://issues.apache.org/jira/browse/HADOOP-14747) | S3AInputStream to implement CanUnbuffer | Major | fs/s3 | Steve Loughran | Sahil Takiar | +| [HADOOP-16237](https://issues.apache.org/jira/browse/HADOOP-16237) | Fix new findbugs issues after update guava to 27.0-jre in hadoop-project trunk | Critical | . | Gabor Bota | Gabor Bota | +| [YARN-9281](https://issues.apache.org/jira/browse/YARN-9281) | Add express upgrade button to Appcatalog UI | Major | . | Eric Yang | Eric Yang | +| [YARN-9474](https://issues.apache.org/jira/browse/YARN-9474) | Remove hard coded sleep from Opportunistic Scheduler tests. | Major | . | Abhishek Modi | Abhishek Modi | +| [YARN-9439](https://issues.apache.org/jira/browse/YARN-9439) | Support asynchronized scheduling mode and multi-node lookup mechanism for app activities | Major | capacityscheduler | Tao Yang | Tao Yang | +| [YARN-7848](https://issues.apache.org/jira/browse/YARN-7848) | Force removal of docker containers that do not get removed on first try | Major | . | Eric Badger | Eric Yang | +| [YARN-8943](https://issues.apache.org/jira/browse/YARN-8943) | Upgrade JUnit from 4 to 5 in hadoop-yarn-api | Major | test | Akira Ajisaka | Akira Ajisaka | +| [HADOOP-16253](https://issues.apache.org/jira/browse/HADOOP-16253) | Update AssertJ to 3.12.2 | Major | test | Takanobu Asanuma | Takanobu Asanuma | +| [HDFS-14422](https://issues.apache.org/jira/browse/HDFS-14422) | RBF: Router shouldn't allow READ operations in safe mode | Major | . | Íñigo Goiri | Íñigo Goiri | +| [YARN-8530](https://issues.apache.org/jira/browse/YARN-8530) | Add security filters to Application catalog | Major | security, yarn-native-services | Eric Yang | Eric Yang | +| [YARN-9466](https://issues.apache.org/jira/browse/YARN-9466) | App catalog navigation stylesheet does not display correctly in Safari | Major | . | Eric Yang | Eric Yang | +| [HADOOP-16508](https://issues.apache.org/jira/browse/HADOOP-16508) | [hadoop-yarn-project] Fix order of actual and expected expression in assert statements | Major | . | Prabhu Joseph | Prabhu Joseph | +| [YARN-9448](https://issues.apache.org/jira/browse/YARN-9448) | Fix Opportunistic Scheduling for node local allocations. | Major | . | Abhishek Modi | Abhishek Modi | +| [YARN-9254](https://issues.apache.org/jira/browse/YARN-9254) | Externalize Solr data storage | Major | . | Eric Yang | Eric Yang | +| [YARN-2889](https://issues.apache.org/jira/browse/YARN-2889) | Limit the number of opportunistic container allocated per AM heartbeat | Major | nodemanager, resourcemanager | Konstantinos Karanasos | Abhishek Modi | +| [YARN-8551](https://issues.apache.org/jira/browse/YARN-8551) | Build Common module for MaWo application | Major | . | Yesha Vora | Yesha Vora | +| [YARN-9475](https://issues.apache.org/jira/browse/YARN-9475) | Create basic VE plugin | Major | nodemanager | Peter Bacsko | Peter Bacsko | +| [HADOOP-16252](https://issues.apache.org/jira/browse/HADOOP-16252) | Use configurable dynamo table name prefix in S3Guard tests | Major | fs/s3 | Ben Roling | Ben Roling | +| [HDFS-13972](https://issues.apache.org/jira/browse/HDFS-13972) | RBF: Support for Delegation Token (WebHDFS) | Major | . | Íñigo Goiri | CR Hota | +| [HADOOP-16222](https://issues.apache.org/jira/browse/HADOOP-16222) | Fix new deprecations after guava 27.0 update in trunk | Major | . | Gabor Bota | Gabor Bota | +| [HDFS-14457](https://issues.apache.org/jira/browse/HDFS-14457) | RBF: Add order text SPACE in CLI command 'hdfs dfsrouteradmin' | Major | rbf | luhuachao | luhuachao | +| [YARN-9486](https://issues.apache.org/jira/browse/YARN-9486) | Docker container exited with failure does not get clean up correctly | Major | . | Eric Yang | Eric Yang | +| [HADOOP-16242](https://issues.apache.org/jira/browse/HADOOP-16242) | ABFS: add bufferpool to AbfsOutputStream | Major | fs/azure | Da Zhou | Da Zhou | +| [YARN-9476](https://issues.apache.org/jira/browse/YARN-9476) | Create unit tests for VE plugin | Major | . | Peter Bacsko | Peter Bacsko | +| [HADOOP-16221](https://issues.apache.org/jira/browse/HADOOP-16221) | S3Guard: fail write that doesn't update metadata store | Major | fs/s3 | Ben Roling | Ben Roling | +| [HDFS-14454](https://issues.apache.org/jira/browse/HDFS-14454) | RBF: getContentSummary() should allow non-existing folders | Major | . | Íñigo Goiri | Íñigo Goiri | +| [YARN-9440](https://issues.apache.org/jira/browse/YARN-9440) | Improve diagnostics for scheduler and app activities | Major | capacityscheduler | Tao Yang | Tao Yang | +| [HADOOP-16270](https://issues.apache.org/jira/browse/HADOOP-16270) | [JDK 11] Remove unintentional override of the version of Maven Dependency Plugin | Major | build | Akira Ajisaka | Xieming Li | +| [HDFS-14401](https://issues.apache.org/jira/browse/HDFS-14401) | Refine the implementation for HDFS cache on SCM | Major | caching, datanode | Feilong He | Feilong He | +| [HADOOP-16269](https://issues.apache.org/jira/browse/HADOOP-16269) | ABFS: add listFileStatus with StartFrom | Major | fs/azure | Da Zhou | Da Zhou | +| [YARN-9489](https://issues.apache.org/jira/browse/YARN-9489) | Support filtering by request-priorities and allocation-request-ids for query results of app activities | Major | . | Tao Yang | Tao Yang | +| [YARN-9539](https://issues.apache.org/jira/browse/YARN-9539) | Improve cleanup process of app activities and make some conditions configurable | Major | capacityscheduler | Tao Yang | Tao Yang | +| [HDFS-14426](https://issues.apache.org/jira/browse/HDFS-14426) | RBF: Add delegation token total count as one of the federation metrics | Major | . | Fengnan Li | Fengnan Li | +| [HADOOP-16306](https://issues.apache.org/jira/browse/HADOOP-16306) | AliyunOSS: Remove temporary files when upload small files to OSS | Major | fs/oss | wujinhu | wujinhu | +| [HDFS-14210](https://issues.apache.org/jira/browse/HDFS-14210) | RBF: ACL commands should work over all the destinations | Major | . | Shubham Dewan | Ayush Saxena | +| [HDFS-14490](https://issues.apache.org/jira/browse/HDFS-14490) | RBF: Remove unnecessary quota checks | Major | . | Ayush Saxena | Ayush Saxena | +| [HADOOP-16050](https://issues.apache.org/jira/browse/HADOOP-16050) | S3A SSL connections should use OpenSSL | Major | fs/s3 | Justin Uang | Sahil Takiar | +| [HDFS-14447](https://issues.apache.org/jira/browse/HDFS-14447) | RBF: Router should support RefreshUserMappingsProtocol | Major | rbf | Shen Yinjie | Shen Yinjie | +| [YARN-9505](https://issues.apache.org/jira/browse/YARN-9505) | Add container allocation latency for Opportunistic Scheduler | Major | . | Abhishek Modi | Abhishek Modi | +| [HADOOP-16085](https://issues.apache.org/jira/browse/HADOOP-16085) | S3Guard: use object version or etags to protect against inconsistent read after replace/overwrite | Major | fs/s3 | Ben Roling | Ben Roling | +| [HDFS-13995](https://issues.apache.org/jira/browse/HDFS-13995) | RBF: Security documentation | Major | . | CR Hota | CR Hota | +| [HADOOP-16287](https://issues.apache.org/jira/browse/HADOOP-16287) | KerberosAuthenticationHandler Trusted Proxy Support for Knox | Major | auth | Prabhu Joseph | Prabhu Joseph | +| [HDFS-14440](https://issues.apache.org/jira/browse/HDFS-14440) | RBF: Optimize the file write process in case of multiple destinations. | Major | . | Ayush Saxena | Ayush Saxena | +| [HDFS-14402](https://issues.apache.org/jira/browse/HDFS-14402) | Use FileChannel.transferTo() method for transferring block to SCM cache | Major | caching, datanode | Feilong He | Feilong He | +| [YARN-9497](https://issues.apache.org/jira/browse/YARN-9497) | Support grouping by diagnostics for query results of scheduler and app activities | Major | . | Tao Yang | Tao Yang | +| [HDFS-13255](https://issues.apache.org/jira/browse/HDFS-13255) | RBF: Fail when try to remove mount point paths | Major | . | Wu Weiwei | Akira Ajisaka | +| [HADOOP-16332](https://issues.apache.org/jira/browse/HADOOP-16332) | Remove S3A's depedency on http core | Critical | build, fs/s3 | Steve Loughran | Steve Loughran | +| [HDFS-13909](https://issues.apache.org/jira/browse/HDFS-13909) | RBF: Add Cache pools and directives related ClientProtocol APIs | Major | . | Dibyendu Karmakar | Ayush Saxena | +| [YARN-8693](https://issues.apache.org/jira/browse/YARN-8693) | Add signalToContainer REST API for RMWebServices | Major | restapi | Tao Yang | Tao Yang | +| [HDFS-14516](https://issues.apache.org/jira/browse/HDFS-14516) | RBF: Create hdfs-rbf-site.xml for RBF specific properties | Major | rbf | Takanobu Asanuma | Takanobu Asanuma | +| [HDFS-13787](https://issues.apache.org/jira/browse/HDFS-13787) | RBF: Add Snapshot related ClientProtocol APIs | Major | federation | Ranith Sardar | Íñigo Goiri | +| [HADOOP-13656](https://issues.apache.org/jira/browse/HADOOP-13656) | fs -expunge to take a filesystem | Minor | fs | Steve Loughran | Shweta | +| [HDFS-14475](https://issues.apache.org/jira/browse/HDFS-14475) | RBF: Expose router security enabled status on the UI | Major | . | CR Hota | CR Hota | +| [HDFS-13480](https://issues.apache.org/jira/browse/HDFS-13480) | RBF: Separate namenodeHeartbeat and routerHeartbeat to different config key | Major | . | maobaolong | Ayush Saxena | +| [HADOOP-16118](https://issues.apache.org/jira/browse/HADOOP-16118) | S3Guard to support on-demand DDB tables | Major | fs/s3 | Steve Loughran | Steve Loughran | +| [HDFS-14508](https://issues.apache.org/jira/browse/HDFS-14508) | RBF: Clean-up and refactor UI components | Minor | . | CR Hota | Takanobu Asanuma | +| [YARN-7537](https://issues.apache.org/jira/browse/YARN-7537) | [Atsv2] load hbase configuration from filesystem rather than URL | Major | . | Rohith Sharma K S | Prabhu Joseph | +| [MAPREDUCE-7210](https://issues.apache.org/jira/browse/MAPREDUCE-7210) | Replace \`mapreduce.job.counters.limit\` with \`mapreduce.job.counters.max\` in mapred-default.xml | Major | . | Wanqiang Ji | Wanqiang Ji | +| [HDFS-14526](https://issues.apache.org/jira/browse/HDFS-14526) | RBF: Update the document of RBF related metrics | Major | documentation | Takanobu Asanuma | Takanobu Asanuma | +| [HDFS-13404](https://issues.apache.org/jira/browse/HDFS-13404) | RBF: TestRouterWebHDFSContractAppend.testRenameFileBeingAppended fails | Major | test | Takanobu Asanuma | Takanobu Asanuma | +| [HADOOP-16314](https://issues.apache.org/jira/browse/HADOOP-16314) | Make sure all end point URL is covered by the same AuthenticationFilter | Major | security | Eric Yang | Prabhu Joseph | +| [HDFS-14356](https://issues.apache.org/jira/browse/HDFS-14356) | Implement HDFS cache on SCM with native PMDK libs | Major | caching, datanode | Feilong He | Feilong He | +| [HADOOP-16117](https://issues.apache.org/jira/browse/HADOOP-16117) | Update AWS SDK to 1.11.563 | Major | build, fs/s3 | Steve Loughran | Steve Loughran | +| [YARN-9590](https://issues.apache.org/jira/browse/YARN-9590) | Correct incompatible, incomplete and redundant activities | Major | . | Tao Yang | Tao Yang | +| [MAPREDUCE-6794](https://issues.apache.org/jira/browse/MAPREDUCE-6794) | Remove unused properties from TTConfig.java | Major | . | Akira Ajisaka | Akira Ajisaka | +| [HDFS-14550](https://issues.apache.org/jira/browse/HDFS-14550) | RBF: Failed to get statistics from NameNodes before 2.9.0 | Major | . | Akira Ajisaka | Xiaoqiao He | +| [HADOOP-15563](https://issues.apache.org/jira/browse/HADOOP-15563) | S3Guard to support creating on-demand DDB tables | Major | fs/s3 | Steve Loughran | Steve Loughran | +| [HDFS-14553](https://issues.apache.org/jira/browse/HDFS-14553) | Make queue size of BlockReportProcessingThread configurable | Major | namenode | Xiaoqiao He | Xiaoqiao He | +| [MAPREDUCE-7214](https://issues.apache.org/jira/browse/MAPREDUCE-7214) | Remove unused pieces related to \`mapreduce.job.userlog.retain.hours\` | Major | . | Wanqiang Ji | Wanqiang Ji | +| [HADOOP-16354](https://issues.apache.org/jira/browse/HADOOP-16354) | Enable AuthFilter as default for WebHdfs | Major | security | Prabhu Joseph | Prabhu Joseph | +| [HADOOP-16367](https://issues.apache.org/jira/browse/HADOOP-16367) | ApplicationHistoryServer related testcases failing | Major | security, test | Prabhu Joseph | Prabhu Joseph | +| [YARN-9578](https://issues.apache.org/jira/browse/YARN-9578) | Add limit/actions/summarize options for app activities REST API | Major | capacityscheduler | Tao Yang | Tao Yang | +| [HADOOP-16366](https://issues.apache.org/jira/browse/HADOOP-16366) | Fix TimelineReaderServer ignores ProxyUserAuthenticationFilterInitializer | Major | security | Prabhu Joseph | Prabhu Joseph | +| [HDFS-14545](https://issues.apache.org/jira/browse/HDFS-14545) | RBF: Router should support GetUserMappingsProtocol | Major | . | Íñigo Goiri | Ayush Saxena | +| [YARN-8499](https://issues.apache.org/jira/browse/YARN-8499) | ATS v2 Generic TimelineStorageMonitor | Major | ATSv2 | Sunil G | Prabhu Joseph | +| [HADOOP-16279](https://issues.apache.org/jira/browse/HADOOP-16279) | S3Guard: Implement time-based (TTL) expiry for entries (and tombstones) | Major | fs/s3 | Gabor Bota | Gabor Bota | +| [HADOOP-16376](https://issues.apache.org/jira/browse/HADOOP-16376) | ABFS: Override access() to no-op for now | Major | fs/azure | Da Zhou | Da Zhou | +| [YARN-9574](https://issues.apache.org/jira/browse/YARN-9574) | ArtifactId of MaWo application is wrong | Major | . | Wanqiang Ji | Wanqiang Ji | +| [HADOOP-16340](https://issues.apache.org/jira/browse/HADOOP-16340) | ABFS driver continues to retry on IOException responses from REST operations | Major | fs/azure | Robert Levas | Robert Levas | +| [HADOOP-16379](https://issues.apache.org/jira/browse/HADOOP-16379) | S3AInputStream#unbuffer should merge input stream stats into fs-wide stats | Major | fs/s3 | Sahil Takiar | Sahil Takiar | +| [HADOOP-15183](https://issues.apache.org/jira/browse/HADOOP-15183) | S3Guard store becomes inconsistent after partial failure of rename | Blocker | fs/s3 | Steve Loughran | Steve Loughran | +| [HADOOP-15658](https://issues.apache.org/jira/browse/HADOOP-15658) | Memory leak in S3AOutputStream | Major | fs/s3 | Piotr Nowojski | Steve Loughran | +| [HADOOP-16364](https://issues.apache.org/jira/browse/HADOOP-16364) | S3Guard table destroy to map IllegalArgumentExceptions to IOEs | Minor | fs/s3 | Steve Loughran | Steve Loughran | +| [HADOOP-15604](https://issues.apache.org/jira/browse/HADOOP-15604) | Bulk commits of S3A MPUs place needless excessive load on S3 & S3Guard | Major | fs/s3 | Gabor Bota | Steve Loughran | +| [HADOOP-16368](https://issues.apache.org/jira/browse/HADOOP-16368) | S3A list operation doesn't pick up etags from results | Minor | fs/s3 | Steve Loughran | Steve Loughran | +| [YARN-9374](https://issues.apache.org/jira/browse/YARN-9374) | HBaseTimelineWriterImpl sync writes has to avoid thread blocking if storage down | Major | ATSv2 | Prabhu Joseph | Prabhu Joseph | +| [HADOOP-16363](https://issues.apache.org/jira/browse/HADOOP-16363) | S3Guard DDB store prune() doesn't translate AWS exceptions to IOEs | Minor | fs/s3 | Steve Loughran | Steve Loughran | +| [HDFS-14590](https://issues.apache.org/jira/browse/HDFS-14590) | [SBN Read] Add the document link to the top page | Major | documentation | Takanobu Asanuma | Takanobu Asanuma | +| [YARN-9477](https://issues.apache.org/jira/browse/YARN-9477) | Implement VE discovery using libudev | Major | . | Peter Bacsko | Peter Bacsko | +| [YARN-6055](https://issues.apache.org/jira/browse/YARN-6055) | ContainersMonitorImpl need be adjusted when NM resource changed. | Major | graceful, nodemanager, scheduler | Junping Du | Íñigo Goiri | +| [HDFS-14036](https://issues.apache.org/jira/browse/HDFS-14036) | RBF: Add hdfs-rbf-default.xml to HdfsConfiguration by default | Major | . | Íñigo Goiri | Takanobu Asanuma | +| [YARN-9623](https://issues.apache.org/jira/browse/YARN-9623) | Auto adjust max queue length of app activities to make sure activities on all nodes can be covered | Major | . | Tao Yang | Tao Yang | +| [YARN-9560](https://issues.apache.org/jira/browse/YARN-9560) | Restructure DockerLinuxContainerRuntime to extend a new OCIContainerRuntime | Major | . | Eric Badger | Eric Badger | +| [HDFS-14620](https://issues.apache.org/jira/browse/HDFS-14620) | RBF: Fix 'not a super user' error when disabling a namespace in kerberos with superuser principal | Major | . | luhuachao | luhuachao | +| [HDFS-14622](https://issues.apache.org/jira/browse/HDFS-14622) | [Dynamometer] State transition err when CCM( HDFS Centralized Cache Management) feature is used | Major | tools | TanYuxin | Erik Krogen | +| [YARN-9660](https://issues.apache.org/jira/browse/YARN-9660) | Enhance documentation of Docker on YARN support | Major | documentation, nodemanager | Peter Bacsko | Peter Bacsko | +| [HDFS-14640](https://issues.apache.org/jira/browse/HDFS-14640) | [Dynamometer] Fix TestDynamometerInfra failures | Major | test, tools | Erik Krogen | Erik Krogen | +| [HDFS-14410](https://issues.apache.org/jira/browse/HDFS-14410) | Make Dynamometer documentation properly compile onto the Hadoop site | Major | . | Erik Krogen | Erik Krogen | +| [HADOOP-16357](https://issues.apache.org/jira/browse/HADOOP-16357) | TeraSort Job failing on S3 DirectoryStagingCommitter: destination path exists | Minor | fs/s3 | Prabhu Joseph | Steve Loughran | +| [HDFS-14611](https://issues.apache.org/jira/browse/HDFS-14611) | Move handshake secret field from Token to BlockAccessToken | Blocker | hdfs | Chen Liang | Chen Liang | +| [HADOOP-16384](https://issues.apache.org/jira/browse/HADOOP-16384) | S3A: Avoid inconsistencies between DDB and S3 | Major | fs/s3 | Steve Loughran | Steve Loughran | +| [HADOOP-16397](https://issues.apache.org/jira/browse/HADOOP-16397) | Hadoop S3Guard Prune command to support a -tombstone option. | Minor | fs/s3 | Steve Loughran | Steve Loughran | +| [HADOOP-16406](https://issues.apache.org/jira/browse/HADOOP-16406) | ITestDynamoDBMetadataStore.testProvisionTable times out intermittently | Minor | fs/s3, test | Steve Loughran | Steve Loughran | +| [HDFS-14458](https://issues.apache.org/jira/browse/HDFS-14458) | Report pmem stats to namenode | Major | . | Feilong He | Feilong He | +| [HDFS-14357](https://issues.apache.org/jira/browse/HDFS-14357) | Update documentation for HDFS cache on SCM support | Major | . | Feilong He | Feilong He | +| [HDFS-14593](https://issues.apache.org/jira/browse/HDFS-14593) | RBF: Implement deletion feature for expired records in State Store | Major | rbf | Takanobu Asanuma | Takanobu Asanuma | +| [HADOOP-16383](https://issues.apache.org/jira/browse/HADOOP-16383) | Pass ITtlTimeProvider instance in initialize method in MetadataStore interface | Major | . | Gabor Bota | Gabor Bota | +| [HDFS-14653](https://issues.apache.org/jira/browse/HDFS-14653) | RBF: Correct the default value for dfs.federation.router.namenode.heartbeat.enable | Major | . | Ayush Saxena | Ayush Saxena | +| [HDFS-14577](https://issues.apache.org/jira/browse/HDFS-14577) | RBF: FederationUtil#newInstance should allow constructor without context | Major | . | CR Hota | CR Hota | +| [HADOOP-15847](https://issues.apache.org/jira/browse/HADOOP-15847) | S3Guard testConcurrentTableCreations to set r & w capacity == 0 | Major | fs/s3, test | Steve Loughran | lqjacklee | +| [HADOOP-16380](https://issues.apache.org/jira/browse/HADOOP-16380) | S3A tombstones can confuse empty directory status | Blocker | fs/s3, test | Steve Loughran | Steve Loughran | +| [HADOOP-16447](https://issues.apache.org/jira/browse/HADOOP-16447) | Upgrade JUnit5 from 5.3.1 to 5.5.1 to support global timeout | Major | test | Akira Ajisaka | Kevin Su | +| [HDFS-14670](https://issues.apache.org/jira/browse/HDFS-14670) | RBF: Create secret manager instance using FederationUtil#newInstance. | Major | . | CR Hota | CR Hota | +| [HDFS-14639](https://issues.apache.org/jira/browse/HDFS-14639) | [Dynamometer] Unnecessary duplicate bin directory appears in dist layout | Major | namenode, test | Erik Krogen | Erik Krogen | +| [HADOOP-16472](https://issues.apache.org/jira/browse/HADOOP-16472) | findbugs warning on LocalMetadataStore.ttlTimeProvider sync | Major | build, fs/s3 | Steve Loughran | Steve Loughran | +| [HADOOP-16275](https://issues.apache.org/jira/browse/HADOOP-16275) | Upgrade Mockito to the latest version | Minor | test | Akira Ajisaka | Kevin Su | +| [HADOOP-16479](https://issues.apache.org/jira/browse/HADOOP-16479) | ABFS FileStatus.getModificationTime returns localized time instead of UTC | Major | fs/azure | Joan Sala Reixach | Bilahari T H | +| [HDFS-14034](https://issues.apache.org/jira/browse/HDFS-14034) | Support getQuotaUsage API in WebHDFS | Major | fs, webhdfs | Erik Krogen | Chao Sun | +| [HDFS-14700](https://issues.apache.org/jira/browse/HDFS-14700) | Clean up pmem cache before setting pmem cache capacity | Minor | caching, datanode | Feilong He | Feilong He | +| [HADOOP-16315](https://issues.apache.org/jira/browse/HADOOP-16315) | ABFS: transform full UPN for named user in AclStatus | Major | fs/azure | Da Zhou | Da Zhou | +| [HADOOP-16499](https://issues.apache.org/jira/browse/HADOOP-16499) | S3A retry policy to be exponential | Critical | fs/s3 | Steve Loughran | Steve Loughran | +| [HADOOP-16481](https://issues.apache.org/jira/browse/HADOOP-16481) | ITestS3GuardDDBRootOperations.test\_300\_MetastorePrune needs to set region | Major | fs/s3, test | Steve Loughran | Steve Loughran | +| [YARN-9694](https://issues.apache.org/jira/browse/YARN-9694) | UI always show default-rack for all the nodes while running SLS. | Major | . | Abhishek Modi | Abhishek Modi | +| [HADOOP-16509](https://issues.apache.org/jira/browse/HADOOP-16509) | [hadoop-mapreduce-project] Fix order of actual and expected expression in assert statements | Major | . | Adam Antal | Adam Antal | +| [HDFS-14717](https://issues.apache.org/jira/browse/HDFS-14717) | Junit not found in hadoop-dynamometer-infra | Major | . | Kevin Su | Kevin Su | +| [YARN-9608](https://issues.apache.org/jira/browse/YARN-9608) | DecommissioningNodesWatcher should get lists of running applications on node from RMNode. | Major | . | Abhishek Modi | Abhishek Modi | +| [HADOOP-16500](https://issues.apache.org/jira/browse/HADOOP-16500) | S3ADelegationTokens to only log at debug on startup | Major | fs/s3 | Steve Loughran | Steve Loughran | +| [YARN-9683](https://issues.apache.org/jira/browse/YARN-9683) | Remove reapDockerContainerNoPid left behind by YARN-9074 | Trivial | yarn | Adam Antal | Kevin Su | +| [HDFS-14713](https://issues.apache.org/jira/browse/HDFS-14713) | RBF: RouterAdmin supports refreshRouterArgs command but not on display | Major | . | wangzhaohui | wangzhaohui | +| [YARN-9765](https://issues.apache.org/jira/browse/YARN-9765) | SLS runner crashes when run with metrics turned off. | Major | . | Abhishek Modi | Abhishek Modi | +| [YARN-9752](https://issues.apache.org/jira/browse/YARN-9752) | Add support for allocation id in SLS. | Major | . | Abhishek Modi | Abhishek Modi | +| [HDFS-14714](https://issues.apache.org/jira/browse/HDFS-14714) | RBF: implement getReplicatedBlockStats interface | Major | . | Chen Zhang | Chen Zhang | +| [HDFS-14756](https://issues.apache.org/jira/browse/HDFS-14756) | RBF: getQuotaUsage may ignore some folders | Major | . | Chen Zhang | Chen Zhang | +| [HDFS-14744](https://issues.apache.org/jira/browse/HDFS-14744) | RBF: Non secured routers should not log in error mode when UGI is default. | Major | . | CR Hota | CR Hota | +| [HDFS-14763](https://issues.apache.org/jira/browse/HDFS-14763) | Fix package name of audit log class in Dynamometer document | Major | documentation, tools | Takanobu Asanuma | Takanobu Asanuma | +| [HADOOP-16470](https://issues.apache.org/jira/browse/HADOOP-16470) | Make last AWS credential provider in default auth chain EC2ContainerCredentialsProviderWrapper | Major | fs/s3 | Steve Loughran | Steve Loughran | +| [HDFS-14755](https://issues.apache.org/jira/browse/HDFS-14755) | [Dynamometer] Hadoop-2 DataNode fail to start | Major | tools | Takanobu Asanuma | Takanobu Asanuma | +| [HADOOP-16469](https://issues.apache.org/jira/browse/HADOOP-16469) | Typo in s3a committers.md doc | Minor | documentation, fs/s3 | Steve Loughran | | +| [HDFS-14674](https://issues.apache.org/jira/browse/HDFS-14674) | [SBN read] Got an unexpected txid when tail editlog | Blocker | . | wangzhaohui | wangzhaohui | +| [HDFS-14766](https://issues.apache.org/jira/browse/HDFS-14766) | RBF: MountTableStoreImpl#getMountTableEntries returns extra entry | Major | . | Chen Zhang | Chen Zhang | +| [YARN-9775](https://issues.apache.org/jira/browse/YARN-9775) | RMWebServices /scheduler-conf GET returns all hadoop configurations for ZKConfigurationStore | Major | restapi | Prabhu Joseph | Prabhu Joseph | +| [YARN-9755](https://issues.apache.org/jira/browse/YARN-9755) | RM fails to start with FileSystemBasedConfigurationProvider | Major | resourcemanager | Prabhu Joseph | Prabhu Joseph | +| [HDFS-14779](https://issues.apache.org/jira/browse/HDFS-14779) | Fix logging error in TestEditLog#testMultiStreamsLoadEditWithConfMaxTxns | Major | . | Jonathan Hung | Jonathan Hung | +| [HADOOP-16416](https://issues.apache.org/jira/browse/HADOOP-16416) | mark DynamoDBMetadataStore.deleteTrackingValueMap as final | Trivial | fs/s3 | Steve Loughran | Kevin Su | +| [HDFS-8631](https://issues.apache.org/jira/browse/HDFS-8631) | WebHDFS : Support setQuota | Major | . | nijel | Chao Sun | +| [YARN-9754](https://issues.apache.org/jira/browse/YARN-9754) | Add support for arbitrary DAG AM Simulator. | Major | . | Abhishek Modi | Abhishek Modi | +| [YARN-9664](https://issues.apache.org/jira/browse/YARN-9664) | Improve response of scheduler/app activities for better understanding | Major | . | Tao Yang | Tao Yang | +| [YARN-8678](https://issues.apache.org/jira/browse/YARN-8678) | Queue Management API - rephrase error messages | Major | . | Akhil PB | Prabhu Joseph | +| [HDFS-14711](https://issues.apache.org/jira/browse/HDFS-14711) | RBF: RBFMetrics throws NullPointerException if stateStore disabled | Major | . | Chen Zhang | Chen Zhang | +| [YARN-9791](https://issues.apache.org/jira/browse/YARN-9791) | Queue Mutation API does not allow to remove a config | Major | capacity scheduler | Prabhu Joseph | Prabhu Joseph | +| [YARN-7982](https://issues.apache.org/jira/browse/YARN-7982) | Do ACLs check while retrieving entity-types per application | Major | . | Rohith Sharma K S | Prabhu Joseph | +| [HDFS-14654](https://issues.apache.org/jira/browse/HDFS-14654) | RBF: TestRouterRpc#testNamenodeMetrics is flaky | Major | . | Takanobu Asanuma | Chen Zhang | +| [YARN-9804](https://issues.apache.org/jira/browse/YARN-9804) | Update ATSv2 document for latest feature supports | Blocker | . | Rohith Sharma K S | Rohith Sharma K S | +| [HDFS-14819](https://issues.apache.org/jira/browse/HDFS-14819) | [Dynamometer] Cannot parse audit logs with ‘=‘ in unexpected places when starting a workload. | Major | . | Soya Miyoshi | Soya Miyoshi | +| [HDFS-14817](https://issues.apache.org/jira/browse/HDFS-14817) | [Dynamometer] start-dynamometer-cluster.sh shows its usage even if correct arguments are given. | Major | tools | Soya Miyoshi | Soya Miyoshi | +| [YARN-9821](https://issues.apache.org/jira/browse/YARN-9821) | NM hangs at serviceStop when ATSV2 Backend Hbase is Down | Major | ATSv2 | Prabhu Joseph | Prabhu Joseph | +| [HADOOP-16430](https://issues.apache.org/jira/browse/HADOOP-16430) | S3AFilesystem.delete to incrementally update s3guard with deletions | Major | fs/s3 | Steve Loughran | Steve Loughran | +| [HADOOP-16490](https://issues.apache.org/jira/browse/HADOOP-16490) | Avoid/handle cached 404s during S3A file creation | Major | fs/s3 | Steve Loughran | Steve Loughran | +| [YARN-9819](https://issues.apache.org/jira/browse/YARN-9819) | Make TestOpportunisticContainerAllocatorAMService more resilient. | Major | . | Abhishek Modi | Abhishek Modi | +| [HADOOP-16562](https://issues.apache.org/jira/browse/HADOOP-16562) | [pb-upgrade] Update docker image to have 3.7.1 protoc executable | Major | . | Vinayakumar B | Vinayakumar B | +| [YARN-9794](https://issues.apache.org/jira/browse/YARN-9794) | RM crashes due to runtime errors in TimelineServiceV2Publisher | Major | . | Tarun Parimi | Tarun Parimi | +| [HADOOP-16371](https://issues.apache.org/jira/browse/HADOOP-16371) | Option to disable GCM for SSL connections when running on Java 8 | Major | fs/s3 | Sahil Takiar | Sahil Takiar | +| [HDFS-14822](https://issues.apache.org/jira/browse/HDFS-14822) | [SBN read] Revisit GlobalStateIdContext locking when getting server state id | Major | hdfs | Chen Liang | Chen Liang | +| [HADOOP-16557](https://issues.apache.org/jira/browse/HADOOP-16557) | [pb-upgrade] Upgrade protobuf.version to 3.7.1 | Major | . | Vinayakumar B | Vinayakumar B | +| [HDFS-14833](https://issues.apache.org/jira/browse/HDFS-14833) | RBF: Router Update Doesn't Sync Quota | Major | . | Ayush Saxena | Ayush Saxena | +| [HADOOP-16589](https://issues.apache.org/jira/browse/HADOOP-16589) | [pb-upgrade] Update docker image to make 3.7.1 protoc as default | Major | . | Vinayakumar B | Vinayakumar B | +| [HDFS-14818](https://issues.apache.org/jira/browse/HDFS-14818) | Check native pmdk lib by 'hadoop checknative' command | Minor | native | Feilong He | Feilong He | +| [HADOOP-16558](https://issues.apache.org/jira/browse/HADOOP-16558) | [COMMON+HDFS] use protobuf-maven-plugin to generate protobuf classes | Major | common | Vinayakumar B | Vinayakumar B | +| [HADOOP-16565](https://issues.apache.org/jira/browse/HADOOP-16565) | Region must be provided when requesting session credentials or SdkClientException will be thrown | Major | fs/s3 | Gabor Bota | Gabor Bota | +| [HADOOP-16138](https://issues.apache.org/jira/browse/HADOOP-16138) | hadoop fs mkdir / of nonexistent abfs container raises NPE | Minor | fs/azure | Steve Loughran | Gabor Bota | +| [HADOOP-16591](https://issues.apache.org/jira/browse/HADOOP-16591) | S3A ITest\*MRjob failures | Major | fs/s3 | Siddharth Seth | Siddharth Seth | +| [HADOOP-16560](https://issues.apache.org/jira/browse/HADOOP-16560) | [YARN] use protobuf-maven-plugin to generate protobuf classes | Major | . | Vinayakumar B | Duo Zhang | +| [HADOOP-16561](https://issues.apache.org/jira/browse/HADOOP-16561) | [MAPREDUCE] use protobuf-maven-plugin to generate protobuf classes | Major | . | Vinayakumar B | Duo Zhang | +| [HDFS-14461](https://issues.apache.org/jira/browse/HDFS-14461) | RBF: Fix intermittently failing kerberos related unit test | Major | . | CR Hota | Xiaoqiao He | +| [HDFS-14785](https://issues.apache.org/jira/browse/HDFS-14785) | [SBN read] Change client logging to be less aggressive | Major | hdfs | Chen Liang | Chen Liang | +| [YARN-9859](https://issues.apache.org/jira/browse/YARN-9859) | Refactor OpportunisticContainerAllocator | Major | . | Abhishek Modi | Abhishek Modi | +| [YARN-9864](https://issues.apache.org/jira/browse/YARN-9864) | Format CS Configuration present in Configuration Store | Major | capacity scheduler | Prabhu Joseph | Prabhu Joseph | +| [YARN-9801](https://issues.apache.org/jira/browse/YARN-9801) | SchedConfCli does not work with https mode | Major | . | Prabhu Joseph | Prabhu Joseph | +| [HADOOP-16458](https://issues.apache.org/jira/browse/HADOOP-16458) | LocatedFileStatusFetcher scans failing intermittently against S3 store | Major | fs/s3 | Steve Loughran | Steve Loughran | +| [HADOOP-16578](https://issues.apache.org/jira/browse/HADOOP-16578) | ABFS: fileSystemExists() should not call container level apis | Major | fs/azure | Da Zhou | Sneha Vijayarajan | +| [YARN-9870](https://issues.apache.org/jira/browse/YARN-9870) | Remove unused function from OpportunisticContainerAllocatorAMService | Minor | . | Abhishek Modi | Abhishek Modi | +| [YARN-9792](https://issues.apache.org/jira/browse/YARN-9792) | Document examples of SchedulerConf with Node Labels | Minor | capacity scheduler | Prabhu Joseph | Prabhu Joseph | +| [HADOOP-13373](https://issues.apache.org/jira/browse/HADOOP-13373) | Add S3A implementation of FSMainOperationsBaseTest | Minor | fs/s3 | Steve Loughran | Steve Loughran | +| [HDFS-14858](https://issues.apache.org/jira/browse/HDFS-14858) | [SBN read] Allow configurably enable/disable AlignmentContext on NameNode | Major | hdfs | Chen Liang | Chen Liang | +| [HADOOP-16620](https://issues.apache.org/jira/browse/HADOOP-16620) | [pb-upgrade] Remove protocol buffers 3.7.1 from requirements in BUILDING.txt | Minor | build, documentation | Akira Ajisaka | Akira Ajisaka | +| [HADOOP-15729](https://issues.apache.org/jira/browse/HADOOP-15729) | [s3a] stop treat fs.s3a.max.threads as the long-term minimum | Major | fs/s3 | Sean Mackrory | Sean Mackrory | +| [HDFS-12979](https://issues.apache.org/jira/browse/HDFS-12979) | StandbyNode should upload FsImage to ObserverNode after checkpointing. | Major | hdfs | Konstantin Shvachko | Chen Liang | +| [YARN-9782](https://issues.apache.org/jira/browse/YARN-9782) | Avoid DNS resolution while running SLS. | Major | . | Abhishek Modi | Abhishek Modi | +| [HADOOP-16207](https://issues.apache.org/jira/browse/HADOOP-16207) | Improved S3A MR tests | Critical | fs/s3, test | Steve Loughran | Steve Loughran | +| [HADOOP-16570](https://issues.apache.org/jira/browse/HADOOP-16570) | S3A committers leak threads/raises OOM on job/task commit at scale | Major | fs/s3 | Steve Loughran | Steve Loughran | +| [HADOOP-16626](https://issues.apache.org/jira/browse/HADOOP-16626) | S3A ITestRestrictedReadAccess fails | Major | fs/s3 | Siddharth Seth | Steve Loughran | +| [HADOOP-16512](https://issues.apache.org/jira/browse/HADOOP-16512) | [hadoop-tools] Fix order of actual and expected expression in assert statements | Major | . | Adam Antal | Kevin Su | +| [HADOOP-16587](https://issues.apache.org/jira/browse/HADOOP-16587) | Make AAD endpoint configurable on all Auth flows | Minor | fs/azure | Bilahari T H | Bilahari T H | +| [YARN-9873](https://issues.apache.org/jira/browse/YARN-9873) | Mutation API Config Change need to update Version Number | Major | . | Prabhu Joseph | Prabhu Joseph | +| [HDFS-14162](https://issues.apache.org/jira/browse/HDFS-14162) | Balancer should work with ObserverNode | Major | . | Konstantin Shvachko | Erik Krogen | +| [HADOOP-16650](https://issues.apache.org/jira/browse/HADOOP-16650) | ITestS3AClosedFS failing -junit test thread | Blocker | fs/s3, test | Steve Loughran | Steve Loughran | +| [YARN-9699](https://issues.apache.org/jira/browse/YARN-9699) | Migration tool that help to generate CS config based on FS config [Phase 1] | Major | . | Wanqiang Ji | Peter Bacsko | +| [HADOOP-16635](https://issues.apache.org/jira/browse/HADOOP-16635) | S3A innerGetFileStatus s"directories only" scan still does a HEAD | Blocker | fs/s3 | Steve Loughran | Steve Loughran | +| [HADOOP-16634](https://issues.apache.org/jira/browse/HADOOP-16634) | S3A ITest failures without S3Guard | Major | fs/s3, test | Steve Loughran | Steve Loughran | +| [YARN-9840](https://issues.apache.org/jira/browse/YARN-9840) | Capacity scheduler: add support for Secondary Group rule mapping | Major | capacity scheduler | Peter Bacsko | Manikandan R | +| [HADOOP-16651](https://issues.apache.org/jira/browse/HADOOP-16651) | S3 getBucketLocation() can return "US" for us-east | Major | fs/s3 | Steve Loughran | Steve Loughran | +| [HADOOP-16478](https://issues.apache.org/jira/browse/HADOOP-16478) | S3Guard bucket-info fails if the bucket location is denied to the caller | Major | fs/s3 | Steve Loughran | Steve Loughran | +| [YARN-9773](https://issues.apache.org/jira/browse/YARN-9773) | Add QueueMetrics for Custom Resources | Major | . | Manikandan R | Manikandan R | +| [YARN-9841](https://issues.apache.org/jira/browse/YARN-9841) | Capacity scheduler: add support for combined %user + %primary\_group mapping | Major | capacity scheduler | Peter Bacsko | Manikandan R | +| [YARN-9884](https://issues.apache.org/jira/browse/YARN-9884) | Make container-executor mount logic modular | Major | . | Eric Badger | Eric Badger | +| [YARN-9875](https://issues.apache.org/jira/browse/YARN-9875) | FSSchedulerConfigurationStore fails to update with hdfs path | Major | capacityscheduler | Prabhu Joseph | Prabhu Joseph | +| [HDFS-14638](https://issues.apache.org/jira/browse/HDFS-14638) | [Dynamometer] Fix scripts to refer to current build structure | Major | namenode, test | Erik Krogen | Takanobu Asanuma | +| [HDFS-14907](https://issues.apache.org/jira/browse/HDFS-14907) | [Dynamometer] DataNode can't find junit jar when using Hadoop-3 binary | Major | . | Takanobu Asanuma | Takanobu Asanuma | +| [HDFS-14824](https://issues.apache.org/jira/browse/HDFS-14824) | [Dynamometer] Dynamometer in org.apache.hadoop.tools does not output the benchmark results. | Major | . | Soya Miyoshi | Takanobu Asanuma | +| [HADOOP-16510](https://issues.apache.org/jira/browse/HADOOP-16510) | [hadoop-common] Fix order of actual and expected expression in assert statements | Major | . | Adam Antal | Adam Antal | +| [YARN-9950](https://issues.apache.org/jira/browse/YARN-9950) | Unset Ordering Policy of Leaf/Parent queue converted from Parent/Leaf queue respectively | Major | capacity scheduler | Prabhu Joseph | Prabhu Joseph | +| [HDFS-14891](https://issues.apache.org/jira/browse/HDFS-14891) | RBF: namenode links in NameFederation Health page (federationhealth.html) cannot use https scheme | Major | rbf, ui | Xieming Li | Xieming Li | +| [YARN-9865](https://issues.apache.org/jira/browse/YARN-9865) | Capacity scheduler: add support for combined %user + %secondary\_group mapping | Major | . | Manikandan R | Manikandan R | +| [YARN-9697](https://issues.apache.org/jira/browse/YARN-9697) | Efficient allocation of Opportunistic containers. | Major | . | Abhishek Modi | Abhishek Modi | +| [HADOOP-16477](https://issues.apache.org/jira/browse/HADOOP-16477) | S3A delegation token tests fail if fs.s3a.encryption.key set | Major | fs/s3, test | Steve Loughran | Steve Loughran | +| [HDFS-14648](https://issues.apache.org/jira/browse/HDFS-14648) | Implement DeadNodeDetector basic model | Major | . | Lisheng Sun | Lisheng Sun | +| [YARN-9900](https://issues.apache.org/jira/browse/YARN-9900) | Revert to previous state when Invalid Config is applied and Refresh Support in SchedulerConfig Format | Major | capacity scheduler | Prabhu Joseph | Prabhu Joseph | +| [HADOOP-16484](https://issues.apache.org/jira/browse/HADOOP-16484) | S3A to warn or fail if S3Guard is disabled | Minor | fs/s3 | Steve Loughran | Gabor Bota | +| [YARN-9562](https://issues.apache.org/jira/browse/YARN-9562) | Add Java changes for the new RuncContainerRuntime | Major | . | Eric Badger | Eric Badger | +| [HADOOP-16657](https://issues.apache.org/jira/browse/HADOOP-16657) | Move remaining log4j APIs over to slf4j in hadoop-common. | Major | . | Minni Mittal | Minni Mittal | +| [HADOOP-16610](https://issues.apache.org/jira/browse/HADOOP-16610) | Upgrade to yetus 0.11.1 and use emoji vote on github pre commit | Major | build | Duo Zhang | Duo Zhang | +| [YARN-9909](https://issues.apache.org/jira/browse/YARN-9909) | Offline format of YarnConfigurationStore | Major | capacity scheduler | Prabhu Joseph | Prabhu Joseph | +| [HADOOP-16632](https://issues.apache.org/jira/browse/HADOOP-16632) | Speculating & Partitioned S3A magic committers can leave pending files under \_\_magic | Minor | fs/s3 | Steve Loughran | Steve Loughran | +| [YARN-9836](https://issues.apache.org/jira/browse/YARN-9836) | General usability improvements in showSimulationTrace.html | Minor | scheduler-load-simulator | Adam Antal | Adam Antal | +| [HADOOP-16707](https://issues.apache.org/jira/browse/HADOOP-16707) | NPE in UGI.getCurrentUser in ITestAbfsIdentityTransformer setup | Major | auth, fs/azure, security | Steve Loughran | Steve Loughran | +| [HADOOP-16687](https://issues.apache.org/jira/browse/HADOOP-16687) | ABFS: Fix testcase added for HADOOP-16138 for namespace enabled account | Major | fs/azure | Sneha Vijayarajan | Sneha Vijayarajan | +| [HDFS-14651](https://issues.apache.org/jira/browse/HDFS-14651) | DeadNodeDetector checks dead node periodically | Major | . | Lisheng Sun | Lisheng Sun | +| [HADOOP-16708](https://issues.apache.org/jira/browse/HADOOP-16708) | HadoopExecutors cleanup to only log at debug | Minor | util | Steve Loughran | David Mollitor | +| [YARN-9899](https://issues.apache.org/jira/browse/YARN-9899) | Migration tool that help to generate CS config based on FS config [Phase 2] | Major | . | Szilard Nemeth | Peter Bacsko | +| [HDFS-14649](https://issues.apache.org/jira/browse/HDFS-14649) | Add suspect probe for DeadNodeDetector | Major | . | Lisheng Sun | Lisheng Sun | +| [HADOOP-16455](https://issues.apache.org/jira/browse/HADOOP-16455) | ABFS: Implement FileSystem.access() method | Minor | fs/azure | Bilahari T H | Bilahari T H | +| [HADOOP-16660](https://issues.apache.org/jira/browse/HADOOP-16660) | ABFS: Make RetryCount in ExponentialRetryPolicy Configurable | Major | fs/azure | Sneha Vijayarajan | Sneha Vijayarajan | +| [HDFS-15019](https://issues.apache.org/jira/browse/HDFS-15019) | Refactor the unit test of TestDeadNodeDetection | Minor | . | Yiqun Lin | Lisheng Sun | +| [HDFS-14825](https://issues.apache.org/jira/browse/HDFS-14825) | [Dynamometer] Workload doesn't start unless an absolute path of Mapper class given | Major | . | Soya Miyoshi | Takanobu Asanuma | +| [HDFS-13811](https://issues.apache.org/jira/browse/HDFS-13811) | RBF: Race condition between router admin quota update and periodic quota update service | Major | . | Dibyendu Karmakar | Jinglun | +| [YARN-9781](https://issues.apache.org/jira/browse/YARN-9781) | SchedConfCli to get current stored scheduler configuration | Major | capacity scheduler | Prabhu Joseph | Prabhu Joseph | +| [YARN-9780](https://issues.apache.org/jira/browse/YARN-9780) | SchedulerConf Mutation API does not Allow Stop and Remove Queue in a single call | Major | capacity scheduler | Prabhu Joseph | Prabhu Joseph | +| [YARN-9789](https://issues.apache.org/jira/browse/YARN-9789) | Disable Option for Write Ahead Logs of LogMutation | Major | capacity scheduler | Prabhu Joseph | Prabhu Joseph | +| [YARN-9607](https://issues.apache.org/jira/browse/YARN-9607) | Auto-configuring rollover-size of IFile format for non-appendable filesystems | Major | log-aggregation, yarn | Adam Antal | Adam Antal | +| [YARN-9561](https://issues.apache.org/jira/browse/YARN-9561) | Add C changes for the new RuncContainerRuntime | Major | . | Eric Badger | Eric Badger | +| [HADOOP-16612](https://issues.apache.org/jira/browse/HADOOP-16612) | Track Azure Blob File System client-perceived latency | Major | fs/azure, hdfs-client | Jeetesh Mangwani | Jeetesh Mangwani | +| [HDFS-15043](https://issues.apache.org/jira/browse/HDFS-15043) | RBF: The detail of the Exception is not shown in ZKDelegationTokenSecretManagerImpl | Major | . | Akira Ajisaka | Akira Ajisaka | +| [HADOOP-16758](https://issues.apache.org/jira/browse/HADOOP-16758) | Refine testing.md to tell user better how to use auth-keys.xml | Minor | fs/s3 | Mingliang Liu | Mingliang Liu | +| [HDFS-14983](https://issues.apache.org/jira/browse/HDFS-14983) | RBF: Add dfsrouteradmin -refreshSuperUserGroupsConfiguration command option | Minor | rbf | Akira Ajisaka | Xieming Li | +| [HDFS-15044](https://issues.apache.org/jira/browse/HDFS-15044) | [Dynamometer] Show the line of audit log when parsing it unsuccessfully | Major | tools | Takanobu Asanuma | Takanobu Asanuma | +| [HADOOP-16757](https://issues.apache.org/jira/browse/HADOOP-16757) | Increase timeout unit test rule for MetadataStoreTestBase | Major | fs/s3 | Mingliang Liu | Mingliang Liu | +| [HADOOP-16764](https://issues.apache.org/jira/browse/HADOOP-16764) | Rewrite Python example codes using Python3 | Minor | documentation | Kengo Seki | Kengo Seki | +| [HADOOP-16751](https://issues.apache.org/jira/browse/HADOOP-16751) | DurationInfo text parsing/formatting should be moved out of hotpath | Minor | fs/s3 | Rajesh Balamohan | Rajesh Balamohan | +| [YARN-10035](https://issues.apache.org/jira/browse/YARN-10035) | Add ability to filter the Cluster Applications API request by name | Major | yarn | Adam Antal | Adam Antal | +| [HDFS-15066](https://issues.apache.org/jira/browse/HDFS-15066) | HttpFS: Implement setErasureCodingPolicy , unsetErasureCodingPolicy , getErasureCodingPolicy | Major | . | hemanthboyina | hemanthboyina | +| [HADOOP-16645](https://issues.apache.org/jira/browse/HADOOP-16645) | S3A Delegation Token extension point to use StoreContext | Minor | fs/s3 | Steve Loughran | Steve Loughran | +| [HADOOP-16699](https://issues.apache.org/jira/browse/HADOOP-16699) | ABFS: Enhance driver debug logs | Major | fs/azure | Sneha Vijayarajan | Sneha Vijayarajan | +| [YARN-10068](https://issues.apache.org/jira/browse/YARN-10068) | TimelineV2Client may leak file descriptors creating ClientResponse objects. | Critical | ATSv2 | Anand Srinivasan | Anand Srinivasan | +| [HADOOP-16642](https://issues.apache.org/jira/browse/HADOOP-16642) | ITestDynamoDBMetadataStoreScale fails when throttled. | Major | fs/s3, test | Steve Loughran | Steve Loughran | +| [HDFS-15100](https://issues.apache.org/jira/browse/HDFS-15100) | RBF: Print stacktrace when DFSRouter fails to fetch/parse JMX output from NameNode | Major | rbf | Akira Ajisaka | Akira Ajisaka | +| [YARN-10071](https://issues.apache.org/jira/browse/YARN-10071) | Sync Mockito version with other modules | Major | build, test | Akira Ajisaka | Adam Antal | +| [HADOOP-16697](https://issues.apache.org/jira/browse/HADOOP-16697) | audit/tune s3a authoritative flag in s3guard DDB Table | Major | fs/s3 | Steve Loughran | Steve Loughran | +| [YARN-10067](https://issues.apache.org/jira/browse/YARN-10067) | Add dry-run feature to FS-CS converter tool | Major | . | Peter Bacsko | Peter Bacsko | +| [YARN-10026](https://issues.apache.org/jira/browse/YARN-10026) | Pull out common code pieces from ATS v1.5 and v2 | Major | ATSv2, yarn | Adam Antal | Adam Antal | +| [HADOOP-16797](https://issues.apache.org/jira/browse/HADOOP-16797) | Add dockerfile for ARM builds | Major | . | Vinayakumar B | Vinayakumar B | +| [YARN-9788](https://issues.apache.org/jira/browse/YARN-9788) | Queue Management API does not support parallel updates | Major | capacity scheduler | Prabhu Joseph | Prabhu Joseph | +| [HADOOP-16005](https://issues.apache.org/jira/browse/HADOOP-16005) | NativeAzureFileSystem does not support setXAttr | Major | fs/azure | Clemens Wolff | Clemens Wolff | +| [YARN-10028](https://issues.apache.org/jira/browse/YARN-10028) | Integrate the new abstract log servlet to the JobHistory server | Major | yarn | Adam Antal | Adam Antal | +| [YARN-10082](https://issues.apache.org/jira/browse/YARN-10082) | FS-CS converter: disable terminal placement rule checking | Critical | . | Peter Bacsko | Peter Bacsko | +| [HADOOP-16621](https://issues.apache.org/jira/browse/HADOOP-16621) | [pb-upgrade] Remove Protobuf classes from signatures of Public APIs | Critical | common | Steve Loughran | Vinayakumar B | +| [YARN-9525](https://issues.apache.org/jira/browse/YARN-9525) | IFile format is not working against s3a remote folder | Major | log-aggregation | Adam Antal | Adam Antal | +| [HADOOP-16346](https://issues.apache.org/jira/browse/HADOOP-16346) | Stabilize S3A OpenSSL support | Blocker | fs/s3 | Steve Loughran | Sahil Takiar | +| [HADOOP-16759](https://issues.apache.org/jira/browse/HADOOP-16759) | Filesystem openFile() builder to take a FileStatus param | Minor | fs, fs/azure, fs/s3 | Steve Loughran | Steve Loughran | +| [YARN-10083](https://issues.apache.org/jira/browse/YARN-10083) | Provide utility to ask whether an application is in final status | Minor | . | Adam Antal | Adam Antal | +| [HADOOP-16792](https://issues.apache.org/jira/browse/HADOOP-16792) | Let s3 clients configure request timeout | Major | fs/s3 | Mustafa Iman | Mustafa Iman | +| [HADOOP-16827](https://issues.apache.org/jira/browse/HADOOP-16827) | TestHarFileSystem.testInheritedMethodsImplemented broken | Major | fs, test | Steve Loughran | Steve Loughran | +| [HADOOP-16746](https://issues.apache.org/jira/browse/HADOOP-16746) | S3A empty dir markers are not created in s3guard as authoritative | Critical | fs/s3 | Steve Loughran | Steve Loughran | +| [YARN-10085](https://issues.apache.org/jira/browse/YARN-10085) | FS-CS converter: remove mixed ordering policy check | Critical | . | Peter Bacsko | Peter Bacsko | +| [YARN-10104](https://issues.apache.org/jira/browse/YARN-10104) | FS-CS converter: dry run should work without output defined | Major | . | Peter Bacsko | Peter Bacsko | +| [YARN-10015](https://issues.apache.org/jira/browse/YARN-10015) | Correct the sample command in SLS README file | Trivial | yarn | Aihua Xu | Aihua Xu | +| [YARN-10099](https://issues.apache.org/jira/browse/YARN-10099) | FS-CS converter: handle allow-undeclared-pools and user-as-default-queue properly and fix misc issues | Major | . | Peter Bacsko | Peter Bacsko | +| [YARN-8982](https://issues.apache.org/jira/browse/YARN-8982) | [Router] Add locality policy | Major | . | Giovanni Matteo Fumarola | Young Chen | +| [HADOOP-16732](https://issues.apache.org/jira/browse/HADOOP-16732) | S3Guard to support encrypted DynamoDB table | Major | fs/s3 | Mingliang Liu | Mingliang Liu | +| [HADOOP-16825](https://issues.apache.org/jira/browse/HADOOP-16825) | ITestAzureBlobFileSystemCheckAccess failing | Major | fs/azure, test | Steve Loughran | Bilahari T H | +| [HADOOP-16845](https://issues.apache.org/jira/browse/HADOOP-16845) | ITestAbfsClient.testContinuationTokenHavingEqualSign failing | Major | fs/azure, test | Sneha Vijayarajan | Sneha Vijayarajan | +| [HADOOP-16596](https://issues.apache.org/jira/browse/HADOOP-16596) | [pb-upgrade] Use shaded protobuf classes from hadoop-thirdparty dependency | Major | . | Vinayakumar B | Vinayakumar B | +| [YARN-10109](https://issues.apache.org/jira/browse/YARN-10109) | Allow stop and convert from leaf to parent queue in a single Mutation API call | Major | capacity scheduler | Prabhu Joseph | Prabhu Joseph | +| [YARN-10101](https://issues.apache.org/jira/browse/YARN-10101) | Support listing of aggregated logs for containers belonging to an application attempt | Major | log-aggregation, yarn | Adam Antal | Adam Antal | +| [YARN-10127](https://issues.apache.org/jira/browse/YARN-10127) | FSQueueConverter should not set App Ordering Policy to Parent Queue | Major | . | Prabhu Joseph | Peter Bacsko | +| [YARN-10022](https://issues.apache.org/jira/browse/YARN-10022) | Create RM Rest API to validate a CapacityScheduler Configuration | Major | . | Kinga Marton | Kinga Marton | +| [HDFS-13989](https://issues.apache.org/jira/browse/HDFS-13989) | RBF: Add FSCK to the Router | Major | . | Íñigo Goiri | Akira Ajisaka | +| [YARN-10029](https://issues.apache.org/jira/browse/YARN-10029) | Add option to UIv2 to get container logs from the new JHS API | Major | yarn | Adam Antal | Adam Antal | +| [HADOOP-16823](https://issues.apache.org/jira/browse/HADOOP-16823) | Large DeleteObject requests are their own Thundering Herd | Minor | fs/s3 | Steve Loughran | Steve Loughran | +| [HDFS-15173](https://issues.apache.org/jira/browse/HDFS-15173) | RBF: Delete repeated configuration 'dfs.federation.router.metrics.enable' | Minor | documentation, rbf | panlijie | panlijie | +| [HADOOP-15961](https://issues.apache.org/jira/browse/HADOOP-15961) | S3A committers: make sure there's regular progress() calls | Minor | fs/s3 | Steve Loughran | lqjacklee | +| [YARN-10139](https://issues.apache.org/jira/browse/YARN-10139) | ValidateAndGetSchedulerConfiguration API fails when cluster max allocation \> default 8GB | Major | . | Prabhu Joseph | Prabhu Joseph | +| [HADOOP-16706](https://issues.apache.org/jira/browse/HADOOP-16706) | ITestClientUrlScheme fails for accounts which don't support HTTP | Minor | fs/azure, test | Steve Loughran | Steve Loughran | +| [HADOOP-16711](https://issues.apache.org/jira/browse/HADOOP-16711) | S3A bucket existence checks to support v2 API and "no checks at all" | Minor | fs/s3 | Rajesh Balamohan | Mukund Thakur | +| [HDFS-15172](https://issues.apache.org/jira/browse/HDFS-15172) | Remove unnecessary deadNodeDetectInterval in DeadNodeDetector#checkDeadNodes() | Major | . | Lisheng Sun | Lisheng Sun | +| [HDFS-15041](https://issues.apache.org/jira/browse/HDFS-15041) | Make MAX\_LOCK\_HOLD\_MS and full queue size configurable | Major | namenode | zhuqi | zhuqi | +| [HADOOP-16853](https://issues.apache.org/jira/browse/HADOOP-16853) | ITestS3GuardOutOfBandOperations failing on versioned S3 buckets | Minor | fs/s3, test | Steve Loughran | Steve Loughran | +| [YARN-10157](https://issues.apache.org/jira/browse/YARN-10157) | FS-CS converter: initPropertyActions() is not called without rules file | Major | . | Peter Bacsko | Peter Bacsko | +| [YARN-10135](https://issues.apache.org/jira/browse/YARN-10135) | FS-CS converter tool: issue warning on dynamic auto-create mapping rules | Major | . | Peter Bacsko | Peter Bacsko | +| [YARN-10130](https://issues.apache.org/jira/browse/YARN-10130) | FS-CS converter: Do not allow output dir to be the same as input dir | Major | . | Szilard Nemeth | Adam Antal | +| [HDFS-14731](https://issues.apache.org/jira/browse/HDFS-14731) | [FGL] Remove redundant locking on NameNode. | Major | namenode | Konstantin Shvachko | Konstantin Shvachko | +| [HADOOP-16767](https://issues.apache.org/jira/browse/HADOOP-16767) | S3AInputStream reopening does not handle non IO exceptions properly | Major | . | Sergei Poganshev | Sergei Poganshev | +| [YARN-10175](https://issues.apache.org/jira/browse/YARN-10175) | FS-CS converter: only convert placement rules if a cmd line switch is defined | Major | . | Peter Bacsko | Peter Bacsko | +| [YARN-10167](https://issues.apache.org/jira/browse/YARN-10167) | FS-CS Converter: Need to validate c-s.xml after converting | Major | . | Wangda Tan | Peter Bacsko | +| [HADOOP-16905](https://issues.apache.org/jira/browse/HADOOP-16905) | Update jackson-databind to 2.10.3 to relieve us from the endless CVE patches | Major | . | Wei-Chiu Chuang | Wei-Chiu Chuang | +| [YARN-6924](https://issues.apache.org/jira/browse/YARN-6924) | Metrics for Federation AMRMProxy | Major | . | Giovanni Matteo Fumarola | Young Chen | +| [YARN-10168](https://issues.apache.org/jira/browse/YARN-10168) | FS-CS Converter: tool doesn't handle min/max resource conversion correctly | Blocker | . | Wangda Tan | Peter Bacsko | +| [HADOOP-16890](https://issues.apache.org/jira/browse/HADOOP-16890) | ABFS: Change in expiry calculation for MSI token provider | Minor | fs/azure | Bilahari T H | Bilahari T H | +| [YARN-10191](https://issues.apache.org/jira/browse/YARN-10191) | FS-CS converter: call System.exit function call for every code path in main method | Blocker | . | Peter Bacsko | Peter Bacsko | +| [YARN-10193](https://issues.apache.org/jira/browse/YARN-10193) | FS-CS converter: fix incorrect capacity conversion | Blocker | . | Peter Bacsko | Peter Bacsko | +| [YARN-10110](https://issues.apache.org/jira/browse/YARN-10110) | In Federation Secure cluster Application submission fails when authorization is enabled | Blocker | federation | Sushanta Sen | Bilwa S T | +| [YARN-9538](https://issues.apache.org/jira/browse/YARN-9538) | Document scheduler/app activities and REST APIs | Major | documentation | Tao Yang | Tao Yang | +| [YARN-9567](https://issues.apache.org/jira/browse/YARN-9567) | Add diagnostics for outstanding resource requests on app attempts page | Major | capacityscheduler | Tao Yang | Tao Yang | +| [HADOOP-16858](https://issues.apache.org/jira/browse/HADOOP-16858) | S3Guard fsck: Add option to remove orphaned entries | Major | fs/s3 | Gabor Bota | Gabor Bota | +| [HDFS-15088](https://issues.apache.org/jira/browse/HDFS-15088) | RBF: Correct annotation typo of RouterPermissionChecker#checkPermission | Trivial | rbf | Xiaoqiao He | Xiaoqiao He | +| [YARN-9879](https://issues.apache.org/jira/browse/YARN-9879) | Allow multiple leaf queues with the same name in CapacityScheduler | Major | . | Gergely Pollak | Gergely Pollak | +| [YARN-10197](https://issues.apache.org/jira/browse/YARN-10197) | FS-CS converter: fix emitted ordering policy string and max-am-resource percent value | Major | . | Peter Bacsko | Peter Bacsko | +| [YARN-10043](https://issues.apache.org/jira/browse/YARN-10043) | FairOrderingPolicy Improvements | Major | . | Manikandan R | Manikandan R | +| [HDFS-13470](https://issues.apache.org/jira/browse/HDFS-13470) | RBF: Add Browse the Filesystem button to the UI | Major | . | Íñigo Goiri | Íñigo Goiri | +| [HADOOP-15430](https://issues.apache.org/jira/browse/HADOOP-15430) | hadoop fs -mkdir -p path-ending-with-slash/ fails with s3guard | Minor | fs/s3 | Steve Loughran | Steve Loughran | +| [HADOOP-16939](https://issues.apache.org/jira/browse/HADOOP-16939) | fs.s3a.authoritative.path should support multiple FS URIs | Major | fs/s3 | Steve Loughran | Steve Loughran | +| [HADOOP-16859](https://issues.apache.org/jira/browse/HADOOP-16859) | ABFS: Add unbuffer support to AbfsInputStream | Major | fs/azure | Sahil Takiar | Sahil Takiar | +| [YARN-10120](https://issues.apache.org/jira/browse/YARN-10120) | In Federation Router Nodes/Applications/About pages throws 500 exception when https is enabled | Critical | federation | Sushanta Sen | Bilwa S T | +| [HADOOP-16465](https://issues.apache.org/jira/browse/HADOOP-16465) | Tune S3AFileSystem.listLocatedStatus | Major | fs/s3 | Steve Loughran | Mukund Thakur | +| [YARN-10234](https://issues.apache.org/jira/browse/YARN-10234) | FS-CS converter: don't enable auto-create queue property for root | Critical | . | Peter Bacsko | Peter Bacsko | +| [HADOOP-13873](https://issues.apache.org/jira/browse/HADOOP-13873) | log DNS addresses on s3a init | Minor | fs/s3 | Steve Loughran | Mukund Thakur | +| [HADOOP-16959](https://issues.apache.org/jira/browse/HADOOP-16959) | Resolve hadoop-cos dependency conflict | Major | build, fs/cos | Yang Yu | Yang Yu | +| [HADOOP-16986](https://issues.apache.org/jira/browse/HADOOP-16986) | s3a to not need wildfly on the classpath | Major | fs/s3 | Steve Loughran | Steve Loughran | +| [HDFS-14742](https://issues.apache.org/jira/browse/HDFS-14742) | RBF:TestRouterFaultTolerant tests are flaky | Major | test | Chen Zhang | Akira Ajisaka | +| [HADOOP-16794](https://issues.apache.org/jira/browse/HADOOP-16794) | S3A reverts KMS encryption to the bucket's default KMS key in rename/copy | Major | fs/s3 | Mukund Thakur | Mukund Thakur | +| [HADOOP-16920](https://issues.apache.org/jira/browse/HADOOP-16920) | ABFS: Make list page size configurable | Minor | fs/azure | Bilahari T H | Bilahari T H | +| [YARN-10194](https://issues.apache.org/jira/browse/YARN-10194) | YARN RMWebServices /scheduler-conf/validate leaks ZK Connections | Blocker | capacityscheduler | Akhil PB | Prabhu Joseph | +| [YARN-10215](https://issues.apache.org/jira/browse/YARN-10215) | Endpoint for obtaining direct URL for the logs | Major | yarn | Adam Antal | Andras Gyori | +| [HDFS-14353](https://issues.apache.org/jira/browse/HDFS-14353) | Erasure Coding: metrics xmitsInProgress become to negative. | Major | datanode, erasure-coding | maobaolong | maobaolong | +| [HADOOP-16953](https://issues.apache.org/jira/browse/HADOOP-16953) | HADOOP-16953. tune s3guard disabled warnings | Minor | fs/s3 | Steve Loughran | Steve Loughran | +| [HADOOP-16835](https://issues.apache.org/jira/browse/HADOOP-16835) | catch and downgrade all exceptions trying to load openssl native libs through wildfly | Minor | fs/azure, fs/s3 | Steve Loughran | Steve Loughran | + + +### OTHER: + +| JIRA | Summary | Priority | Component | Reporter | Contributor | +|:---- |:---- | :--- |:---- |:---- |:---- | +| [HDFS-13338](https://issues.apache.org/jira/browse/HDFS-13338) | Update BUILDING.txt for building native libraries | Critical | build, documentation, native | Takanobu Asanuma | Takanobu Asanuma | +| [HDFS-13403](https://issues.apache.org/jira/browse/HDFS-13403) | libhdfs++: Use hdfs::IoService object rather than asio::io\_service | Critical | . | James Clampffer | James Clampffer | +| [HDFS-13534](https://issues.apache.org/jira/browse/HDFS-13534) | libhdfs++: Fix GCC7 build | Major | . | James Clampffer | James Clampffer | +| [HADOOP-15816](https://issues.apache.org/jira/browse/HADOOP-15816) | Upgrade Apache Zookeeper version due to security concerns | Major | . | Boris Vulikh | Akira Ajisaka | +| [HADOOP-15882](https://issues.apache.org/jira/browse/HADOOP-15882) | Upgrade maven-shade-plugin from 2.4.3 to 3.2.0 | Major | . | Takanobu Asanuma | Takanobu Asanuma | +| [HADOOP-15815](https://issues.apache.org/jira/browse/HADOOP-15815) | Upgrade Eclipse Jetty version to 9.3.24 | Major | . | Boris Vulikh | Boris Vulikh | +| [HDFS-13870](https://issues.apache.org/jira/browse/HDFS-13870) | WebHDFS: Document ALLOWSNAPSHOT and DISALLOWSNAPSHOT API doc | Minor | documentation, webhdfs | Siyao Meng | Siyao Meng | +| [YARN-8489](https://issues.apache.org/jira/browse/YARN-8489) | Need to support "dominant" component concept inside YARN service | Major | yarn-native-services | Wangda Tan | Zac Zhou | +| [HDFS-12729](https://issues.apache.org/jira/browse/HDFS-12729) | Document special paths in HDFS | Major | documentation | Christopher Douglas | Masatake Iwasaki | +| [YARN-9191](https://issues.apache.org/jira/browse/YARN-9191) | Add cli option in DS to support enforceExecutionType in resource requests. | Major | . | Abhishek Modi | Abhishek Modi | +| [YARN-9428](https://issues.apache.org/jira/browse/YARN-9428) | Add metrics for paused containers in NodeManager | Major | . | Abhishek Modi | Abhishek Modi | +| [HDFS-14394](https://issues.apache.org/jira/browse/HDFS-14394) | Add -std=c99 / -std=gnu99 to libhdfs compile flags | Major | hdfs-client, libhdfs, native | Sahil Takiar | Sahil Takiar | +| [HADOOP-15242](https://issues.apache.org/jira/browse/HADOOP-15242) | Fix typos in hadoop-functions.sh | Trivial | . | Ray Chiang | Ray Chiang | +| [YARN-9433](https://issues.apache.org/jira/browse/YARN-9433) | Remove unused constants from RMAuditLogger | Minor | yarn | Adam Antal | Igor Rudenko | +| [HDFS-14433](https://issues.apache.org/jira/browse/HDFS-14433) | Remove the extra empty space in the DataStreamer logging | Trivial | hdfs | Yishuang Lu | Yishuang Lu | +| [YARN-9469](https://issues.apache.org/jira/browse/YARN-9469) | Fix typo in YarnConfiguration: physical memory | Trivial | yarn | Adam Antal | Igor Rudenko | +| [HADOOP-16263](https://issues.apache.org/jira/browse/HADOOP-16263) | Update BUILDING.txt with macOS native build instructions | Minor | . | Siyao Meng | Siyao Meng | +| [HADOOP-16365](https://issues.apache.org/jira/browse/HADOOP-16365) | Upgrade jackson-databind to 2.9.9 | Major | build | Shweta | Shweta | +| [YARN-9599](https://issues.apache.org/jira/browse/YARN-9599) | TestContainerSchedulerQueuing#testQueueShedding fails intermittently. | Minor | . | Abhishek Modi | Abhishek Modi | +| [YARN-9559](https://issues.apache.org/jira/browse/YARN-9559) | Create AbstractContainersLauncher for pluggable ContainersLauncher logic | Major | . | Jonathan Hung | Jonathan Hung | +| [HADOOP-16491](https://issues.apache.org/jira/browse/HADOOP-16491) | Upgrade jetty version to 9.3.27 | Major | . | Hrishikesh Gadre | Hrishikesh Gadre | +| [HADOOP-16351](https://issues.apache.org/jira/browse/HADOOP-16351) | Change ":" to ApplicationConstants.CLASS\_PATH\_SEPARATOR | Trivial | common | Kevin Su | Kevin Su | +| [HDFS-14729](https://issues.apache.org/jira/browse/HDFS-14729) | Upgrade Bootstrap and jQuery versions used in HDFS UIs | Major | ui | Vivek Ratnavel Subramanian | Vivek Ratnavel Subramanian | +| [HADOOP-16438](https://issues.apache.org/jira/browse/HADOOP-16438) | Introduce a config to control SSL Channel mode in Azure DataLake Store Gen1 | Major | fs/adl | Sneha Vijayarajan | Sneha Vijayarajan | +| [HADOOP-16542](https://issues.apache.org/jira/browse/HADOOP-16542) | Update commons-beanutils version to 1.9.4 | Major | . | Wei-Chiu Chuang | Kevin Su | +| [HADOOP-16555](https://issues.apache.org/jira/browse/HADOOP-16555) | Update commons-compress to 1.19 | Major | . | Wei-Chiu Chuang | YiSheng Lien | +| [YARN-9730](https://issues.apache.org/jira/browse/YARN-9730) | Support forcing configured partitions to be exclusive based on app node label | Major | . | Jonathan Hung | Jonathan Hung | +| [HADOOP-16675](https://issues.apache.org/jira/browse/HADOOP-16675) | Upgrade jackson-databind to 2.9.10.1 | Blocker | security | Wei-Chiu Chuang | Lisheng Sun | +| [HADOOP-16656](https://issues.apache.org/jira/browse/HADOOP-16656) | Document FairCallQueue configs in core-default.xml | Major | conf, documentation | Siyao Meng | Siyao Meng | +| [HDFS-14959](https://issues.apache.org/jira/browse/HDFS-14959) | [SBNN read] access time should be turned off | Major | documentation | Wei-Chiu Chuang | Chao Sun | +| [HADOOP-16654](https://issues.apache.org/jira/browse/HADOOP-16654) | Delete hadoop-ozone and hadoop-hdds subprojects from apache trunk | Major | . | Marton Elek | Sandeep Nemuri | +| [HDFS-15047](https://issues.apache.org/jira/browse/HDFS-15047) | Document the new decommission monitor (HDFS-14854) | Major | documentation | Wei-Chiu Chuang | Masatake Iwasaki | +| [HADOOP-16784](https://issues.apache.org/jira/browse/HADOOP-16784) | Update the year to 2020 | Major | . | Ayush Saxena | Ayush Saxena | +| [HADOOP-16358](https://issues.apache.org/jira/browse/HADOOP-16358) | Add an ARM CI for Hadoop | Major | build | Zhenyu Zheng | Zhenyu Zheng | +| [HADOOP-16803](https://issues.apache.org/jira/browse/HADOOP-16803) | Upgrade jackson-databind to 2.9.10.2 | Blocker | security | Akira Ajisaka | Masatake Iwasaki | +| [HADOOP-16670](https://issues.apache.org/jira/browse/HADOOP-16670) | Stripping Submarine code from Hadoop codebase. | Blocker | . | Wei-Chiu Chuang | Zhankun Tang | +| [HADOOP-16871](https://issues.apache.org/jira/browse/HADOOP-16871) | Upgrade Netty version to 4.1.45.Final to handle CVE-2019-20444,CVE-2019-16869 | Major | . | Aray Chenchu Sukesh | Aray Chenchu Sukesh | +| [HADOOP-16647](https://issues.apache.org/jira/browse/HADOOP-16647) | Support OpenSSL 1.1.1 LTS | Critical | security | Wei-Chiu Chuang | Rakesh Radhakrishnan | +| [HADOOP-16982](https://issues.apache.org/jira/browse/HADOOP-16982) | Update Netty to 4.1.48.Final | Blocker | . | Wei-Chiu Chuang | Lisheng Sun | +| [YARN-10247](https://issues.apache.org/jira/browse/YARN-10247) | Application priority queue ACLs are not respected | Blocker | capacity scheduler | Sunil G | Sunil G | + + diff --git a/hadoop-common-project/hadoop-common/src/site/markdown/release/3.3.0/RELEASENOTES.3.3.0.md b/hadoop-common-project/hadoop-common/src/site/markdown/release/3.3.0/RELEASENOTES.3.3.0.md new file mode 100644 index 0000000000000..371a8b08497d9 --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/site/markdown/release/3.3.0/RELEASENOTES.3.3.0.md @@ -0,0 +1,540 @@ + + +# Apache Hadoop 3.3.0 Release Notes + +These release notes cover new developer and user-facing incompatibilities, important issues, features, and major improvements. + + +--- + +* [YARN-8587](https://issues.apache.org/jira/browse/YARN-8587) | *Major* | **Delays are noticed to launch docker container** + +add "docker inspect" retries to discover container exit code. + + +--- + +* [HDFS-14053](https://issues.apache.org/jira/browse/HDFS-14053) | *Major* | **Provide ability for NN to re-replicate based on topology changes.** + +A new option (-replicate) is added to fsck command to re-trigger the replication for mis-replicated blocks. This option should be used instead of previous workaround of increasing and then decreasing replication factor (using hadoop fs -setrep command). + + +--- + +* [HADOOP-15358](https://issues.apache.org/jira/browse/HADOOP-15358) | *Critical* | **SFTPConnectionPool connections leakage** + +Fixed SFTPConnectionPool connections leakage + + +--- + +* [YARN-8986](https://issues.apache.org/jira/browse/YARN-8986) | *Minor* | **publish all exposed ports to random ports when using bridge network** + +support -p and -P for bridge type network; + + +--- + +* [MAPREDUCE-6190](https://issues.apache.org/jira/browse/MAPREDUCE-6190) | *Major* | **If a task stucks before its first heartbeat, it never timeouts and the MR job becomes stuck** + +Added "mapreduce.task.stuck.timeout-ms" parameter to timeout a task before sending the task's first heartbeat. The default value is 600000 milliseconds (10 minutes). + + +--- + +* [YARN-9071](https://issues.apache.org/jira/browse/YARN-9071) | *Critical* | **NM and service AM don't have updated status for reinitialized containers** + +In progress upgrade status may show READY state sooner than actual upgrade operations. External caller to upgrade API is recommended to wait minimum 30 seconds before querying yarn app -status. + + +--- + +* [HADOOP-15428](https://issues.apache.org/jira/browse/HADOOP-15428) | *Major* | **s3guard bucket-info will create s3guard table if FS is set to do this automatically** + +If -unguarded flag is passed to \`hadoop s3guard bucket-info\`, it will now proceed with S3Guard disabled instead of failing if S3Guard is not already disabled. + + +--- + +* [HADOOP-16000](https://issues.apache.org/jira/browse/HADOOP-16000) | *Major* | **Remove TLSv1 and SSLv2Hello from the default value of hadoop.ssl.enabled.protocols** + +TLSv1 and SSLv2Hello were removed from the default value of "hadoop.ssl.enabled.protocols". + + +--- + +* [YARN-9084](https://issues.apache.org/jira/browse/YARN-9084) | *Major* | **Service Upgrade: With default readiness check, the status of upgrade is reported to be successful prematurely** + +Improve transient container status accuracy for upgrade. + + +--- + +* [YARN-8762](https://issues.apache.org/jira/browse/YARN-8762) | *Major* | **[Umbrella] Support Interactive Docker Shell to running Containers** + +- Add shell access to YARN containers + + +--- + +* [HADOOP-15965](https://issues.apache.org/jira/browse/HADOOP-15965) | *Major* | **Upgrade to ADLS SDK which has major performance improvement for ingress/egress** + + + + +--- + +* [HADOOP-15996](https://issues.apache.org/jira/browse/HADOOP-15996) | *Major* | **Plugin interface to support more complex usernames in Hadoop** + +This patch enables "Hadoop" and "MIT" as options for "hadoop.security.auth\_to\_local.mechanism" and defaults to 'hadoop'. This should be backward compatible with pre-HADOOP-12751. + +This is basically HADOOP-12751 plus configurable + extended tests. + + +--- + +* [YARN-8489](https://issues.apache.org/jira/browse/YARN-8489) | *Major* | **Need to support "dominant" component concept inside YARN service** + +- Improved YARN service status report based on dominant component status. + + +--- + +* [HADOOP-15922](https://issues.apache.org/jira/browse/HADOOP-15922) | *Major* | **DelegationTokenAuthenticationFilter get wrong doAsUser since it does not decode URL** + +- Fix DelegationTokenAuthentication filter for incorrectly double encode doAs user parameter. + + +--- + +* [YARN-9116](https://issues.apache.org/jira/browse/YARN-9116) | *Major* | **Capacity Scheduler: implements queue level maximum-allocation inheritance** + +After this change, capacity scheduler queue is able to inherit and override max-allocation from its parent queue. The subqueue's max-allocation can be larger or smaller than the parent queue's but can't exceed the global "yarn.scheduler.capacity.maximum-allocation". User can set queue-level max-allocation on any level of queues, using configuration property "yarn.scheduler.capacity.%QUEUE\_PATH%.maximum-allocation". And this property allows user to set max-resource-allocation for customized resource types, e.g "memory=20G,vcores=20,gpu=3". + + +--- + +* [HDFS-14158](https://issues.apache.org/jira/browse/HDFS-14158) | *Minor* | **Checkpointer ignores configured time period \> 5 minutes** + +Fix the Checkpointer not to ignore the configured "dfs.namenode.checkpoint.period" \> 5 minutes + + +--- + +* [YARN-8761](https://issues.apache.org/jira/browse/YARN-8761) | *Major* | **Service AM support for decommissioning component instances** + +- Component instance number is not linear increment when decommission feature is used. Application with assumption of linear increment component instance number maybe impacted by introduction of this feature. + + +--- + +* [HDFS-14273](https://issues.apache.org/jira/browse/HDFS-14273) | *Trivial* | **Fix checkstyle issues in BlockLocation's method javadoc** + +Thanks for the patch, [~shwetayakkali], and review, [~knanasi]. Committed to trunk. + + +--- + +* [HDFS-14118](https://issues.apache.org/jira/browse/HDFS-14118) | *Major* | **Support using DNS to resolve nameservices to IP addresses** + +HDFS clients can use a single domain name to discover servers (namenodes/routers/observers) instead of explicitly listing out all hosts in the config + + +--- + +* [HDFS-7133](https://issues.apache.org/jira/browse/HDFS-7133) | *Major* | **Support clearing namespace quota on "/"** + +Namespace Quota on root can be cleared now. + + +--- + +* [HADOOP-16011](https://issues.apache.org/jira/browse/HADOOP-16011) | *Major* | **OsSecureRandom very slow compared to other SecureRandom implementations** + +The default RNG is now OpensslSecureRandom instead of OsSecureRandom.The high-performance hardware random number generator (RDRAND instruction) will be used if available. If not, it will fall back to OpenSSL secure random generator. +If you insist on using OsSecureRandom, set hadoop.security.secure.random.impl in core-site.xml to org.apache.hadoop.crypto.random.OsSecureRandom. + + +--- + +* [HADOOP-16210](https://issues.apache.org/jira/browse/HADOOP-16210) | *Critical* | **Update guava to 27.0-jre in hadoop-project trunk** + +Guava has been updated to 27.0. Code built against this new version may not link against older releases because of new overloads of methods like Preconditions.checkArgument(). + + +--- + +* [HADOOP-16085](https://issues.apache.org/jira/browse/HADOOP-16085) | *Major* | **S3Guard: use object version or etags to protect against inconsistent read after replace/overwrite** + +S3Guard will now track the etag of uploaded files and, if an S3 bucket is versioned, the object version. You can then control how to react to a mismatch between the data in the DynamoDB table and that in the store: warn, fail, or, when using versions, return the original value. + +This adds two new columns to the table: etag and version. This is transparent to older S3A clients -but when such clients add/update data to the S3Guard table, they will not add these values. As a result, the etag/version checks will not work with files uploaded by older clients. + +For a consistent experience, upgrade all clients to use the latest hadoop version. + + +--- + +* [HADOOP-15563](https://issues.apache.org/jira/browse/HADOOP-15563) | *Major* | **S3Guard to support creating on-demand DDB tables** + +S3Guard now defaults to creating DynamoDB tables as "On-Demand", rather than with a prepaid IO capacity. This reduces costs when idle to only the storage of the metadata entries, while delivering significantly faster performance during query planning and other bursts of IO. Consult the S3Guard documentation for further details. + + +--- + +* [YARN-9578](https://issues.apache.org/jira/browse/YARN-9578) | *Major* | **Add limit/actions/summarize options for app activities REST API** + +This patch changes the format of app activities query URL to http://{rm-host}:{port}/ws/v1/cluster/scheduler/app-activities/{app-id}/, app-id now is a path parameter instead of a query parameter. + + +--- + +* [HDFS-14339](https://issues.apache.org/jira/browse/HDFS-14339) | *Major* | **Inconsistent log level practices in RpcProgramNfs3.java** + +**WARNING: No release note provided for this change.** + + +--- + +* [HDFS-14403](https://issues.apache.org/jira/browse/HDFS-14403) | *Major* | **Cost-Based RPC FairCallQueue** + +This adds an extension to the IPC FairCallQueue which allows for the consideration of the \*cost\* of a user's operations when deciding how they should be prioritized, as opposed to the number of operations. This can be helpful for protecting the NameNode from clients which submit very expensive operations (e.g. large listStatus operations or recursive getContentSummary operations). + +This can be enabled by setting the \`ipc.\.costprovder.impl\` configuration to \`org.apache.hadoop.ipc.WeightedTimeCostProvider\`. + + +--- + +* [HADOOP-16460](https://issues.apache.org/jira/browse/HADOOP-16460) | *Major* | **ABFS: fix for Sever Name Indication (SNI)** + +ABFS: Bug fix to support Server Name Indication (SNI). + + +--- + +* [HADOOP-16452](https://issues.apache.org/jira/browse/HADOOP-16452) | *Major* | **Increase ipc.maximum.data.length default from 64MB to 128MB** + +Default ipc.maximum.data.length is now 128 MB in order to accommodate huge block reports. + + +--- + +* [HDFS-13783](https://issues.apache.org/jira/browse/HDFS-13783) | *Major* | **Balancer: make balancer to be a long service process for easy to monitor it.** + +Adds a new parameter to the Balancer CLI, "-asService", to enable the process to be long-running. + + +--- + +* [HADOOP-16398](https://issues.apache.org/jira/browse/HADOOP-16398) | *Major* | **Exports Hadoop metrics to Prometheus** + +If "hadoop.prometheus.endpoint.enabled" is set to true, Prometheus-friendly formatted metrics can be obtained from '/prom' endpoint of Hadoop daemons. The default value of the property is false. + + +--- + +* [HADOOP-16479](https://issues.apache.org/jira/browse/HADOOP-16479) | *Major* | **ABFS FileStatus.getModificationTime returns localized time instead of UTC** + +private long parseLastModifiedTime(final String lastModifiedTime) + +Timezone is part of lastModifiedTime String as it's last field. But when parsed it ignores timezone field and assumes JVM timezone. +Fix: Made timezone field considered in lastModifiedTime when the same is parsed + + +--- + +* [HADOOP-16499](https://issues.apache.org/jira/browse/HADOOP-16499) | *Critical* | **S3A retry policy to be exponential** + +The S3A filesystem now backs off exponentially on failures. if you have customized the fs.s3a.retry.limit and fs.s3a.retry.interval options you may wish to review these settings + + +--- + +* [HDFS-13505](https://issues.apache.org/jira/browse/HDFS-13505) | *Major* | **Turn on HDFS ACLs by default.** + +By default, dfs.namenode.acls.enabled is now set to true. If you haven't set dfs.namenode.acls.enabled=false before and want to keep ACLs feature disabled, you must explicitly set it to false in hdfs-site.xml. i.e. + \ + \dfs.namenode.acls.enabled\ + \false\ + \ + + +--- + +* [YARN-2599](https://issues.apache.org/jira/browse/YARN-2599) | *Major* | **Standby RM should expose jmx endpoint** + +YARN /jmx URL end points will be accessible per resource manager process. Hence there will not be any redirection to active resource manager while accessing /jmx endpoints. + + +--- + +* [HDFS-13101](https://issues.apache.org/jira/browse/HDFS-13101) | *Critical* | **Yet another fsimage corruption related to snapshot** + +Fix a corner case in deleting HDFS snapshots. A regression was later found and fixed by HDFS-15012. + + +--- + +* [HDFS-14675](https://issues.apache.org/jira/browse/HDFS-14675) | *Major* | **Increase Balancer Defaults Further** + +Increase default bandwidth limit for rebalancing per DataNode (dfs.datanode.balance.bandwidthPerSec) from 10MB/s to 100MB/s. + +Increase default maximum threads of DataNode balancer (dfs.datanode.balance.max.concurrent.moves) from 50 to 100. + + +--- + +* [HDFS-14617](https://issues.apache.org/jira/browse/HDFS-14617) | *Major* | **Improve fsimage load time by writing sub-sections to the fsimage index** + +This change allows the inode and inode directory sections of the fsimage to be loaded in parallel. Tests on large images have shown this change to reduce the image load time to about 50% of the pre-change run time. + +It works by writing sub-section entries to the image index, effectively splitting each image section into many sub-sections which can be processed in parallel. By default 12 sub-sections per image section are created when the image is saved, and 4 threads are used to load the image at startup. + +This is disabled by default for any image with more than 1M inodes (dfs.image.parallel.inode.threshold) and can be enabled by setting dfs.image.parallel.load to true. When the feature is enabled, the next HDFS checkpoint will write the image sub-sections and subsequent namenode restarts can load the image in parallel. + +A image with the parallel sections can be read even if the feature is disabled, but HDFS versions without this Jira cannot load an image with parallel sections. OIV can process a parallel enabled image without issues. + +Key configuration parameters are: + +dfs.image.parallel.load=false - enable or disable the feature + +dfs.image.parallel.target.sections = 12 - The target number of subsections. Aim for 2 to 3 times the number of dfs.image.parallel.threads. + +dfs.image.parallel.inode.threshold = 1000000 - Only save and load in parallel if the image has more than this number of inodes. + +dfs.image.parallel.threads = 4 - The number of threads used to load the image. Testing has shown 4 to be optimal, but this may depends on the environment + + +--- + +* [HDFS-14396](https://issues.apache.org/jira/browse/HDFS-14396) | *Blocker* | **Failed to load image from FSImageFile when downgrade from 3.x to 2.x** + +During a rolling upgrade from Hadoop 2.x to 3.x, NameNode cannot persist erasure coding information, and therefore a user cannot start using erasure coding feature until finalize is done. + + +--- + +* [HADOOP-16554](https://issues.apache.org/jira/browse/HADOOP-16554) | *Major* | **mvn javadoc:javadoc fails in hadoop-aws** + +Fixed the link for javadoc of hadoop-aws CommitOperations + + +--- + +* [HADOOP-16557](https://issues.apache.org/jira/browse/HADOOP-16557) | *Major* | **[pb-upgrade] Upgrade protobuf.version to 3.7.1** + +Upgraded the protobuf to 3.7.1. + + +--- + +* [HDFS-14845](https://issues.apache.org/jira/browse/HDFS-14845) | *Critical* | **Ignore AuthenticationFilterInitializer for HttpFSServerWebServer and honor hadoop.http.authentication configs** + +httpfs.authentication.\* configs become deprecated and hadoop.http.authentication.\* configs are honored in HttpFS. If the both configs are set, httpfs.authentication.\* configs are effective for compatibility. + + +--- + +* [HADOOP-15616](https://issues.apache.org/jira/browse/HADOOP-15616) | *Major* | **Incorporate Tencent Cloud COS File System Implementation** + +Tencent cloud is top 2 cloud vendors in China market and the object store COS is widely used among China’s cloud users. This task implements a COSN filesytem to support Tencent cloud COS natively in Hadoop. With simple configuration, Hadoop applications, like Spark and Hive, can read/write data from COS without any code change. + + +--- + +* [HDFS-13762](https://issues.apache.org/jira/browse/HDFS-13762) | *Major* | **Support non-volatile storage class memory(SCM) in HDFS cache directives** + +Non-volatile storage class memory (SCM, also known as persistent memory) is supported in HDFS cache. To enable SCM cache, user just needs to configure SCM volume for property “dfs.datanode.cache.pmem.dirs” in hdfs-site.xml. And all HDFS cache directives keep unchanged. There are two implementations for HDFS SCM Cache, one is pure java code implementation and the other is native PMDK based implementation. The latter implementation can bring user better performance gain in cache write and cache read. If PMDK native libs could be loaded, it will use PMDK based implementation otherwise it will fallback to java code implementation. To enable PMDK based implementation, user should install PMDK library by referring to the official site http://pmem.io/. Then, build Hadoop with PMDK support by referring to "PMDK library build options" section in \`BUILDING.txt\` in the source code. If multiple SCM volumes are configured, a round-robin policy is used to select an available volume for caching a block. Consistent with DRAM cache, SCM cache also has no cache eviction mechanism. When DataNode receives a data read request from a client, if the corresponding block is cached into SCM, DataNode will instantiate an InputStream with the block location path on SCM (pure java implementation) or cache address on SCM (PMDK based implementation). Once the InputStream is created, DataNode will send the cached data to the client. Please refer "Centralized Cache Management" guide for more details. + + +--- + +* [HDFS-14890](https://issues.apache.org/jira/browse/HDFS-14890) | *Blocker* | **Setting permissions on name directory fails on non posix compliant filesystems** + +- Fixed namenode/journal startup on Windows. + + +--- + +* [HADOOP-16152](https://issues.apache.org/jira/browse/HADOOP-16152) | *Major* | **Upgrade Eclipse Jetty version to 9.4.x** + +Upgraded Jetty to 9.4.20.v20190813. +Downstream applications that still depend on Jetty 9.3.x may break. + + +--- + +* [HDFS-12943](https://issues.apache.org/jira/browse/HDFS-12943) | *Major* | **Consistent Reads from Standby Node** + +Observer is a new type of a NameNode in addition to Active and Standby Nodes in HA settings. An Observer Node maintains a replica of the namespace same as a Standby Node. It additionally allows execution of clients read requests. + +To ensure read-after-write consistency within a single client, a state ID is introduced in RPC headers. The Observer responds to the client request only after its own state has caught up with the client’s state ID, which it previously received from the Active NameNode. + +Clients can explicitly invoke a new client protocol call msync(), which ensures that subsequent reads by this client from an Observer are consistent. + +A new client-side ObserverReadProxyProvider is introduced to provide automatic switching between Active and Observer NameNodes for submitting respectively write and read requests. + + +--- + +* [HDFS-13571](https://issues.apache.org/jira/browse/HDFS-13571) | *Major* | **Deadnode detection** + +When dead node blocks DFSInputStream, Deadnode detection can find it and share this information to other DFSInputStreams in the same DFSClient. Thus, these DFSInputStreams will not read from the dead node and be blocked by this dead node. + + +--- + +* [HADOOP-16771](https://issues.apache.org/jira/browse/HADOOP-16771) | *Major* | **Update checkstyle to 8.26 and maven-checkstyle-plugin to 3.1.0** + +Updated checkstyle to 8.26 and updated maven-checkstyle-plugin to 3.1.0. + + +--- + +* [YARN-10036](https://issues.apache.org/jira/browse/YARN-10036) | *Major* | **Install yarnpkg and upgrade nodejs in Dockerfile** + +In the Dockerfile, nodejs is upgraded to 8.17.0 and yarn 1.12.1 is installed. + + +--- + +* [HADOOP-16621](https://issues.apache.org/jira/browse/HADOOP-16621) | *Critical* | **[pb-upgrade] Remove Protobuf classes from signatures of Public APIs** + +Following APIs have been removed from Token.java to avoid protobuf classes in signature. +1. o.a.h.security.token.Token(TokenProto tokenPB) +2. o.a.h.security.token.Token.toTokenProto() + + +--- + +* [HADOOP-16670](https://issues.apache.org/jira/browse/HADOOP-16670) | *Blocker* | **Stripping Submarine code from Hadoop codebase.** + +The Submarine subproject is moved to its own Apache Top Level Project. Therefore, the Submarine code is removed from Hadoop 3.3.0 and forward. Please visit https://submarine.apache.org/ for more information. + + +--- + +* [YARN-8472](https://issues.apache.org/jira/browse/YARN-8472) | *Major* | **YARN Container Phase 2** + +- Improved debugging Docker container on YARN +- Improved security for running Docker containers +- Improved cgroup management for docker container. + + +--- + +* [HADOOP-16732](https://issues.apache.org/jira/browse/HADOOP-16732) | *Major* | **S3Guard to support encrypted DynamoDB table** + +Support server-side encrypted DynamoDB table for S3Guard. Users don't need to do anything (provide any configuration or change application code) if they don't want to enable server side encryption. Existing tables and the default configuration values will keep existing behavior, which is encrypted using Amazon owned customer master key (CMK). + +To enable server side encryption, users can set "fs.s3a.s3guard.ddb.table.sse.enabled" as true. This uses Amazon managed CMK "alias/aws/dynamodb". When it's enabled, a user can also specify her own custom KMS CMK with config "fs.s3a.s3guard.ddb.table.sse.cmk". + + +--- + +* [HADOOP-16596](https://issues.apache.org/jira/browse/HADOOP-16596) | *Major* | **[pb-upgrade] Use shaded protobuf classes from hadoop-thirdparty dependency** + +All protobuf classes will be used from hadooop-shaded-protobuf\_3\_7 artifact with package prefix as 'org.apache.hadoop.thirdparty.protobuf' instead of 'com.google.protobuf' + + +--- + +* [HADOOP-16823](https://issues.apache.org/jira/browse/HADOOP-16823) | *Minor* | **Large DeleteObject requests are their own Thundering Herd** + +The page size for bulk delete operations has been reduced from 1000 to 250 to reduce the likelihood of overloading an S3 partition, especially because the retry policy on throttling is simply to try again. + +The page size can be set in "fs.s3a.bulk.delete.page.size" + +There is also an option to control whether or not the AWS client retries requests, or whether it is handled exclusively in the S3A code. This option "fs.s3a.experimental.aws.s3.throttling" is true by default. If set to false: everything is handled in the S3A client. While this means that metrics may be more accurate, it may mean that throttling failures in helper threads of the AWS SDK (especially those used in copy/rename) may not be handled properly. This is experimental, and should be left at "true" except when seeking more detail about throttling rates. + + +--- + +* [HADOOP-16850](https://issues.apache.org/jira/browse/HADOOP-16850) | *Major* | **Support getting thread info from thread group for JvmMetrics to improve the performance** + +Add a new configuration "hadoop.metrics.jvm.use-thread-mxbean". If set to true, ThreadMXBean is used for getting thread info in JvmMetrics, otherwise, ThreadGroup is used. The default value is false (use ThreadGroup for better performance). + + +--- + +* [HADOOP-16711](https://issues.apache.org/jira/browse/HADOOP-16711) | *Minor* | **S3A bucket existence checks to support v2 API and "no checks at all"** + +The probe for an S3 bucket existing now uses the V2 API, which fails fast if the caller lacks access to the bucket. +The property fs.s3a.bucket.probe can be used to change this. If using a third party library which doesn't support this API call, change it to "1". + +to skip the probe entirely, use "0". This will make filesystem instantiation slightly faster, at a cost of postponing all issues related to bucket existence and client authentication until the filesystem API calls are first used. + + +--- + +* [HDFS-15186](https://issues.apache.org/jira/browse/HDFS-15186) | *Critical* | **Erasure Coding: Decommission may generate the parity block's content with all 0 in some case** + +**WARNING: No release note provided for this change.** + + +--- + +* [HDFS-14743](https://issues.apache.org/jira/browse/HDFS-14743) | *Critical* | **Enhance INodeAttributeProvider/ AccessControlEnforcer Interface in HDFS to support Authorization of mkdir, rm, rmdir, copy, move etc...** + +A new INodeAttributeProvider API checkPermissionWithContext(AuthorizationContext) is added. Authorization provider implementations may implement this API to get additional context (operation name and caller context) of an authorization request. + + +--- + +* [HDFS-14820](https://issues.apache.org/jira/browse/HDFS-14820) | *Major* | ** The default 8KB buffer of BlockReaderRemote#newBlockReader#BufferedOutputStream is too big** + +Reduce the output stream buffer size of a DFSClient remote read from 8KB to 512 bytes. + + +--- + +* [HADOOP-16661](https://issues.apache.org/jira/browse/HADOOP-16661) | *Major* | **Support TLS 1.3** + +Starting with Hadoop 3.3.0, TLS 1.3 is supported on Java 11 Runtime (11.0.3 and above). +This support does not include cloud connectors. + +To enable TLSv1.3, add to the core-site.xml: +\ + \hadoop.ssl.enabled.protocols\ + \TLSv1.3\ +\ + + +--- + +* [HADOOP-15620](https://issues.apache.org/jira/browse/HADOOP-15620) | *Major* | **Über-jira: S3A phase VI: Hadoop 3.3 features** + +Lots of enhancements to the S3A code, including +\* Delegation Token support +\* better handling of 404 caching +\* S3guard performance, resilience improvements + + +--- + +* [HADOOP-16986](https://issues.apache.org/jira/browse/HADOOP-16986) | *Major* | **s3a to not need wildfly on the classpath** + +hadoop-aws can use native openssl libraries for better HTTPS performance -consult the S3A performance document for details. + +To enable this, wildfly.jar is declared as a compile-time dependency of the hadoop-aws module, so ensuring it ends up on the classpath of the hadoop command line, distribution packages and downstream modules. + +It is however, still optional, unless fs.s3a.ssl.channel.mode is set to openssl + + + diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FCStatisticsBaseTest.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FCStatisticsBaseTest.java index 2aa5407c056af..dc12f44fc2758 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FCStatisticsBaseTest.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FCStatisticsBaseTest.java @@ -178,7 +178,8 @@ public Boolean get() { * * @param stats */ - protected abstract void verifyWrittenBytes(Statistics stats); + protected abstract void verifyWrittenBytes(Statistics stats) + throws IOException; /** * Returns the filesystem uri. Should be set diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestJsonSerialization.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestJsonSerialization.java index 991697d96bc95..4a106e8fdf1f3 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestJsonSerialization.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestJsonSerialization.java @@ -28,9 +28,11 @@ import org.junit.Test; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.LocalFileSystem; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.PathIOException; import org.apache.hadoop.test.HadoopTestBase; import org.apache.hadoop.test.LambdaTestUtils; @@ -151,6 +153,9 @@ public void testEmptyFile() throws Throwable { } } + /** + * round trip through both load APIs. + */ @Test public void testFileSystemRoundTrip() throws Throwable { File tempFile = File.createTempFile("Keyval", ".json"); @@ -159,19 +164,30 @@ public void testFileSystemRoundTrip() throws Throwable { LocalFileSystem fs = FileSystem.getLocal(new Configuration()); try { serDeser.save(fs, tempPath, source, false); - assertEquals(source, serDeser.load(fs, tempPath)); + assertEquals("JSON loaded with load(fs, path)", + source, + serDeser.load(fs, tempPath)); + assertEquals("JSON loaded with load(fs, path, status)", + source, + serDeser.load(fs, tempPath, fs.getFileStatus(tempPath))); } finally { fs.delete(tempPath, false); } } + /** + * 0 byte file through the load(path) API will fail with a wrapped + * Parser exception. + * 0 byte file through the load(path, status) API will fail with a wrapped + * Parser exception. + */ @Test public void testFileSystemEmptyPath() throws Throwable { File tempFile = File.createTempFile("Keyval", ".json"); Path tempPath = new Path(tempFile.toURI()); LocalFileSystem fs = FileSystem.getLocal(new Configuration()); try { - LambdaTestUtils.intercept(EOFException.class, + LambdaTestUtils.intercept(PathIOException.class, () -> serDeser.load(fs, tempPath)); fs.delete(tempPath, false); LambdaTestUtils.intercept(FileNotFoundException.class, @@ -181,5 +197,23 @@ public void testFileSystemEmptyPath() throws Throwable { } } + /** + * 0 byte file through the load(path, status) API will fail with an + * EOFException. + */ + @Test + public void testFileSystemEmptyStatus() throws Throwable { + File tempFile = File.createTempFile("Keyval", ".json"); + Path tempPath = new Path(tempFile.toURI()); + LocalFileSystem fs = FileSystem.getLocal(new Configuration()); + try { + final FileStatus st = fs.getFileStatus(tempPath); + LambdaTestUtils.intercept(EOFException.class, + () -> serDeser.load(fs, tempPath, st)); + } finally { + fs.delete(tempPath, false); + } + } + } diff --git a/hadoop-common-project/hadoop-kms/pom.xml b/hadoop-common-project/hadoop-kms/pom.xml index fedb0375ef72d..d15ac0c97c838 100644 --- a/hadoop-common-project/hadoop-kms/pom.xml +++ b/hadoop-common-project/hadoop-kms/pom.xml @@ -267,8 +267,6 @@ true Use hadoop-common provided implementations rather than the one provided by Guava - org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting - com.google.common.annotations.VisibleForTesting org.apache.hadoop.thirdparty.com.google.common.base.Preconditions com.google.common.base.Preconditions diff --git a/hadoop-common-project/hadoop-nfs/pom.xml b/hadoop-common-project/hadoop-nfs/pom.xml index 600b2ae1a43ea..757a06b00f4a5 100644 --- a/hadoop-common-project/hadoop-nfs/pom.xml +++ b/hadoop-common-project/hadoop-nfs/pom.xml @@ -137,8 +137,6 @@ true Use hadoop-common provided implementations rather than the one provided by Guava - org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting - com.google.common.annotations.VisibleForTesting org.apache.hadoop.thirdparty.com.google.common.base.Preconditions com.google.common.base.Preconditions diff --git a/hadoop-common-project/hadoop-registry/pom.xml b/hadoop-common-project/hadoop-registry/pom.xml index 753b2b28fb0f8..7623bfd6d3601 100644 --- a/hadoop-common-project/hadoop-registry/pom.xml +++ b/hadoop-common-project/hadoop-registry/pom.xml @@ -294,8 +294,6 @@ true Use hadoop-common provided implementations rather than the one provided by Guava - org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting - com.google.common.annotations.VisibleForTesting org.apache.hadoop.thirdparty.com.google.common.base.Preconditions com.google.common.base.Preconditions diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/pom.xml b/hadoop-hdfs-project/hadoop-hdfs-client/pom.xml index f521480281810..c7ddf06061525 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/pom.xml +++ b/hadoop-hdfs-project/hadoop-hdfs-client/pom.xml @@ -201,8 +201,8 @@ https://maven.apache.org/xsd/maven-4.0.0.xsd"> true Use hadoop-common provided VisibleForTesting rather than the one provided by Guava - org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting - com.google.common.annotations.VisibleForTesting + org.apache.hadoop.thirdparty.com.google.common.base.Preconditions + com.google.common.base.Preconditions diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/ClientContext.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/ClientContext.java index 5a8d033c993f7..50e7025f23f43 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/ClientContext.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/ClientContext.java @@ -43,7 +43,7 @@ import org.apache.hadoop.util.ReflectionUtils; import org.apache.hadoop.classification.VisibleForTesting; -import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; +import org.apache.hadoop.util.Preconditions; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java index 2fa67d9449f77..b0b44b473217d 100755 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java @@ -196,7 +196,7 @@ import org.apache.hadoop.classification.VisibleForTesting; import org.apache.hadoop.thirdparty.com.google.common.base.Joiner; -import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; +import org.apache.hadoop.util.Preconditions; import org.apache.hadoop.thirdparty.com.google.common.net.InetAddresses; /******************************************************** diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java index 0cf11d4c19687..0deaa41cf4175 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java @@ -73,7 +73,7 @@ import org.slf4j.LoggerFactory; import org.apache.hadoop.classification.VisibleForTesting; -import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; +import org.apache.hadoop.util.Preconditions; import static org.apache.hadoop.hdfs.client.HdfsClientConfigKeys.Write.RECOVER_LEASE_ON_CLOSE_EXCEPTION_DEFAULT; import static org.apache.hadoop.hdfs.client.HdfsClientConfigKeys.Write.RECOVER_LEASE_ON_CLOSE_EXCEPTION_KEY; diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSStripedOutputStream.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSStripedOutputStream.java index bd7ff048dc8a9..0f60027269d8f 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSStripedOutputStream.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSStripedOutputStream.java @@ -18,7 +18,7 @@ package org.apache.hadoop.hdfs; import org.apache.hadoop.classification.VisibleForTesting; -import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; +import org.apache.hadoop.util.Preconditions; import org.apache.hadoop.HadoopIllegalArgumentException; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.fs.CreateFlag; diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSUtilClient.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSUtilClient.java index 6f54de05eb6ac..7c71c64ec3657 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSUtilClient.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSUtilClient.java @@ -17,9 +17,11 @@ */ package org.apache.hadoop.hdfs; +import org.apache.commons.collections.list.TreeList; +import org.apache.hadoop.ipc.RpcNoSuchMethodException; import org.apache.hadoop.net.DomainNameResolver; import org.apache.hadoop.thirdparty.com.google.common.base.Joiner; -import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; +import org.apache.hadoop.util.Preconditions; import org.apache.hadoop.thirdparty.com.google.common.collect.Maps; import org.apache.hadoop.thirdparty.com.google.common.primitives.SignedBytes; import java.net.URISyntaxException; @@ -30,6 +32,7 @@ import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hdfs.client.HdfsClientConfigKeys; +import org.apache.hadoop.hdfs.client.impl.SnapshotDiffReportGenerator; import org.apache.hadoop.hdfs.net.BasicInetPeer; import org.apache.hadoop.hdfs.net.NioInetPeer; import org.apache.hadoop.hdfs.net.Peer; @@ -43,8 +46,11 @@ import org.apache.hadoop.hdfs.protocol.LocatedBlocks; import org.apache.hadoop.hdfs.protocol.datatransfer.IOStreamPair; import org.apache.hadoop.hdfs.protocol.ReconfigurationProtocol; +import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport; +import org.apache.hadoop.hdfs.protocol.SnapshotDiffReportListing; import org.apache.hadoop.hdfs.protocol.datatransfer.sasl.DataEncryptionKeyFactory; import org.apache.hadoop.hdfs.protocol.datatransfer.sasl.SaslDataTransferClient; +import org.apache.hadoop.hdfs.protocol.SnapshotDiffReportListing.DiffReportListingEntry; import org.apache.hadoop.hdfs.protocolPB.ClientDatanodeProtocolTranslatorPB; import org.apache.hadoop.hdfs.protocolPB.ReconfigurationProtocolTranslatorPB; import org.apache.hadoop.hdfs.security.token.block.BlockTokenIdentifier; @@ -55,6 +61,7 @@ import org.apache.hadoop.net.NodeBase; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.security.token.Token; +import org.apache.hadoop.util.ChunkedArrayList; import org.apache.hadoop.util.Daemon; import org.apache.hadoop.util.StringUtils; import org.slf4j.Logger; @@ -145,7 +152,10 @@ public static byte[][] bytes2byteArray(byte[] bytes) { */ public static byte[][] bytes2byteArray(byte[] bytes, int len, byte separator) { - Preconditions.checkPositionIndex(len, bytes.length); + if (len < 0 || len > bytes.length) { + throw new IndexOutOfBoundsException( + "Incorrect index [len, size] [" + len + ", " + bytes.length + "]"); + } if (len == 0) { return new byte[][]{null}; } @@ -357,6 +367,13 @@ public static byte[] byteArray2bytes(byte[][] pathComponents) { return path; } + /** + * Given a list of path components returns a string. + */ + public static String byteArray2String(byte[][] pathComponents) { + return bytes2String(byteArray2bytes(pathComponents)); + } + /** * Decode a specific range of bytes of the given byte array to a string * using UTF8. @@ -1126,4 +1143,68 @@ public static String getSnapshotTrashRoot(String ssRoot, return (ssRoot.equals("/") ? ssRoot : ssRoot + Path.SEPARATOR) + FileSystem.TRASH_PREFIX + Path.SEPARATOR + ugi.getShortUserName(); } + + /** + * Returns true if the name of snapshot is vlaid. + * @param snapshotName name of the snapshot. + * @return true if the name of snapshot is vlaid. + */ + public static boolean isValidSnapshotName(String snapshotName) { + // If any of the snapshots specified in the getSnapshotDiffReport call + // is null or empty, it points to the current tree. + return (snapshotName != null && !snapshotName.isEmpty()); + } + + public static SnapshotDiffReport getSnapshotDiffReport( + String snapshotDir, String fromSnapshot, String toSnapshot, + SnapshotDiffReportFunction withoutListing, + SnapshotDiffReportListingFunction withListing) throws IOException { + // In case the diff needs to be computed between a snapshot and the current + // tree, we should not do iterative diffReport computation as the iterative + // approach might fail if in between the rpc calls the current tree + // changes in absence of the global fsn lock. + if (!isValidSnapshotName(fromSnapshot) || !isValidSnapshotName(toSnapshot)) { + return withoutListing.apply(snapshotDir, fromSnapshot, toSnapshot); + } + byte[] startPath = EMPTY_BYTES; + int index = -1; + SnapshotDiffReportGenerator snapshotDiffReport; + List modifiedList = new TreeList(); + List createdList = new ChunkedArrayList<>(); + List deletedList = new ChunkedArrayList<>(); + SnapshotDiffReportListing report; + do { + try { + report = withListing.apply(snapshotDir, fromSnapshot, toSnapshot, startPath, index); + } catch (RpcNoSuchMethodException|UnsupportedOperationException e) { + // In case the server doesn't support getSnapshotDiffReportListing, + // fallback to getSnapshotDiffReport. + LOG.warn("Falling back to getSnapshotDiffReport {}", e.getMessage()); + return withoutListing.apply(snapshotDir, fromSnapshot, toSnapshot); + } + startPath = report.getLastPath(); + index = report.getLastIndex(); + modifiedList.addAll(report.getModifyList()); + createdList.addAll(report.getCreateList()); + deletedList.addAll(report.getDeleteList()); + } while (!(Arrays.equals(startPath, EMPTY_BYTES) + && index == -1)); + snapshotDiffReport = + new SnapshotDiffReportGenerator(snapshotDir, fromSnapshot, toSnapshot, + report.getIsFromEarlier(), modifiedList, createdList, deletedList); + return snapshotDiffReport.generateReport(); + } + + @FunctionalInterface + public interface SnapshotDiffReportFunction { + SnapshotDiffReport apply(String snapshotDir, String fromSnapshot, String toSnapshot) + throws IOException; + } + + @FunctionalInterface + public interface SnapshotDiffReportListingFunction { + SnapshotDiffReportListing apply(String snapshotDir, String fromSnapshot, String toSnapshot, + byte[] startPath, int index) + throws IOException; + } } diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java index 449d0562b7070..499f5a764a014 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java @@ -18,12 +18,9 @@ package org.apache.hadoop.hdfs; - -import org.apache.hadoop.ipc.RpcNoSuchMethodException; import org.apache.hadoop.security.AccessControlException; import org.apache.hadoop.classification.VisibleForTesting; -import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; -import org.apache.commons.collections.list.TreeList; +import org.apache.hadoop.util.Preconditions; import org.apache.hadoop.HadoopIllegalArgumentException; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; @@ -107,8 +104,6 @@ import org.apache.hadoop.hdfs.protocol.RollingUpgradeInfo; import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport; import org.apache.hadoop.hdfs.protocol.SnapshotDiffReportListing; -import org.apache.hadoop.hdfs.protocol.SnapshotDiffReportListing.DiffReportListingEntry; -import org.apache.hadoop.hdfs.client.impl.SnapshotDiffReportGenerator; import org.apache.hadoop.hdfs.protocol.SnapshottableDirectoryStatus; import org.apache.hadoop.hdfs.protocol.SnapshotStatus; import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenIdentifier; @@ -116,7 +111,6 @@ import org.apache.hadoop.net.NetUtils; import org.apache.hadoop.security.token.Token; import org.apache.hadoop.security.token.DelegationTokenIssuer; -import org.apache.hadoop.util.ChunkedArrayList; import org.apache.hadoop.util.Lists; import org.apache.hadoop.util.Progressable; import org.slf4j.Logger; @@ -2298,8 +2292,8 @@ snapshotDiffReportListingRemoteIterator( @Override public RemoteIterator doCall(final Path p) throws IOException { - if (!isValidSnapshotName(fromSnapshot) || !isValidSnapshotName( - toSnapshot)) { + if (!DFSUtilClient.isValidSnapshotName(fromSnapshot) || + !DFSUtilClient.isValidSnapshotName(toSnapshot)) { throw new UnsupportedOperationException("Remote Iterator is" + "supported for snapshotDiffReport between two snapshots"); } @@ -2364,52 +2358,11 @@ public SnapshotDiffReportListing next() throws IOException { } } - private boolean isValidSnapshotName(String snapshotName) { - // If any of the snapshots specified in the getSnapshotDiffReport call - // is null or empty, it points to the current tree. - return (snapshotName != null && !snapshotName.isEmpty()); - } - private SnapshotDiffReport getSnapshotDiffReportInternal( final String snapshotDir, final String fromSnapshot, final String toSnapshot) throws IOException { - // In case the diff needs to be computed between a snapshot and the current - // tree, we should not do iterative diffReport computation as the iterative - // approach might fail if in between the rpc calls the current tree - // changes in absence of the global fsn lock. - if (!isValidSnapshotName(fromSnapshot) || !isValidSnapshotName( - toSnapshot)) { - return dfs.getSnapshotDiffReport(snapshotDir, fromSnapshot, toSnapshot); - } - byte[] startPath = DFSUtilClient.EMPTY_BYTES; - int index = -1; - SnapshotDiffReportGenerator snapshotDiffReport; - List modifiedList = new TreeList(); - List createdList = new ChunkedArrayList<>(); - List deletedList = new ChunkedArrayList<>(); - SnapshotDiffReportListing report; - do { - try { - report = dfs.getSnapshotDiffReportListing(snapshotDir, fromSnapshot, - toSnapshot, startPath, index); - } catch (RpcNoSuchMethodException e) { - // In case the server doesn't support getSnapshotDiffReportListing, - // fallback to getSnapshotDiffReport. - DFSClient.LOG.warn( - "Falling back to getSnapshotDiffReport {}", e.getMessage()); - return dfs.getSnapshotDiffReport(snapshotDir, fromSnapshot, toSnapshot); - } - startPath = report.getLastPath(); - index = report.getLastIndex(); - modifiedList.addAll(report.getModifyList()); - createdList.addAll(report.getCreateList()); - deletedList.addAll(report.getDeleteList()); - } while (!(Arrays.equals(startPath, DFSUtilClient.EMPTY_BYTES) - && index == -1)); - snapshotDiffReport = - new SnapshotDiffReportGenerator(snapshotDir, fromSnapshot, toSnapshot, - report.getIsFromEarlier(), modifiedList, createdList, deletedList); - return snapshotDiffReport.generateReport(); + return DFSUtilClient.getSnapshotDiffReport(snapshotDir, fromSnapshot, toSnapshot, + dfs::getSnapshotDiffReport, dfs::getSnapshotDiffReportListing); } /** diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/NameNodeProxiesClient.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/NameNodeProxiesClient.java index 1d8c8e632a38b..aa9577330cfae 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/NameNodeProxiesClient.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/NameNodeProxiesClient.java @@ -36,7 +36,7 @@ import org.slf4j.LoggerFactory; import org.apache.hadoop.classification.VisibleForTesting; -import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; +import org.apache.hadoop.util.Preconditions; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hdfs.client.HdfsClientConfigKeys; diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/PeerCache.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/PeerCache.java index abcc7e011ade5..a26a518a8395d 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/PeerCache.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/PeerCache.java @@ -24,7 +24,7 @@ import java.util.Map.Entry; import org.apache.hadoop.classification.VisibleForTesting; -import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; +import org.apache.hadoop.util.Preconditions; import org.apache.hadoop.thirdparty.com.google.common.collect.LinkedListMultimap; import org.apache.hadoop.classification.InterfaceAudience; diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/PositionStripeReader.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/PositionStripeReader.java index efadedb8f082f..12328ebb0f01b 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/PositionStripeReader.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/PositionStripeReader.java @@ -17,7 +17,7 @@ */ package org.apache.hadoop.hdfs; -import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; +import org.apache.hadoop.util.Preconditions; import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy; import org.apache.hadoop.hdfs.protocol.LocatedBlock; import org.apache.hadoop.hdfs.util.StripedBlockUtil; diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/StatefulStripeReader.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/StatefulStripeReader.java index 8bc6951f593ea..9069a558b67d7 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/StatefulStripeReader.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/StatefulStripeReader.java @@ -17,7 +17,7 @@ */ package org.apache.hadoop.hdfs; -import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; +import org.apache.hadoop.util.Preconditions; import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy; import org.apache.hadoop.hdfs.protocol.LocatedBlock; import org.apache.hadoop.hdfs.util.StripedBlockUtil; diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/StripeReader.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/StripeReader.java index 69aad06363c6c..932ddb491cc2f 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/StripeReader.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/StripeReader.java @@ -17,7 +17,7 @@ */ package org.apache.hadoop.hdfs; -import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; +import org.apache.hadoop.util.Preconditions; import org.apache.hadoop.fs.ChecksumException; import org.apache.hadoop.hdfs.protocol.DatanodeInfo; import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy; diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/XAttrHelper.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/XAttrHelper.java index ed6dd2fa2093f..7e8812fc13f9b 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/XAttrHelper.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/XAttrHelper.java @@ -27,7 +27,7 @@ import org.apache.hadoop.util.Lists; import org.apache.hadoop.util.StringUtils; -import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; +import org.apache.hadoop.util.Preconditions; import org.apache.hadoop.thirdparty.com.google.common.collect.Maps; @InterfaceAudience.Private diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/HdfsDataInputStream.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/HdfsDataInputStream.java index 5a615bbd62de4..af1e92e2fb449 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/HdfsDataInputStream.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/HdfsDataInputStream.java @@ -31,7 +31,7 @@ import org.apache.hadoop.hdfs.protocol.ExtendedBlock; import org.apache.hadoop.hdfs.protocol.LocatedBlock; -import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; +import org.apache.hadoop.util.Preconditions; /** * The Hdfs implementation of {@link FSDataInputStream}. diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/HdfsDataOutputStream.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/HdfsDataOutputStream.java index 8af3417ca9fdf..cc7e7cd21b090 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/HdfsDataOutputStream.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/HdfsDataOutputStream.java @@ -28,7 +28,7 @@ import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.hdfs.DFSOutputStream; -import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; +import org.apache.hadoop.util.Preconditions; /** * The Hdfs implementation of {@link FSDataOutputStream}. diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/impl/BlockReaderFactory.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/impl/BlockReaderFactory.java index a7c49128ba524..b2cf5348cfa2c 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/impl/BlockReaderFactory.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/impl/BlockReaderFactory.java @@ -75,7 +75,7 @@ import org.apache.hadoop.util.Time; import org.apache.hadoop.classification.VisibleForTesting; -import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; +import org.apache.hadoop.util.Preconditions; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/impl/BlockReaderLocal.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/impl/BlockReaderLocal.java index f013be6c3a6c0..50b1d61282d23 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/impl/BlockReaderLocal.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/impl/BlockReaderLocal.java @@ -18,7 +18,7 @@ package org.apache.hadoop.hdfs.client.impl; import org.apache.hadoop.classification.VisibleForTesting; -import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; +import org.apache.hadoop.util.Preconditions; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.fs.ReadOption; import org.apache.hadoop.fs.StorageType; diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/impl/DfsClientConf.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/impl/DfsClientConf.java index 890c23bc9bc2f..7a4508d1d3360 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/impl/DfsClientConf.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/impl/DfsClientConf.java @@ -18,7 +18,7 @@ package org.apache.hadoop.hdfs.client.impl; import org.apache.hadoop.classification.VisibleForTesting; -import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; +import org.apache.hadoop.util.Preconditions; import org.apache.hadoop.HadoopIllegalArgumentException; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.CommonConfigurationKeysPublic; diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/CacheDirectiveInfo.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/CacheDirectiveInfo.java index 3e5a4431372ef..0661a40bbb304 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/CacheDirectiveInfo.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/CacheDirectiveInfo.java @@ -25,7 +25,7 @@ import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.fs.Path; -import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; +import org.apache.hadoop.util.Preconditions; import org.apache.hadoop.hdfs.DFSUtilClient; /** diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/CacheDirectiveIterator.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/CacheDirectiveIterator.java index b3354344d7237..20ecfd4e1ec83 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/CacheDirectiveIterator.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/CacheDirectiveIterator.java @@ -26,7 +26,7 @@ import org.apache.hadoop.fs.InvalidRequestException; import org.apache.hadoop.ipc.RemoteException; -import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; +import org.apache.hadoop.util.Preconditions; import org.apache.hadoop.tracing.TraceScope; import org.apache.hadoop.tracing.Tracer; diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ErasureCodingPolicy.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ErasureCodingPolicy.java index 02c4f9a1f21b1..dc84bbbeae197 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ErasureCodingPolicy.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ErasureCodingPolicy.java @@ -17,7 +17,7 @@ */ package org.apache.hadoop.hdfs.protocol; -import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; +import org.apache.hadoop.util.Preconditions; import org.apache.commons.lang3.builder.EqualsBuilder; import org.apache.commons.lang3.builder.HashCodeBuilder; import org.apache.hadoop.classification.InterfaceAudience; diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ErasureCodingPolicyInfo.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ErasureCodingPolicyInfo.java index 48b581dfe6acb..79f10429a41d5 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ErasureCodingPolicyInfo.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ErasureCodingPolicyInfo.java @@ -17,7 +17,7 @@ */ package org.apache.hadoop.hdfs.protocol; -import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; +import org.apache.hadoop.util.Preconditions; import org.apache.commons.lang3.builder.EqualsBuilder; import org.apache.commons.lang3.builder.HashCodeBuilder; import org.apache.hadoop.classification.InterfaceAudience; diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsPartialListing.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsPartialListing.java index d96c7892b76ea..f64ca17a23b7b 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsPartialListing.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsPartialListing.java @@ -18,7 +18,7 @@ package org.apache.hadoop.hdfs.protocol; -import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; +import org.apache.hadoop.util.Preconditions; import org.apache.commons.lang3.builder.ToStringBuilder; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.ipc.RemoteException; diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/LocatedBlock.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/LocatedBlock.java index 9085e87ce60e1..01c276ae469e2 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/LocatedBlock.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/LocatedBlock.java @@ -22,7 +22,7 @@ import java.util.Comparator; import java.util.List; -import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; +import org.apache.hadoop.util.Preconditions; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.fs.StorageType; diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ReencryptionStatus.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ReencryptionStatus.java index b89786d37ec7e..d7c8e4b5e438e 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ReencryptionStatus.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ReencryptionStatus.java @@ -18,7 +18,7 @@ package org.apache.hadoop.hdfs.protocol; import org.apache.hadoop.classification.VisibleForTesting; -import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; +import org.apache.hadoop.util.Preconditions; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.fs.BatchedRemoteIterator.BatchedListEntries; import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.ReencryptionInfoProto; diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/SnapshotDiffReportListing.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/SnapshotDiffReportListing.java index 74329bc1e8a12..3fc13dec298d3 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/SnapshotDiffReportListing.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/SnapshotDiffReportListing.java @@ -20,7 +20,7 @@ import java.util.Collections; import java.util.List; -import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; +import org.apache.hadoop.util.Preconditions; import org.apache.hadoop.hdfs.DFSUtilClient; diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ZoneReencryptionStatus.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ZoneReencryptionStatus.java index 10884f27f90a2..858dff6204104 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ZoneReencryptionStatus.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ZoneReencryptionStatus.java @@ -17,7 +17,7 @@ */ package org.apache.hadoop.hdfs.protocol; -import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; +import org.apache.hadoop.util.Preconditions; import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.ReencryptionInfoProto; /** diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/datatransfer/PacketHeader.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/datatransfer/PacketHeader.java index cc958e35df116..ef1a3658305b1 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/datatransfer/PacketHeader.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/datatransfer/PacketHeader.java @@ -27,7 +27,7 @@ import org.apache.hadoop.hdfs.protocol.proto.DataTransferProtos.PacketHeaderProto; import org.apache.hadoop.hdfs.util.ByteBufferOutputStream; -import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; +import org.apache.hadoop.util.Preconditions; import org.apache.hadoop.thirdparty.com.google.common.primitives.Shorts; import org.apache.hadoop.thirdparty.com.google.common.primitives.Ints; import org.apache.hadoop.thirdparty.protobuf.InvalidProtocolBufferException; diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/datatransfer/PacketReceiver.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/datatransfer/PacketReceiver.java index 6949a9d40a656..56dc37d77e22e 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/datatransfer/PacketReceiver.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/datatransfer/PacketReceiver.java @@ -31,7 +31,7 @@ import org.apache.hadoop.util.DirectBufferPool; import org.apache.hadoop.io.IOUtils; -import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; +import org.apache.hadoop.util.Preconditions; import org.apache.hadoop.thirdparty.com.google.common.primitives.Ints; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelperClient.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelperClient.java index 9de9c373040e7..6097d2f495dac 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelperClient.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelperClient.java @@ -28,7 +28,7 @@ import java.util.Map; import java.util.Set; -import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; +import org.apache.hadoop.util.Preconditions; import org.apache.hadoop.thirdparty.com.google.common.cache.CacheBuilder; import org.apache.hadoop.thirdparty.com.google.common.cache.CacheLoader; import org.apache.hadoop.thirdparty.com.google.common.cache.LoadingCache; diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/server/datanode/DiskBalancerWorkItem.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/server/datanode/DiskBalancerWorkItem.java index 130e8c1c9c728..d1ad5a2079f5f 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/server/datanode/DiskBalancerWorkItem.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/server/datanode/DiskBalancerWorkItem.java @@ -22,7 +22,7 @@ import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.ObjectReader; -import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; +import org.apache.hadoop.util.Preconditions; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/server/datanode/DiskBalancerWorkStatus.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/server/datanode/DiskBalancerWorkStatus.java index 7e3954c562a58..5a5da7326a4e0 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/server/datanode/DiskBalancerWorkStatus.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/server/datanode/DiskBalancerWorkStatus.java @@ -23,7 +23,7 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.ObjectReader; import com.fasterxml.jackson.databind.SerializationFeature; -import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; +import org.apache.hadoop.util.Preconditions; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/shortcircuit/DfsClientShm.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/shortcircuit/DfsClientShm.java index 7bf768935d9d9..ae445efaf566b 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/shortcircuit/DfsClientShm.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/shortcircuit/DfsClientShm.java @@ -26,7 +26,7 @@ import org.apache.hadoop.net.unix.DomainSocket; import org.apache.hadoop.net.unix.DomainSocketWatcher; -import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; +import org.apache.hadoop.util.Preconditions; /** * DfsClientShm is a subclass of ShortCircuitShm which is used by the diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/shortcircuit/DfsClientShmManager.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/shortcircuit/DfsClientShmManager.java index f165ab90a1d4a..2b9bf68e469c6 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/shortcircuit/DfsClientShmManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/shortcircuit/DfsClientShmManager.java @@ -44,7 +44,7 @@ import org.apache.hadoop.net.unix.DomainSocketWatcher; import org.apache.hadoop.classification.VisibleForTesting; -import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; +import org.apache.hadoop.util.Preconditions; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/shortcircuit/DomainSocketFactory.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/shortcircuit/DomainSocketFactory.java index 9f08f0901a7ea..46d6718adb37c 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/shortcircuit/DomainSocketFactory.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/shortcircuit/DomainSocketFactory.java @@ -30,7 +30,7 @@ import org.apache.hadoop.net.unix.DomainSocket; import org.apache.hadoop.util.PerformanceAdvisory; -import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; +import org.apache.hadoop.util.Preconditions; import org.apache.hadoop.thirdparty.com.google.common.cache.Cache; import org.apache.hadoop.thirdparty.com.google.common.cache.CacheBuilder; diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/shortcircuit/ShortCircuitCache.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/shortcircuit/ShortCircuitCache.java index ccc39ddf5d55f..a950388a31239 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/shortcircuit/ShortCircuitCache.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/shortcircuit/ShortCircuitCache.java @@ -55,7 +55,7 @@ import org.apache.hadoop.util.Waitable; import org.apache.hadoop.classification.VisibleForTesting; -import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; +import org.apache.hadoop.util.Preconditions; import org.apache.hadoop.thirdparty.com.google.common.util.concurrent.ThreadFactoryBuilder; import org.slf4j.Logger; diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/shortcircuit/ShortCircuitReplica.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/shortcircuit/ShortCircuitReplica.java index fd58ef187525b..2177d9108f54e 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/shortcircuit/ShortCircuitReplica.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/shortcircuit/ShortCircuitReplica.java @@ -32,7 +32,7 @@ import org.apache.hadoop.util.Time; import org.apache.hadoop.classification.VisibleForTesting; -import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; +import org.apache.hadoop.util.Preconditions; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -160,7 +160,7 @@ boolean isStale() { long deltaMs = Time.monotonicNow() - creationTimeMs; long staleThresholdMs = cache.getStaleThresholdMs(); if (deltaMs > staleThresholdMs) { - LOG.trace("{} is stale because it's {} ms old and staleThreadholdMS={}", + LOG.trace("{} is stale because it's {} ms old and staleThresholdMs={}", this, deltaMs, staleThresholdMs); return true; } else { diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/shortcircuit/ShortCircuitShm.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/shortcircuit/ShortCircuitShm.java index 1cb123bb58f3b..c6f7a50368152 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/shortcircuit/ShortCircuitShm.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/shortcircuit/ShortCircuitShm.java @@ -39,7 +39,7 @@ import sun.misc.Unsafe; -import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; +import org.apache.hadoop.util.Preconditions; import org.apache.hadoop.thirdparty.com.google.common.collect.ComparisonChain; import org.apache.hadoop.thirdparty.com.google.common.primitives.Ints; diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/util/ByteArrayManager.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/util/ByteArrayManager.java index 059280e494678..f076969c9b17b 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/util/ByteArrayManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/util/ByteArrayManager.java @@ -26,7 +26,7 @@ import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.util.Time; -import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; +import org.apache.hadoop.util.Preconditions; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/util/StripedBlockUtil.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/util/StripedBlockUtil.java index 540187daa2a2c..25ee8d30c9290 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/util/StripedBlockUtil.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/util/StripedBlockUtil.java @@ -28,7 +28,7 @@ import org.apache.hadoop.hdfs.protocol.LocatedBlock; import org.apache.hadoop.hdfs.protocol.LocatedStripedBlock; -import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; +import org.apache.hadoop.util.Preconditions; import org.apache.hadoop.hdfs.security.token.block.BlockTokenIdentifier; import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy; import org.apache.hadoop.hdfs.DFSStripedOutputStream; @@ -245,8 +245,7 @@ public static long getSafeLength(ErasureCodingPolicy ecPolicy, Arrays.sort(cpy); // full stripe is a stripe has at least dataBlkNum full cells. // lastFullStripeIdx is the index of the last full stripe. - int lastFullStripeIdx = - (int) (cpy[cpy.length - dataBlkNum] / cellSize); + long lastFullStripeIdx = cpy[cpy.length - dataBlkNum] / cellSize; return lastFullStripeIdx * stripeSize; // return the safeLength // TODO: Include lastFullStripeIdx+1 stripe in safeLength, if there exists // such a stripe (and it must be partial). @@ -271,9 +270,9 @@ private static int lastCellSize(int size, int cellSize, int numDataBlocks, */ public static long offsetInBlkToOffsetInBG(int cellSize, int dataBlkNum, long offsetInBlk, int idxInBlockGroup) { - int cellIdxInBlk = (int) (offsetInBlk / cellSize); + long cellIdxInBlk = offsetInBlk / cellSize; return cellIdxInBlk * cellSize * dataBlkNum // n full stripes before offset - + idxInBlockGroup * cellSize // m full cells before offset + + (long)idxInBlockGroup * cellSize // m full cells before offset + offsetInBlk % cellSize; // partial cell } diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/JsonUtilClient.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/JsonUtilClient.java index 6acd062568e86..75163c16d7e5a 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/JsonUtilClient.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/JsonUtilClient.java @@ -19,7 +19,7 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.ObjectReader; -import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; +import org.apache.hadoop.util.Preconditions; import org.apache.hadoop.thirdparty.com.google.common.collect.Maps; import org.apache.hadoop.fs.ContentSummary; import org.apache.hadoop.fs.FileChecksum; @@ -39,6 +39,8 @@ import org.apache.hadoop.hdfs.protocol.DatanodeInfo; import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy; import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport; +import org.apache.hadoop.hdfs.protocol.SnapshotDiffReportListing; +import org.apache.hadoop.hdfs.protocol.SnapshotDiffReportListing.DiffReportListingEntry; import org.apache.hadoop.hdfs.protocol.SnapshottableDirectoryStatus; import org.apache.hadoop.hdfs.protocol.SnapshotStatus; import org.apache.hadoop.hdfs.protocol.DatanodeInfo.DatanodeInfoBuilder; @@ -245,6 +247,14 @@ static ExtendedBlock toExtendedBlock(final Map m) { return new ExtendedBlock(blockPoolId, blockId, numBytes, generationStamp); } + static boolean getBoolean(Map m, String key, final boolean defaultValue) { + Object value = m.get(key); + if (value == null) { + return defaultValue; + } + return ((Boolean) value).booleanValue(); + } + static int getInt(Map m, String key, final int defaultValue) { Object value = m.get(key); if (value == null) { @@ -834,6 +844,53 @@ private static SnapshotDiffReport.DiffReportEntry toDiffReportEntry( return new SnapshotDiffReport.DiffReportEntry(type, sourcePath, targetPath); } + public static SnapshotDiffReportListing toSnapshotDiffReportListing( + final Map json) { + if (json == null) { + return null; + } + + Map m = + (Map) json.get(SnapshotDiffReportListing.class.getSimpleName()); + byte[] lastPath = DFSUtilClient.string2Bytes(getString(m, "lastPath", "")); + int lastIndex = getInt(m, "lastIndex", -1); + boolean isFromEarlier = getBoolean(m, "isFromEarlier", false); + List modifyList = + toDiffListingList(getList(m, "modifyList")); + List createList = + toDiffListingList(getList(m, "createList")); + List deleteList = + toDiffListingList(getList(m, "deleteList")); + + return new SnapshotDiffReportListing( + lastPath, modifyList, createList, deleteList, lastIndex, isFromEarlier); + } + + public static List toDiffListingList(List objs) { + if (objs == null) { + return null; + } + List diffList = new ChunkedArrayList<>(); + for (int i = 0; i < objs.size(); i++) { + diffList.add(toDiffReportListingEntry((Map) objs.get(i))); + } + return diffList; + } + + private static DiffReportListingEntry toDiffReportListingEntry( + Map json) { + if (json == null) { + return null; + } + long dirId = getLong(json, "dirId", 0); + long fileId = getLong(json, "fileId", 0); + byte[] sourcePath = toByteArray(getString(json, "sourcePath", null)); + byte[] targetPath = toByteArray(getString(json, "targetPath", null)); + boolean isReference = getBoolean(json, "isReference", false); + return new DiffReportListingEntry( + dirId, fileId, sourcePath, isReference, targetPath); + } + private static byte[] toByteArray(String str) { if (str == null) { return null; diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java index 15d1b7b4b7ff3..2cba43b32dd70 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java @@ -46,7 +46,6 @@ import java.util.EnumSet; import java.util.HashSet; import java.util.List; -import java.util.Locale; import java.util.Map; import java.util.Optional; import java.util.Set; @@ -63,7 +62,6 @@ import org.apache.hadoop.crypto.key.KeyProviderTokenIssuer; import org.apache.hadoop.fs.BlockLocation; import org.apache.hadoop.fs.CommonConfigurationKeys; -import org.apache.hadoop.fs.CommonPathCapabilities; import org.apache.hadoop.fs.ContentSummary; import org.apache.hadoop.fs.CreateFlag; import org.apache.hadoop.fs.DelegationTokenRenewer; @@ -78,7 +76,6 @@ import org.apache.hadoop.fs.GlobalStorageStatistics.StorageStatisticsProvider; import org.apache.hadoop.fs.MultipartUploaderBuilder; import org.apache.hadoop.fs.QuotaUsage; -import org.apache.hadoop.fs.PathCapabilities; import org.apache.hadoop.fs.StorageStatistics; import org.apache.hadoop.fs.StorageType; import org.apache.hadoop.fs.impl.FileSystemMultipartUploaderBuilder; @@ -105,6 +102,7 @@ import org.apache.hadoop.hdfs.protocol.HdfsConstants; import org.apache.hadoop.hdfs.protocol.HdfsFileStatus; import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport; +import org.apache.hadoop.hdfs.protocol.SnapshotDiffReportListing; import org.apache.hadoop.hdfs.protocol.SnapshottableDirectoryStatus; import org.apache.hadoop.hdfs.protocol.SnapshotStatus; import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.FileEncryptionInfoProto; @@ -138,7 +136,7 @@ import org.apache.hadoop.classification.VisibleForTesting; import org.apache.hadoop.thirdparty.com.google.common.base.Charsets; -import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; +import org.apache.hadoop.util.Preconditions; import static org.apache.hadoop.fs.impl.PathCapabilitiesSupport.validatePathCapabilityArgs; @@ -1445,19 +1443,45 @@ public void renameSnapshot(final Path path, final String snapshotOldName, new SnapshotNameParam(snapshotNewName)).run(); } + private SnapshotDiffReport getSnapshotDiffReport( + final String snapshotDir, final String fromSnapshot, final String toSnapshot) + throws IOException { + return new FsPathResponseRunner( + GetOpParam.Op.GETSNAPSHOTDIFF, + new Path(snapshotDir), + new OldSnapshotNameParam(fromSnapshot), + new SnapshotNameParam(toSnapshot)) { + @Override + SnapshotDiffReport decodeResponse(Map json) { + return JsonUtilClient.toSnapshotDiffReport(json); + } + }.run(); + } + + private SnapshotDiffReportListing getSnapshotDiffReportListing( + String snapshotDir, final String fromSnapshot, final String toSnapshot, + byte[] startPath, int index) throws IOException { + return new FsPathResponseRunner( + GetOpParam.Op.GETSNAPSHOTDIFFLISTING, + new Path(snapshotDir), + new OldSnapshotNameParam(fromSnapshot), + new SnapshotNameParam(toSnapshot), + new SnapshotDiffStartPathParam(DFSUtilClient.bytes2String(startPath)), + new SnapshotDiffIndexParam(index)) { + @Override + SnapshotDiffReportListing decodeResponse(Map json) { + return JsonUtilClient.toSnapshotDiffReportListing(json); + } + }.run(); + } + public SnapshotDiffReport getSnapshotDiffReport(final Path snapshotDir, final String fromSnapshot, final String toSnapshot) throws IOException { statistics.incrementReadOps(1); storageStatistics.incrementOpCounter(OpType.GET_SNAPSHOT_DIFF); - final HttpOpParam.Op op = GetOpParam.Op.GETSNAPSHOTDIFF; - return new FsPathResponseRunner(op, snapshotDir, - new OldSnapshotNameParam(fromSnapshot), - new SnapshotNameParam(toSnapshot)) { - @Override - SnapshotDiffReport decodeResponse(Map json) { - return JsonUtilClient.toSnapshotDiffReport(json); - } - }.run(); + return DFSUtilClient.getSnapshotDiffReport( + snapshotDir.toUri().getPath(), fromSnapshot, toSnapshot, + this::getSnapshotDiffReport, this::getSnapshotDiffReportListing); } public SnapshottableDirectoryStatus[] getSnapshottableDirectoryList() diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/resources/GetOpParam.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/resources/GetOpParam.java index abbef9d8e9b91..14938c3c45b62 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/resources/GetOpParam.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/resources/GetOpParam.java @@ -62,6 +62,7 @@ public enum Op implements HttpOpParam.Op { LISTSTATUS_BATCH(false, HttpURLConnection.HTTP_OK), GETSERVERDEFAULTS(false, HttpURLConnection.HTTP_OK), GETSNAPSHOTDIFF(false, HttpURLConnection.HTTP_OK), + GETSNAPSHOTDIFFLISTING(false, HttpURLConnection.HTTP_OK), GETSNAPSHOTTABLEDIRECTORYLIST(false, HttpURLConnection.HTTP_OK), GETSNAPSHOTLIST(false, HttpURLConnection.HTTP_OK); diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/resources/SnapshotDiffIndexParam.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/resources/SnapshotDiffIndexParam.java new file mode 100644 index 0000000000000..cdec18cdcf63d --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/resources/SnapshotDiffIndexParam.java @@ -0,0 +1,49 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hdfs.web.resources; + +/** resuming index of snapshotDiffReportListing operation. */ +public class SnapshotDiffIndexParam extends IntegerParam { + /** Parameter name. */ + public static final String NAME = "snapshotdiffindex"; + /** Default parameter value. */ + public static final String DEFAULT = "-1"; + + private static final Domain DOMAIN = new Domain(NAME); + + /** + * Constructor. + * @param value the parameter value. + */ + public SnapshotDiffIndexParam(final Integer value) { + super(DOMAIN, value, -1, null); + } + + /** + * Constructor. + * @param str a string representation of the parameter value. + */ + public SnapshotDiffIndexParam(final String str) { + this(DOMAIN.parse(str)); + } + + @Override + public String getName() { + return NAME; + } +} diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/resources/SnapshotDiffStartPathParam.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/resources/SnapshotDiffStartPathParam.java new file mode 100644 index 0000000000000..22bce429b3fd5 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/resources/SnapshotDiffStartPathParam.java @@ -0,0 +1,40 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hdfs.web.resources; + +/** + * The snapshot startPath parameter used by snapshotDiffReportListing. + */ +public class SnapshotDiffStartPathParam extends StringParam { + /** Parameter name. */ + public static final String NAME = "snapshotdiffstartpath"; + + /** Default parameter value. */ + public static final String DEFAULT = ""; + + private static final Domain DOMAIN = new Domain(NAME, null); + + public SnapshotDiffStartPathParam(final String str) { + super(DOMAIN, str); + } + + @Override + public String getName() { + return NAME; + } +} diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/pom.xml b/hadoop-hdfs-project/hadoop-hdfs-httpfs/pom.xml index 57005e9d80b0c..559eb5a6597ea 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-httpfs/pom.xml +++ b/hadoop-hdfs-project/hadoop-hdfs-httpfs/pom.xml @@ -366,8 +366,8 @@ true Use hadoop-common provided VisibleForTesting rather than the one provided by Guava - org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting - com.google.common.annotations.VisibleForTesting + org.apache.hadoop.thirdparty.com.google.common.base.Preconditions + com.google.common.base.Preconditions diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/client/HttpFSFileSystem.java b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/client/HttpFSFileSystem.java index 549eb704ee12f..9f7dfd4647b11 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/client/HttpFSFileSystem.java +++ b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/client/HttpFSFileSystem.java @@ -73,7 +73,7 @@ import org.json.simple.parser.JSONParser; import org.json.simple.parser.ParseException; -import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; +import org.apache.hadoop.util.Preconditions; import org.apache.hadoop.thirdparty.com.google.common.collect.Maps; import java.io.BufferedInputStream; diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/pom.xml b/hadoop-hdfs-project/hadoop-hdfs-native-client/pom.xml index 279b0e6dc1abd..82db2d14facae 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-native-client/pom.xml +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/pom.xml @@ -95,38 +95,6 @@ https://maven.apache.org/xsd/maven-4.0.0.xsd"> - - org.apache.maven.plugins - maven-enforcer-plugin - - - de.skuzzle.enforcer - restrict-imports-enforcer-rule - ${restrict-imports.enforcer.version} - - - - - banned-illegal-imports - process-sources - - enforce - - - - - true - Use hadoop-common provided VisibleForTesting rather than the one provided by Guava - - org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting - com.google.common.annotations.VisibleForTesting - - - - - - - diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/CMakeLists.txt b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/CMakeLists.txt index d8c2012fb88de..9a92171b8d360 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/CMakeLists.txt +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/CMakeLists.txt @@ -108,14 +108,13 @@ set(STORED_CMAKE_FIND_LIBRARY_SUFFIXES ${CMAKE_FIND_LIBRARY_SUFFIXES}) hadoop_set_find_shared_library_without_version() set(OPENSSL_NAME "crypto") if(${CMAKE_SYSTEM_NAME} MATCHES "Windows") - SET(OPENSSL_NAME "eay32") + SET(OPENSSL_NAME "libcrypto") endif() message("CUSTOM_OPENSSL_PREFIX = ${CUSTOM_OPENSSL_PREFIX}") find_library(OPENSSL_LIBRARY NAMES ${OPENSSL_NAME} PATHS ${CUSTOM_OPENSSL_PREFIX} ${CUSTOM_OPENSSL_PREFIX}/lib ${CUSTOM_OPENSSL_PREFIX}/lib64 ${CUSTOM_OPENSSL_LIB} NO_DEFAULT_PATH) -find_library(OPENSSL_LIBRARY NAMES ${OPENSSL_NAME}) find_path(OPENSSL_INCLUDE_DIR NAMES openssl/evp.h PATHS ${CUSTOM_OPENSSL_PREFIX} ${CUSTOM_OPENSSL_PREFIX}/include diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/tools/CMakeLists.txt b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/tools/CMakeLists.txt index 48580a4ccab19..b5c21c7915baf 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/tools/CMakeLists.txt +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/tools/CMakeLists.txt @@ -18,7 +18,10 @@ add_executable(hdfs_tool_tests hdfs-allow-snapshot-mock.cc + hdfs-disallow-snapshot-mock.cc + hdfs-rename-snapshot-mock.cc hdfs-delete-snapshot-mock.cc + hdfs-create-snapshot-mock.cc hdfs-cat-mock.cc hdfs-tool-test-fixtures.cc hdfs-tool-tests.cc @@ -29,12 +32,18 @@ target_include_directories(hdfs_tool_tests PRIVATE ../../tools ../../tools/hdfs-df ../../tools/hdfs-allow-snapshot + ../../tools/hdfs-disallow-snapshot ../../tools/hdfs-delete-snapshot + ../../tools/hdfs-create-snapshot + ../../tools/hdfs-rename-snapshot ../../tools/hdfs-cat) target_link_libraries(hdfs_tool_tests PRIVATE gmock_main hdfs_df_lib hdfs_allowSnapshot_lib + hdfs_disallowSnapshot_lib hdfs_deleteSnapshot_lib + hdfs_createSnapshot_lib + hdfs_renameSnapshot_lib hdfs_cat_lib) add_test(hdfs_tool_tests hdfs_tool_tests) diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/tools/hdfs-create-snapshot-mock.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/tools/hdfs-create-snapshot-mock.cc new file mode 100644 index 0000000000000..323963181fc37 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/tools/hdfs-create-snapshot-mock.cc @@ -0,0 +1,58 @@ +/* + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. +*/ + +#include +#include +#include +#include +#include + +#include +#include + +#include "hdfs-create-snapshot-mock.h" +#include "hdfs-tool-tests.h" + +namespace hdfs::tools::test { +CreateSnapshotMock::~CreateSnapshotMock() = default; + +void CreateSnapshotMock::SetExpectations( + std::function()> test_case, + const std::vector &args) const { + // Get the pointer to the function that defines the test case + const auto test_case_func = + test_case.target (*)()>(); + ASSERT_NE(test_case_func, nullptr); + + // Set the expected method calls and their corresponding arguments for each + // test case + if (*test_case_func == &CallHelp) { + EXPECT_CALL(*this, HandleHelp()).Times(1).WillOnce(testing::Return(true)); + return; + } + + if (*test_case_func == &PassNOptAndAPath) { + const auto arg1 = args[1]; + const auto arg2 = std::optional{args[0]}; + EXPECT_CALL(*this, HandleSnapshot(arg1, arg2)) + .Times(1) + .WillOnce(testing::Return(true)); + } +} +} // namespace hdfs::tools::test diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/tools/hdfs-create-snapshot-mock.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/tools/hdfs-create-snapshot-mock.h new file mode 100644 index 0000000000000..e159ec96c5370 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/tools/hdfs-create-snapshot-mock.h @@ -0,0 +1,72 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef LIBHDFSPP_TOOLS_HDFS_CREATE_SNAPSHOT_MOCK +#define LIBHDFSPP_TOOLS_HDFS_CREATE_SNAPSHOT_MOCK + +#include +#include +#include +#include +#include + +#include + +#include "hdfs-create-snapshot.h" + +namespace hdfs::tools::test { +/** + * {@class CreateSnapshotMock} is an {@class CreateSnapshot} whereby it mocks + * the HandleHelp and HandleSnapshot methods for testing their functionality. + */ +class CreateSnapshotMock : public hdfs::tools::CreateSnapshot { +public: + /** + * {@inheritdoc} + */ + CreateSnapshotMock(const int argc, char **argv) + : CreateSnapshot(argc, argv) {} + + // Abiding to the Rule of 5 + CreateSnapshotMock(const CreateSnapshotMock &) = delete; + CreateSnapshotMock(CreateSnapshotMock &&) = delete; + CreateSnapshotMock &operator=(const CreateSnapshotMock &) = delete; + CreateSnapshotMock &operator=(CreateSnapshotMock &&) = delete; + ~CreateSnapshotMock() override; + + /** + * Defines the methods and the corresponding arguments that are expected + * to be called on this instance of {@link HdfsTool} for the given test case. + * + * @param test_case An {@link std::function} object that points to the + * function defining the test case + * @param args The arguments that are passed to this test case + */ + void SetExpectations( + std::function()> test_case, + const std::vector &args = {}) const; + + MOCK_METHOD(bool, HandleHelp, (), (const, override)); + + MOCK_METHOD(bool, HandleSnapshot, + (const std::string &, const std::optional &), + (const, override)); +}; +} // namespace hdfs::tools::test + +#endif diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/tools/hdfs-disallow-snapshot-mock.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/tools/hdfs-disallow-snapshot-mock.cc new file mode 100644 index 0000000000000..445cb3851cc69 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/tools/hdfs-disallow-snapshot-mock.cc @@ -0,0 +1,56 @@ +/* + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. +*/ + +#include +#include +#include +#include + +#include +#include + +#include "hdfs-disallow-snapshot-mock.h" +#include "hdfs-tool-tests.h" + +namespace hdfs::tools::test { +DisallowSnapshotMock::~DisallowSnapshotMock() = default; + +void DisallowSnapshotMock::SetExpectations( + std::function()> test_case, + const std::vector &args) const { + // Get the pointer to the function that defines the test case + const auto test_case_func = + test_case.target (*)()>(); + ASSERT_NE(test_case_func, nullptr); + + // Set the expected method calls and their corresponding arguments for each + // test case + if (*test_case_func == &CallHelp) { + EXPECT_CALL(*this, HandleHelp()).Times(1).WillOnce(testing::Return(true)); + return; + } + + if (*test_case_func == &PassAPath) { + const auto arg1 = args[0]; + EXPECT_CALL(*this, HandleSnapshot(arg1)) + .Times(1) + .WillOnce(testing::Return(true)); + } +} +} // namespace hdfs::tools::test diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/tools/hdfs-disallow-snapshot-mock.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/tools/hdfs-disallow-snapshot-mock.h new file mode 100644 index 0000000000000..a6e2c240781d9 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/tools/hdfs-disallow-snapshot-mock.h @@ -0,0 +1,70 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef LIBHDFSPP_TOOLS_HDFS_DISALLOW_SNAPSHOT_MOCK +#define LIBHDFSPP_TOOLS_HDFS_DISALLOW_SNAPSHOT_MOCK + +#include +#include +#include +#include + +#include + +#include "hdfs-disallow-snapshot.h" + +namespace hdfs::tools::test { +/** + * {@class DisallowSnapshotMock} is an {@class DisallowSnapshot} whereby it + * mocks the HandleHelp and HandleSnapshot methods for testing their + * functionality. + */ +class DisallowSnapshotMock : public hdfs::tools::DisallowSnapshot { +public: + /** + * {@inheritdoc} + */ + DisallowSnapshotMock(const int argc, char **argv) + : DisallowSnapshot(argc, argv) {} + + // Abiding to the Rule of 5 + DisallowSnapshotMock(const DisallowSnapshotMock &) = delete; + DisallowSnapshotMock(DisallowSnapshotMock &&) = delete; + DisallowSnapshotMock &operator=(const DisallowSnapshotMock &) = delete; + DisallowSnapshotMock &operator=(DisallowSnapshotMock &&) = delete; + ~DisallowSnapshotMock() override; + + /** + * Defines the methods and the corresponding arguments that are expected + * to be called on this instance of {@link HdfsTool} for the given test case. + * + * @param test_case An {@link std::function} object that points to the + * function defining the test case + * @param args The arguments that are passed to this test case + */ + void SetExpectations( + std::function()> test_case, + const std::vector &args = {}) const; + + MOCK_METHOD(bool, HandleHelp, (), (const, override)); + + MOCK_METHOD(bool, HandleSnapshot, (const std::string &), (const, override)); +}; +} // namespace hdfs::tools::test + +#endif diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/tools/hdfs-rename-snapshot-mock.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/tools/hdfs-rename-snapshot-mock.cc new file mode 100644 index 0000000000000..6ee7f173d847a --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/tools/hdfs-rename-snapshot-mock.cc @@ -0,0 +1,58 @@ +/* + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. +*/ + +#include +#include +#include +#include + +#include +#include + +#include "hdfs-rename-snapshot-mock.h" +#include "hdfs-tool-tests.h" + +namespace hdfs::tools::test { +RenameSnapshotMock::~RenameSnapshotMock() = default; + +void RenameSnapshotMock::SetExpectations( + std::function()> test_case, + const std::vector &args) const { + // Get the pointer to the function that defines the test case + const auto test_case_func = + test_case.target (*)()>(); + ASSERT_NE(test_case_func, nullptr); + + // Set the expected method calls and their corresponding arguments for each + // test case + if (*test_case_func == &CallHelp) { + EXPECT_CALL(*this, HandleHelp()).Times(1).WillOnce(testing::Return(true)); + return; + } + + if (*test_case_func == &Pass3Paths) { + const auto arg1 = args[0]; + const auto arg2 = args[1]; + const auto arg3 = args[2]; + EXPECT_CALL(*this, HandleSnapshot(arg1, arg2, arg3)) + .Times(1) + .WillOnce(testing::Return(true)); + } +} +} // namespace hdfs::tools::test diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/tools/hdfs-rename-snapshot-mock.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/tools/hdfs-rename-snapshot-mock.h new file mode 100644 index 0000000000000..657c2f179146a --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/tools/hdfs-rename-snapshot-mock.h @@ -0,0 +1,72 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef LIBHDFSPP_TOOLS_HDFS_RENAME_SNAPSHOT_MOCK +#define LIBHDFSPP_TOOLS_HDFS_RENAME_SNAPSHOT_MOCK + +#include +#include +#include +#include + +#include + +#include "hdfs-rename-snapshot.h" + +namespace hdfs::tools::test { +/** + * {@class RenameSnapshotMock} is an {@class RenameSnapshot} whereby it + * mocks the HandleHelp and HandleSnapshot methods for testing their + * functionality. + */ +class RenameSnapshotMock : public hdfs::tools::RenameSnapshot { +public: + /** + * {@inheritdoc} + */ + RenameSnapshotMock(const int argc, char **argv) + : RenameSnapshot(argc, argv) {} + + // Abiding to the Rule of 5 + RenameSnapshotMock(const RenameSnapshotMock &) = delete; + RenameSnapshotMock(RenameSnapshotMock &&) = delete; + RenameSnapshotMock &operator=(const RenameSnapshotMock &) = delete; + RenameSnapshotMock &operator=(RenameSnapshotMock &&) = delete; + ~RenameSnapshotMock() override; + + /** + * Defines the methods and the corresponding arguments that are expected + * to be called on this instance of {@link HdfsTool} for the given test case. + * + * @param test_case An {@link std::function} object that points to the + * function defining the test case + * @param args The arguments that are passed to this test case + */ + void SetExpectations( + std::function()> test_case, + const std::vector &args = {}) const; + + MOCK_METHOD(bool, HandleHelp, (), (const, override)); + + MOCK_METHOD(bool, HandleSnapshot, + (const std::string &, const std::string &, const std::string &), + (const, override)); +}; +} // namespace hdfs::tools::test + +#endif diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/tools/hdfs-tool-tests.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/tools/hdfs-tool-tests.cc index 8ffce3b3f6a40..33480a933df0b 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/tools/hdfs-tool-tests.cc +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/tools/hdfs-tool-tests.cc @@ -20,9 +20,12 @@ #include #include "hdfs-allow-snapshot-mock.h" -#include "hdfs-df-mock.h" #include "hdfs-cat-mock.h" +#include "hdfs-create-snapshot-mock.h" #include "hdfs-delete-snapshot-mock.h" +#include "hdfs-df-mock.h" +#include "hdfs-disallow-snapshot-mock.h" +#include "hdfs-rename-snapshot-mock.h" #include "hdfs-tool-test-fixtures.h" #include "hdfs-tool-tests.h" @@ -32,17 +35,27 @@ * hdfs-tool-test.h} to yield the test suite. */ -/** - * This file combines the test fixtures defined in {@file - * hdfs-tool-test-fixtures.h} and the test cases defined in {@file - * hdfs-tool-test.h} to yield the test structure. - */ - +// Basic tests INSTANTIATE_TEST_SUITE_P( HdfsAllowSnapshot, HdfsToolBasicTest, testing::Values(PassAPath, CallHelp)); +INSTANTIATE_TEST_SUITE_P( + HdfsDisallowSnapshot, HdfsToolBasicTest, + testing::Values(PassAPath, + CallHelp)); + +INSTANTIATE_TEST_SUITE_P( + HdfsRenameSnapshot, HdfsToolBasicTest, + testing::Values(Pass3Paths, + CallHelp)); + +INSTANTIATE_TEST_SUITE_P( + HdfsCreateSnapshot, HdfsToolBasicTest, + testing::Values(PassNOptAndAPath, + CallHelp)); + INSTANTIATE_TEST_SUITE_P(HdfsCat, HdfsToolBasicTest, testing::Values(PassAPath, CallHelp)); @@ -56,10 +69,25 @@ INSTANTIATE_TEST_SUITE_P( testing::Values(CallHelp, Pass2Paths)); +// Negative tests INSTANTIATE_TEST_SUITE_P( HdfsAllowSnapshot, HdfsToolNegativeTestThrows, testing::Values(Pass2Paths)); +INSTANTIATE_TEST_SUITE_P( + HdfsRenameSnapshot, HdfsToolNegativeTestNoThrow, + testing::Values(PassAPath, + Pass2Paths)); + +INSTANTIATE_TEST_SUITE_P( + HdfsCreateSnapshot, HdfsToolNegativeTestThrows, + testing::Values(Pass2Paths, + Pass3Paths)); + +INSTANTIATE_TEST_SUITE_P( + HdfsDisallowSnapshot, HdfsToolNegativeTestThrows, + testing::Values(Pass2Paths)); + INSTANTIATE_TEST_SUITE_P( HdfsDf, HdfsToolNegativeTestThrows, testing::Values(Pass2Paths)); diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/tools/hdfs-tool-tests.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/tools/hdfs-tool-tests.h index f7fd6a706a646..188a76c439478 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/tools/hdfs-tool-tests.h +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/tools/hdfs-tool-tests.h @@ -17,8 +17,8 @@ under the License. */ -#ifndef LIBHDFSPP_TOOLS_HDFS_TOOL_TEST -#define LIBHDFSPP_TOOLS_HDFS_TOOL_TEST +#ifndef LIBHDFSPP_TOOLS_HDFS_TOOL_TESTS +#define LIBHDFSPP_TOOLS_HDFS_TOOL_TESTS #include #include @@ -69,4 +69,32 @@ template std::unique_ptr Pass2Paths() { return hdfs_tool; } +template std::unique_ptr Pass3Paths() { + constexpr auto argc = 4; + static std::string exe("hdfs_tool_name"); + static std::string arg1("a/b/c"); + static std::string arg2("d/e/f"); + static std::string arg3("g/h/i"); + + static char *argv[] = {exe.data(), arg1.data(), arg2.data(), arg3.data()}; + + auto hdfs_tool = std::make_unique(argc, argv); + hdfs_tool->SetExpectations(Pass3Paths, {arg1, arg2, arg3}); + return hdfs_tool; +} + +template std::unique_ptr PassNOptAndAPath() { + constexpr auto argc = 4; + static std::string exe("hdfs_tool_name"); + static std::string arg1("-n"); + static std::string arg2("some_name"); + static std::string arg3("g/h/i"); + + static char *argv[] = {exe.data(), arg1.data(), arg2.data(), arg3.data()}; + + auto hdfs_tool = std::make_unique(argc, argv); + hdfs_tool->SetExpectations(PassNOptAndAPath, {arg2, arg3}); + return hdfs_tool; +} + #endif diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/CMakeLists.txt b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/CMakeLists.txt index c1222502547cc..c4aed54eeabb7 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/CMakeLists.txt +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/CMakeLists.txt @@ -79,14 +79,11 @@ target_link_libraries(hdfs_setrep tools_common hdfspp_static) add_subdirectory(hdfs-allow-snapshot) -add_executable(hdfs_disallowSnapshot hdfs_disallowSnapshot.cc) -target_link_libraries(hdfs_disallowSnapshot tools_common hdfspp_static) +add_subdirectory(hdfs-disallow-snapshot) -add_executable(hdfs_createSnapshot hdfs_createSnapshot.cc) -target_link_libraries(hdfs_createSnapshot tools_common hdfspp_static) +add_subdirectory(hdfs-create-snapshot) -add_executable(hdfs_renameSnapshot hdfs_renameSnapshot.cc) -target_link_libraries(hdfs_renameSnapshot tools_common hdfspp_static) +add_subdirectory(hdfs-rename-snapshot) add_subdirectory(hdfs-delete-snapshot) diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-create-snapshot/CMakeLists.txt b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-create-snapshot/CMakeLists.txt new file mode 100644 index 0000000000000..871fcd8d732c0 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-create-snapshot/CMakeLists.txt @@ -0,0 +1,27 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +add_library(hdfs_createSnapshot_lib STATIC $ hdfs-create-snapshot.cc) +target_include_directories(hdfs_createSnapshot_lib PRIVATE ../../tools ${Boost_INCLUDE_DIRS}) +target_link_libraries(hdfs_createSnapshot_lib PRIVATE Boost::boost Boost::program_options tools_common hdfspp_static) + +add_executable(hdfs_createSnapshot main.cc) +target_include_directories(hdfs_createSnapshot PRIVATE ../../tools) +target_link_libraries(hdfs_createSnapshot PRIVATE hdfs_createSnapshot_lib) + +install(TARGETS hdfs_createSnapshot RUNTIME DESTINATION bin) diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-create-snapshot/hdfs-create-snapshot.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-create-snapshot/hdfs-create-snapshot.cc new file mode 100644 index 0000000000000..6036e3a76f9f3 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-create-snapshot/hdfs-create-snapshot.cc @@ -0,0 +1,138 @@ +/* + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. +*/ + +#include +#include +#include +#include +#include +#include + +#include "hdfs-create-snapshot.h" +#include "tools_common.h" + +namespace hdfs::tools { +CreateSnapshot::CreateSnapshot(const int argc, char **argv) + : HdfsTool(argc, argv) {} + +bool CreateSnapshot::Initialize() { + auto add_options = opt_desc_.add_options(); + add_options("help,h", "Show the help for hdfs_createSnapshot"); + add_options("path", po::value(), + "The path to the directory for creating the snapshot"); + add_options("name,n", po::value(), + "The snapshot name, a default name is selected if omitted"); + + // We allow only one argument to be passed to path option. An exception is + // thrown if multiple arguments are passed to this. + pos_opt_desc_.add("path", 1); + + po::store(po::command_line_parser(argc_, argv_) + .options(opt_desc_) + .positional(pos_opt_desc_) + .run(), + opt_val_); + po::notify(opt_val_); + return true; +} + +bool CreateSnapshot::ValidateConstraints() const { + // If the name option is specified, there will be 4 arguments in total + if (argc_ == 4) { + return opt_val_.count("name") > 0; + } + + // Rest of the cases must contain more than 1 argument on the command line + return argc_ > 1; +} + +std::string CreateSnapshot::GetDescription() const { + std::stringstream desc; + desc << "Usage: hdfs_createSnapshot [OPTION] PATH" << std::endl + << std::endl + << "Create a snapshot of a snapshot-able directory." << std::endl + << "This operation requires owner privilege of the snapshot-able " + "directory." + << std::endl + << std::endl + << " -n NAME The snapshot name. When it is omitted, a default name " + "is generated" + << std::endl + << " using a timestamp with the format:" << std::endl + << R"( "'s'yyyyMMdd-HHmmss.SSS", e.g. s20130412-151029.033)" + << std::endl + << " -h display this help and exit" << std::endl + << std::endl + << "Examples:" << std::endl + << "hdfs_createSnapshot hdfs://localhost.localdomain:8020/dir" + << std::endl + << "hdfs_createSnapshot -n MySnapshot /dir1/dir2" << std::endl; + return desc.str(); +} + +bool CreateSnapshot::Do() { + if (!Initialize()) { + std::cerr << "Unable to initialize HDFS create snapshot tool" << std::endl; + return false; + } + + if (!ValidateConstraints()) { + std::cout << GetDescription(); + return false; + } + + if (opt_val_.count("help") > 0) { + return HandleHelp(); + } + + if (opt_val_.count("path") > 0) { + const auto path = opt_val_["path"].as(); + const auto name = opt_val_.count("name") > 0 + ? std::optional{opt_val_["name"].as()} + : std::nullopt; + return HandleSnapshot(path, name); + } + + return true; +} + +bool CreateSnapshot::HandleHelp() const { + std::cout << GetDescription(); + return true; +} + +bool CreateSnapshot::HandleSnapshot( + const std::string &path, const std::optional &name) const { + // Building a URI object from the given uri_path + auto uri = hdfs::parse_path_or_exit(path); + + const auto fs = hdfs::doConnect(uri, false); + if (!fs) { + std::cerr << "Could not connect the file system. " << std::endl; + return false; + } + + const auto status = fs->CreateSnapshot(uri.get_path(), name.value_or("")); + if (!status.ok()) { + std::cerr << "Error: " << status.ToString() << std::endl; + return false; + } + return true; +} +} // namespace hdfs::tools diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-create-snapshot/hdfs-create-snapshot.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-create-snapshot/hdfs-create-snapshot.h new file mode 100644 index 0000000000000..9f1fc09ff2cc3 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-create-snapshot/hdfs-create-snapshot.h @@ -0,0 +1,96 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef LIBHDFSPP_TOOLS_HDFS_CREATE_SNAPSHOT +#define LIBHDFSPP_TOOLS_HDFS_CREATE_SNAPSHOT + +#include +#include + +#include + +#include "hdfs-tool.h" + +namespace hdfs::tools { +/** + * {@class CreateSnapshot} is an {@class HdfsTool} that facilitates the creation + * of the snapshot of a snapshot-able directory located at PATH. + */ +class CreateSnapshot : public HdfsTool { +public: + /** + * {@inheritdoc} + */ + CreateSnapshot(int argc, char **argv); + + // Abiding to the Rule of 5 + CreateSnapshot(const CreateSnapshot &) = default; + CreateSnapshot(CreateSnapshot &&) = default; + CreateSnapshot &operator=(const CreateSnapshot &) = delete; + CreateSnapshot &operator=(CreateSnapshot &&) = delete; + ~CreateSnapshot() override = default; + + /** + * {@inheritdoc} + */ + [[nodiscard]] std::string GetDescription() const override; + + /** + * {@inheritdoc} + */ + [[nodiscard]] bool Do() override; + +protected: + /** + * {@inheritdoc} + */ + [[nodiscard]] bool Initialize() override; + + /** + * {@inheritdoc} + */ + [[nodiscard]] bool ValidateConstraints() const override; + + /** + * {@inheritdoc} + */ + [[nodiscard]] bool HandleHelp() const override; + + /** + * Handle the path argument that's passed to this tool. + * + * @param path The path to the snapshot-able directory for which the snapshot + * needs to be created. + * @param name The name to assign to the snapshot after creating it. + * + * @return A boolean indicating the result of this operation. + */ + [[nodiscard]] virtual bool + HandleSnapshot(const std::string &path, + const std::optional &name = std::nullopt) const; + +private: + /** + * A boost data-structure containing the description of positional arguments + * passed to the command-line. + */ + po::positional_options_description pos_opt_desc_; +}; +} // namespace hdfs::tools + +#endif diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-create-snapshot/main.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-create-snapshot/main.cc new file mode 100644 index 0000000000000..7801bb4c60475 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-create-snapshot/main.cc @@ -0,0 +1,54 @@ +/* + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. +*/ + +#include +#include +#include + +#include + +#include "hdfs-create-snapshot.h" + +int main(int argc, char *argv[]) { + const auto result = std::atexit([]() -> void { + // Clean up static data on exit and prevent valgrind memory leaks + google::protobuf::ShutdownProtobufLibrary(); + }); + if (result != 0) { + std::cerr << "Error: Unable to schedule clean-up tasks for HDFS create " + "snapshot tool, exiting" + << std::endl; + std::exit(EXIT_FAILURE); + } + + hdfs::tools::CreateSnapshot create_snapshot(argc, argv); + auto success = false; + + try { + success = create_snapshot.Do(); + } catch (const std::exception &e) { + std::cerr << "Error: " << e.what() << std::endl; + success = false; + } + + if (!success) { + std::exit(EXIT_FAILURE); + } + return 0; +} diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-disallow-snapshot/CMakeLists.txt b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-disallow-snapshot/CMakeLists.txt new file mode 100644 index 0000000000000..263fc3adb9683 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-disallow-snapshot/CMakeLists.txt @@ -0,0 +1,27 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +add_library(hdfs_disallowSnapshot_lib STATIC $ hdfs-disallow-snapshot.cc) +target_include_directories(hdfs_disallowSnapshot_lib PRIVATE ../../tools ${Boost_INCLUDE_DIRS}) +target_link_libraries(hdfs_disallowSnapshot_lib PRIVATE Boost::boost Boost::program_options tools_common hdfspp_static) + +add_executable(hdfs_disallowSnapshot main.cc) +target_include_directories(hdfs_disallowSnapshot PRIVATE ../../tools) +target_link_libraries(hdfs_disallowSnapshot PRIVATE hdfs_disallowSnapshot_lib) + +install(TARGETS hdfs_disallowSnapshot RUNTIME DESTINATION bin) diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-disallow-snapshot/hdfs-disallow-snapshot.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-disallow-snapshot/hdfs-disallow-snapshot.cc new file mode 100644 index 0000000000000..576c1bec30fa7 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-disallow-snapshot/hdfs-disallow-snapshot.cc @@ -0,0 +1,117 @@ +/* + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. +*/ + +#include +#include +#include +#include +#include + +#include "hdfs-disallow-snapshot.h" +#include "tools_common.h" + +namespace hdfs::tools { +DisallowSnapshot::DisallowSnapshot(const int argc, char **argv) + : HdfsTool(argc, argv) {} + +bool DisallowSnapshot::Initialize() { + auto add_options = opt_desc_.add_options(); + add_options("help,h", "Show the help for hdfs_disallowSnapshot"); + add_options( + "path", po::value(), + "The path to the directory for which snapshot-ing must be disallowed"); + + // We allow only one argument to be passed to this tool. An exception is + // thrown if multiple arguments are passed. + pos_opt_desc_.add("path", 1); + + po::store(po::command_line_parser(argc_, argv_) + .options(opt_desc_) + .positional(pos_opt_desc_) + .run(), + opt_val_); + po::notify(opt_val_); + return true; +} + +std::string DisallowSnapshot::GetDescription() const { + std::stringstream desc; + desc << "Usage: hdfs_disallowSnapshot [OPTION] PATH" << std::endl + << std::endl + << "Disallowing snapshots of a directory at PATH to be created." + << std::endl + << "All snapshots of the directory must be deleted before disallowing " + "snapshots." + << std::endl + << std::endl + << " -h display this help and exit" << std::endl + << std::endl + << "Examples:" << std::endl + << "hdfs_disallowSnapshot hdfs://localhost.localdomain:8020/dir" + << std::endl + << "hdfs_disallowSnapshot /dir1/dir2" << std::endl; + return desc.str(); +} + +bool DisallowSnapshot::Do() { + if (!Initialize()) { + std::cerr << "Unable to initialize HDFS allow snapshot tool" << std::endl; + return false; + } + + if (!ValidateConstraints()) { + std::cout << GetDescription(); + return false; + } + + if (opt_val_.count("help") > 0) { + return HandleHelp(); + } + + if (opt_val_.count("path") > 0) { + const auto path = opt_val_["path"].as(); + return HandleSnapshot(path); + } + + return true; +} + +bool DisallowSnapshot::HandleHelp() const { + std::cout << GetDescription(); + return true; +} + +bool DisallowSnapshot::HandleSnapshot(const std::string &path) const { + // Building a URI object from the given uri_path + auto uri = hdfs::parse_path_or_exit(path); + + const auto fs = hdfs::doConnect(uri, false); + if (!fs) { + std::cerr << "Could not connect the file system. " << std::endl; + return false; + } + + const auto status = fs->DisallowSnapshot(uri.get_path()); + if (!status.ok()) { + std::cerr << "Error: " << status.ToString() << std::endl; + return false; + } + return true; +} +} // namespace hdfs::tools diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-disallow-snapshot/hdfs-disallow-snapshot.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-disallow-snapshot/hdfs-disallow-snapshot.h new file mode 100644 index 0000000000000..b4174873cff1c --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-disallow-snapshot/hdfs-disallow-snapshot.h @@ -0,0 +1,92 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef LIBHDFSPP_TOOLS_HDFS_DISALLOW_SNAPSHOT +#define LIBHDFSPP_TOOLS_HDFS_DISALLOW_SNAPSHOT + +#include + +#include + +#include "hdfs-tool.h" + +namespace hdfs::tools { +/** + * {@class DisallowSnapshot} is an {@class HdfsTool} that disallows the + * snapshots from being created for a directory. All the existing snapshots must + * be deleted first. + */ +class DisallowSnapshot : public HdfsTool { +public: + /** + * {@inheritdoc} + */ + DisallowSnapshot(int argc, char **argv); + + // Abiding to the Rule of 5 + DisallowSnapshot(const DisallowSnapshot &) = default; + DisallowSnapshot(DisallowSnapshot &&) = default; + DisallowSnapshot &operator=(const DisallowSnapshot &) = delete; + DisallowSnapshot &operator=(DisallowSnapshot &&) = delete; + ~DisallowSnapshot() override = default; + + /** + * {@inheritdoc} + */ + [[nodiscard]] std::string GetDescription() const override; + + /** + * {@inheritdoc} + */ + [[nodiscard]] bool Do() override; + +protected: + /** + * {@inheritdoc} + */ + [[nodiscard]] bool Initialize() override; + + /** + * {@inheritdoc} + */ + [[nodiscard]] bool ValidateConstraints() const override { return argc_ > 1; } + + /** + * {@inheritdoc} + */ + [[nodiscard]] bool HandleHelp() const override; + + /** + * Handle the path argument that's passed to this tool. + * + * @param path The path to the directory for which snapshot-ing must be + * disallowed. + * + * @return A boolean indicating the result of this operation. + */ + [[nodiscard]] virtual bool HandleSnapshot(const std::string &path) const; + +private: + /** + * A boost data-structure containing the description of positional arguments + * passed to the command-line. + */ + po::positional_options_description pos_opt_desc_; +}; +} // namespace hdfs::tools +#endif diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-disallow-snapshot/main.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-disallow-snapshot/main.cc new file mode 100644 index 0000000000000..07009200f3994 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-disallow-snapshot/main.cc @@ -0,0 +1,54 @@ +/* + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. +*/ + +#include +#include +#include + +#include + +#include "hdfs-disallow-snapshot.h" + +int main(int argc, char *argv[]) { + const auto result = std::atexit([]() -> void { + // Clean up static data on exit and prevent valgrind memory leaks + google::protobuf::ShutdownProtobufLibrary(); + }); + if (result != 0) { + std::cerr << "Error: Unable to schedule clean-up tasks for HDFS disallow " + "snapshot tool, exiting" + << std::endl; + std::exit(EXIT_FAILURE); + } + + hdfs::tools::DisallowSnapshot disallow_snapshot(argc, argv); + auto success = false; + + try { + success = disallow_snapshot.Do(); + } catch (const std::exception &e) { + std::cerr << "Error: " << e.what() << std::endl; + success = false; + } + + if (!success) { + std::exit(EXIT_FAILURE); + } + return 0; +} diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-rename-snapshot/CMakeLists.txt b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-rename-snapshot/CMakeLists.txt new file mode 100644 index 0000000000000..44d9447eac791 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-rename-snapshot/CMakeLists.txt @@ -0,0 +1,27 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +add_library(hdfs_renameSnapshot_lib STATIC $ hdfs-rename-snapshot.cc) +target_include_directories(hdfs_renameSnapshot_lib PRIVATE ../../tools ${Boost_INCLUDE_DIRS}) +target_link_libraries(hdfs_renameSnapshot_lib PRIVATE Boost::boost Boost::program_options tools_common hdfspp_static) + +add_executable(hdfs_renameSnapshot main.cc) +target_include_directories(hdfs_renameSnapshot PRIVATE ../../tools) +target_link_libraries(hdfs_renameSnapshot PRIVATE hdfs_renameSnapshot_lib) + +install(TARGETS hdfs_renameSnapshot RUNTIME DESTINATION bin) diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-rename-snapshot/hdfs-rename-snapshot.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-rename-snapshot/hdfs-rename-snapshot.cc new file mode 100644 index 0000000000000..80858c5471b4f --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-rename-snapshot/hdfs-rename-snapshot.cc @@ -0,0 +1,139 @@ +/* + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. +*/ + +#include +#include +#include +#include +#include + +#include "hdfs-rename-snapshot.h" +#include "tools_common.h" + +namespace hdfs::tools { +RenameSnapshot::RenameSnapshot(const int argc, char **argv) + : HdfsTool(argc, argv) {} + +bool RenameSnapshot::Initialize() { + auto add_options = opt_desc_.add_options(); + add_options("help,h", "Show the help for hdfs_renameSnapshot"); + add_options("path", po::value(), + "The path to the directory that is snapshot-able"); + add_options("old-name", po::value(), + "The old/current name of the snapshot"); + add_options("new-name", po::value(), + "The new name for the snapshot"); + + // Register "path", "old-name" and "new-name" as positional arguments. + // We allow only three arguments to be passed to this tool. An exception is + // thrown if more than three arguments are passed. + pos_opt_desc_.add("path", 1); + pos_opt_desc_.add("old-name", 1); + pos_opt_desc_.add("new-name", 1); + + po::store(po::command_line_parser(argc_, argv_) + .options(opt_desc_) + .positional(pos_opt_desc_) + .run(), + opt_val_); + po::notify(opt_val_); + return true; +} + +bool RenameSnapshot::ValidateConstraints() const { + // Only "help" is allowed as single argument + if (argc_ == 2) { + return opt_val_.count("help") > 0; + } + + // Rest of the cases must contain more than 2 argument on the command line + return argc_ > 3; +} + +std::string RenameSnapshot::GetDescription() const { + std::stringstream desc; + desc << "Usage: hdfs_renameSnapshot [OPTION] PATH OLD_NAME NEW_NAME" + << std::endl + << std::endl + << "Rename a snapshot from OLD_NAME to NEW_NAME." << std::endl + << "This operation requires owner privilege of the snapshot-able " + "directory." + << std::endl + << std::endl + << " -h display this help and exit" << std::endl + << std::endl + << "Examples:" << std::endl + << "hdfs_renameSnapshot hdfs://localhost.localdomain:8020/dir oldDir " + "newDir" + << std::endl + << "hdfs_renameSnapshot /dir1/dir2 oldSnap newSnap" << std::endl; + return desc.str(); +} + +bool RenameSnapshot::Do() { + if (!Initialize()) { + std::cerr << "Unable to initialize HDFS rename snapshot tool" << std::endl; + return false; + } + + if (!ValidateConstraints()) { + std::cout << GetDescription(); + return false; + } + + if (opt_val_.count("help") > 0) { + return HandleHelp(); + } + + if (opt_val_.count("path") > 0 && opt_val_.count("old-name") > 0 && + opt_val_.count("new-name") > 0) { + const auto path = opt_val_["path"].as(); + const auto old_name = opt_val_["old-name"].as(); + const auto new_name = opt_val_["new-name"].as(); + return HandleSnapshot(path, old_name, new_name); + } + + return true; +} + +bool RenameSnapshot::HandleHelp() const { + std::cout << GetDescription(); + return true; +} + +bool RenameSnapshot::HandleSnapshot(const std::string &path, + const std::string &old_name, + const std::string &new_name) const { + // Building a URI object from the given path + auto uri = hdfs::parse_path_or_exit(path); + + const auto fs = hdfs::doConnect(uri, false); + if (!fs) { + std::cerr << "Could not connect the file system. " << std::endl; + return false; + } + + const auto status = fs->RenameSnapshot(uri.get_path(), old_name, new_name); + if (!status.ok()) { + std::cerr << "Error: " << status.ToString() << std::endl; + return false; + } + return true; +} +} // namespace hdfs::tools diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-rename-snapshot/hdfs-rename-snapshot.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-rename-snapshot/hdfs-rename-snapshot.h new file mode 100644 index 0000000000000..5133999b87f14 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-rename-snapshot/hdfs-rename-snapshot.h @@ -0,0 +1,95 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef LIBHDFSPP_TOOLS_HDFS_RENAME_SNAPSHOT +#define LIBHDFSPP_TOOLS_HDFS_RENAME_SNAPSHOT + +#include + +#include + +#include "hdfs-tool.h" + +namespace hdfs::tools { +/** + * {@class RenameSnapshot} is an {@class HdfsTool} that facilitates the + * renaming of the snapshots. + */ +class RenameSnapshot : public HdfsTool { +public: + /** + * {@inheritdoc} + */ + RenameSnapshot(int argc, char **argv); + + // Abiding to the Rule of 5 + RenameSnapshot(const RenameSnapshot &) = default; + RenameSnapshot(RenameSnapshot &&) = default; + RenameSnapshot &operator=(const RenameSnapshot &) = delete; + RenameSnapshot &operator=(RenameSnapshot &&) = delete; + ~RenameSnapshot() override = default; + + /** + * {@inheritdoc} + */ + [[nodiscard]] std::string GetDescription() const override; + + /** + * {@inheritdoc} + */ + [[nodiscard]] bool Do() override; + +protected: + /** + * {@inheritdoc} + */ + [[nodiscard]] bool Initialize() override; + + /** + * {@inheritdoc} + */ + [[nodiscard]] bool ValidateConstraints() const override; + + /** + * {@inheritdoc} + */ + [[nodiscard]] bool HandleHelp() const override; + + /** + * Handle the arguments that are passed to this tool. + * + * @param path The path to the directory that is snapshot-able. + * @param old_name The name of the current/older snapshot. + * @param new_name The new name for the old snapshot. + * + * @return A boolean indicating the result of this operation. + */ + [[nodiscard]] virtual bool HandleSnapshot(const std::string &path, + const std::string &old_name, + const std::string &new_name) const; + +private: + /** + * A boost data-structure containing the description of positional arguments + * passed to the command-line. + */ + po::positional_options_description pos_opt_desc_; +}; +} // namespace hdfs::tools + +#endif diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-rename-snapshot/main.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-rename-snapshot/main.cc new file mode 100644 index 0000000000000..c52425411d382 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-rename-snapshot/main.cc @@ -0,0 +1,54 @@ +/* + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. +*/ + +#include +#include +#include + +#include + +#include "hdfs-rename-snapshot.h" + +int main(int argc, char *argv[]) { + const auto result = std::atexit([]() -> void { + // Clean up static data on exit and prevent valgrind memory leaks + google::protobuf::ShutdownProtobufLibrary(); + }); + if (result != 0) { + std::cerr << "Error: Unable to schedule clean-up tasks for HDFS rename " + "snapshot tool, exiting" + << std::endl; + std::exit(EXIT_FAILURE); + } + + hdfs::tools::RenameSnapshot rename_snapshot(argc, argv); + auto success = false; + + try { + success = rename_snapshot.Do(); + } catch (const std::exception &e) { + std::cerr << "Error: " << e.what() << std::endl; + success = false; + } + + if (!success) { + std::exit(EXIT_FAILURE); + } + return 0; +} diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_createSnapshot.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_createSnapshot.cc deleted file mode 100644 index 1236dd61130e6..0000000000000 --- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_createSnapshot.cc +++ /dev/null @@ -1,95 +0,0 @@ -/* - Licensed to the Apache Software Foundation (ASF) under one - or more contributor license agreements. See the NOTICE file - distributed with this work for additional information - regarding copyright ownership. The ASF licenses this file - to you under the Apache License, Version 2.0 (the - "License"); you may not use this file except in compliance - with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, - software distributed under the License is distributed on an - "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - KIND, either express or implied. See the License for the - specific language governing permissions and limitations - under the License. -*/ - -#include -#include -#include "tools_common.h" - -void usage(){ - std::cout << "Usage: hdfs_createSnapshot [OPTION] PATH" - << std::endl - << std::endl << "Create a snapshot of a snapshottable directory." - << std::endl << "This operation requires owner privilege of the snapshottable directory." - << std::endl - << std::endl << " -n NAME The snapshot name. When it is omitted, a default name is generated" - << std::endl << " using a timestamp with the format:" - << std::endl << " \"'s'yyyyMMdd-HHmmss.SSS\", e.g. s20130412-151029.033" - << std::endl << " -h display this help and exit" - << std::endl - << std::endl << "Examples:" - << std::endl << "hdfs_createSnapshot hdfs://localhost.localdomain:8020/dir" - << std::endl << "hdfs_createSnapshot -n MySnapshot /dir1/dir2" - << std::endl; -} - -int main(int argc, char *argv[]) { - //We should have at least 2 arguments - if (argc < 2) { - usage(); - exit(EXIT_FAILURE); - } - - int input; - std::string name; - - //Using GetOpt to read in the values - opterr = 0; - while ((input = getopt(argc, argv, "hn:")) != -1) { - switch (input) - { - case 'h': - usage(); - exit(EXIT_SUCCESS); - case 'n': - name = optarg; - break; - case '?': - if (optopt == 'n') - std::cerr << "Option -" << (char) optopt << " requires an argument." << std::endl; - else if (isprint(optopt)) - std::cerr << "Unknown option `-" << (char) optopt << "'." << std::endl; - else - std::cerr << "Unknown option character `" << (char) optopt << "'." << std::endl; - usage(); - exit(EXIT_FAILURE); - default: - exit(EXIT_FAILURE); - } - } - std::string uri_path = argv[optind]; - - //Building a URI object from the given uri_path - hdfs::URI uri = hdfs::parse_path_or_exit(uri_path); - - std::shared_ptr fs = hdfs::doConnect(uri, false); - if (!fs) { - std::cerr << "Could not connect the file system. " << std::endl; - exit(EXIT_FAILURE); - } - - hdfs::Status status = fs->CreateSnapshot(uri.get_path(), name); - if (!status.ok()) { - std::cerr << "Error: " << status.ToString() << std::endl; - exit(EXIT_FAILURE); - } - - // Clean up static data and prevent valgrind memory leaks - google::protobuf::ShutdownProtobufLibrary(); - return 0; -} diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_disallowSnapshot.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_disallowSnapshot.cc deleted file mode 100644 index b5c97ab9033f3..0000000000000 --- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_disallowSnapshot.cc +++ /dev/null @@ -1,86 +0,0 @@ -/* - Licensed to the Apache Software Foundation (ASF) under one - or more contributor license agreements. See the NOTICE file - distributed with this work for additional information - regarding copyright ownership. The ASF licenses this file - to you under the Apache License, Version 2.0 (the - "License"); you may not use this file except in compliance - with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, - software distributed under the License is distributed on an - "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - KIND, either express or implied. See the License for the - specific language governing permissions and limitations - under the License. -*/ - -#include -#include -#include "tools_common.h" - -void usage(){ - std::cout << "Usage: hdfs_disallowSnapshot [OPTION] PATH" - << std::endl - << std::endl << "Disallowing snapshots of a directory at PATH to be created." - << std::endl << "All snapshots of the directory must be deleted before disallowing snapshots." - << std::endl - << std::endl << " -h display this help and exit" - << std::endl - << std::endl << "Examples:" - << std::endl << "hdfs_disallowSnapshot hdfs://localhost.localdomain:8020/dir" - << std::endl << "hdfs_disallowSnapshot /dir1/dir2" - << std::endl; -} - -int main(int argc, char *argv[]) { - //We should have at least 2 arguments - if (argc < 2) { - usage(); - exit(EXIT_FAILURE); - } - - int input; - - //Using GetOpt to read in the values - opterr = 0; - while ((input = getopt(argc, argv, "Rh")) != -1) { - switch (input) - { - case 'h': - usage(); - exit(EXIT_SUCCESS); - case '?': - if (isprint(optopt)) - std::cerr << "Unknown option `-" << (char) optopt << "'." << std::endl; - else - std::cerr << "Unknown option character `" << (char) optopt << "'." << std::endl; - usage(); - exit(EXIT_FAILURE); - default: - exit(EXIT_FAILURE); - } - } - std::string uri_path = argv[optind]; - - //Building a URI object from the given uri_path - hdfs::URI uri = hdfs::parse_path_or_exit(uri_path); - - std::shared_ptr fs = hdfs::doConnect(uri, false); - if (!fs) { - std::cerr << "Could not connect the file system. " << std::endl; - exit(EXIT_FAILURE); - } - - hdfs::Status status = fs->DisallowSnapshot(uri.get_path()); - if (!status.ok()) { - std::cerr << "Error: " << status.ToString() << std::endl; - exit(EXIT_FAILURE); - } - - // Clean up static data and prevent valgrind memory leaks - google::protobuf::ShutdownProtobufLibrary(); - return 0; -} diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_renameSnapshot.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_renameSnapshot.cc deleted file mode 100644 index 8d38fc2292d1e..0000000000000 --- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_renameSnapshot.cc +++ /dev/null @@ -1,88 +0,0 @@ -/* - Licensed to the Apache Software Foundation (ASF) under one - or more contributor license agreements. See the NOTICE file - distributed with this work for additional information - regarding copyright ownership. The ASF licenses this file - to you under the Apache License, Version 2.0 (the - "License"); you may not use this file except in compliance - with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, - software distributed under the License is distributed on an - "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - KIND, either express or implied. See the License for the - specific language governing permissions and limitations - under the License. -*/ - -#include -#include -#include "tools_common.h" - -void usage(){ - std::cout << "Usage: hdfs_renameSnapshot [OPTION] PATH OLD_NAME NEW_NAME" - << std::endl - << std::endl << "Rename a snapshot from OLD_NAME to NEW_NAME." - << std::endl << "This operation requires owner privilege of the snapshottable directory." - << std::endl - << std::endl << " -h display this help and exit" - << std::endl - << std::endl << "Examples:" - << std::endl << "hdfs_renameSnapshot hdfs://localhost.localdomain:8020/dir oldDir newDir" - << std::endl << "hdfs_renameSnapshot /dir1/dir2 oldSnap newSnap" - << std::endl; -} - -int main(int argc, char *argv[]) { - //We should have at least 2 arguments - if (argc < 2) { - usage(); - exit(EXIT_FAILURE); - } - - int input; - - //Using GetOpt to read in the values - opterr = 0; - while ((input = getopt(argc, argv, "h")) != -1) { - switch (input) - { - case 'h': - usage(); - exit(EXIT_SUCCESS); - case '?': - if (isprint(optopt)) - std::cerr << "Unknown option `-" << (char) optopt << "'." << std::endl; - else - std::cerr << "Unknown option character `" << (char) optopt << "'." << std::endl; - usage(); - exit(EXIT_FAILURE); - default: - exit(EXIT_FAILURE); - } - } - std::string uri_path = argv[optind]; - std::string old_name = argv[optind+1]; - std::string new_name = argv[optind+2]; - - //Building a URI object from the given uri_path - hdfs::URI uri = hdfs::parse_path_or_exit(uri_path); - - std::shared_ptr fs = hdfs::doConnect(uri, false); - if (!fs) { - std::cerr << "Could not connect the file system. " << std::endl; - exit(EXIT_FAILURE); - } - - hdfs::Status status = fs->RenameSnapshot(uri.get_path(), old_name, new_name); - if (!status.ok()) { - std::cerr << "Error: " << status.ToString() << std::endl; - exit(EXIT_FAILURE); - } - - // Clean up static data and prevent valgrind memory leaks - google::protobuf::ShutdownProtobufLibrary(); - return 0; -} diff --git a/hadoop-hdfs-project/hadoop-hdfs-nfs/pom.xml b/hadoop-hdfs-project/hadoop-hdfs-nfs/pom.xml index a6cc3f1ed39aa..9043d4972ce3a 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-nfs/pom.xml +++ b/hadoop-hdfs-project/hadoop-hdfs-nfs/pom.xml @@ -242,8 +242,8 @@ https://maven.apache.org/xsd/maven-4.0.0.xsd"> true Use hadoop-common provided VisibleForTesting rather than the one provided by Guava - org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting - com.google.common.annotations.VisibleForTesting + org.apache.hadoop.thirdparty.com.google.common.base.Preconditions + com.google.common.base.Preconditions diff --git a/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/DFSClientCache.java b/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/DFSClientCache.java index 75c29d30314b7..dc1eb8746964e 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/DFSClientCache.java +++ b/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/DFSClientCache.java @@ -29,7 +29,7 @@ import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; -import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; +import org.apache.hadoop.util.Preconditions; import org.apache.hadoop.fs.FSDataInputStream; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.hdfs.DFSClient; diff --git a/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/OffsetRange.java b/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/OffsetRange.java index 3995fa5566bb0..5fb1f1a01200b 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/OffsetRange.java +++ b/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/OffsetRange.java @@ -19,7 +19,7 @@ import java.util.Comparator; -import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; +import org.apache.hadoop.util.Preconditions; /** * OffsetRange is the range of read/write request. A single point (e.g.,[5,5]) diff --git a/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/OpenFileCtx.java b/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/OpenFileCtx.java index 0e07b1bfb62aa..8e4026926c83b 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/OpenFileCtx.java +++ b/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/OpenFileCtx.java @@ -58,7 +58,7 @@ import org.apache.hadoop.util.Time; import org.apache.hadoop.classification.VisibleForTesting; -import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; +import org.apache.hadoop.util.Preconditions; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/OpenFileCtxCache.java b/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/OpenFileCtxCache.java index feb681d27a811..70ae4b29e9f96 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/OpenFileCtxCache.java +++ b/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/OpenFileCtxCache.java @@ -31,7 +31,7 @@ import org.apache.hadoop.util.Time; import org.apache.hadoop.classification.VisibleForTesting; -import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; +import org.apache.hadoop.util.Preconditions; import org.apache.hadoop.thirdparty.com.google.common.collect.Maps; /** diff --git a/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/WriteCtx.java b/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/WriteCtx.java index 882e9cda86aa0..ed1856c9a851c 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/WriteCtx.java +++ b/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/WriteCtx.java @@ -30,7 +30,7 @@ import org.apache.hadoop.nfs.nfs3.Nfs3Constant.WriteStableHow; import org.apache.hadoop.classification.VisibleForTesting; -import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; +import org.apache.hadoop.util.Preconditions; /** * WriteCtx saves the context of one write request, such as request, channel, diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/pom.xml b/hadoop-hdfs-project/hadoop-hdfs-rbf/pom.xml index 2959fc54b7d97..0eaff17b844a9 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/pom.xml +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/pom.xml @@ -328,8 +328,8 @@ https://maven.apache.org/xsd/maven-4.0.0.xsd"> true Use hadoop-common provided VisibleForTesting rather than the one provided by Guava - org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting - com.google.common.annotations.VisibleForTesting + org.apache.hadoop.thirdparty.com.google.common.base.Preconditions + com.google.common.base.Preconditions diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/fairness/AbstractRouterRpcFairnessPolicyController.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/fairness/AbstractRouterRpcFairnessPolicyController.java index 629f21e565181..548f1a82f6d83 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/fairness/AbstractRouterRpcFairnessPolicyController.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/fairness/AbstractRouterRpcFairnessPolicyController.java @@ -23,6 +23,8 @@ import java.util.concurrent.Semaphore; import java.util.concurrent.TimeUnit; +import org.codehaus.jettison.json.JSONException; +import org.codehaus.jettison.json.JSONObject; import org.apache.hadoop.conf.Configuration; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -75,4 +77,19 @@ protected void insertNameServiceWithPermits(String nsId, int maxPermits) { protected int getAvailablePermits(String nsId) { return this.permits.get(nsId).availablePermits(); } + + @Override + public String getAvailableHandlerOnPerNs() { + JSONObject json = new JSONObject(); + for (Map.Entry entry : permits.entrySet()) { + try { + String nsId = entry.getKey(); + int availableHandler = entry.getValue().availablePermits(); + json.put(nsId, availableHandler); + } catch (JSONException e) { + LOG.warn("Cannot put {} into JSONObject", entry.getKey(), e); + } + } + return json.toString(); + } } diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/fairness/NoRouterRpcFairnessPolicyController.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/fairness/NoRouterRpcFairnessPolicyController.java index b6a7df48431a8..3b85da59e1f52 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/fairness/NoRouterRpcFairnessPolicyController.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/fairness/NoRouterRpcFairnessPolicyController.java @@ -46,4 +46,9 @@ public void releasePermit(String nsId) { public void shutdown() { // Nothing for now. } + + @Override + public String getAvailableHandlerOnPerNs(){ + return "N/A"; + } } diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/fairness/RouterRpcFairnessPolicyController.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/fairness/RouterRpcFairnessPolicyController.java index 80e7d218b2a7d..354383a168f4e 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/fairness/RouterRpcFairnessPolicyController.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/fairness/RouterRpcFairnessPolicyController.java @@ -62,4 +62,9 @@ public interface RouterRpcFairnessPolicyController { * Shutdown steps to stop accepting new permission requests and clean-up. */ void shutdown(); + + /** + * Returns the JSON string of the available handler for each Ns. + */ + String getAvailableHandlerOnPerNs(); } diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/FederationRPCMBean.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/FederationRPCMBean.java index a4469a3025a11..979e7504a872b 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/FederationRPCMBean.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/FederationRPCMBean.java @@ -109,6 +109,12 @@ public interface FederationRPCMBean { */ String getRpcClientConnections(); + /** + * JSON representation of the available handler per Ns. + * @return JSON string representation. + */ + String getAvailableHandlerOnPerNs(); + /** * Get the JSON representation of the async caller thread pool. * @return JSON string representation of the async caller thread pool. @@ -120,4 +126,16 @@ public interface FederationRPCMBean { * @return Number of operations rejected due to lack of permits. */ long getProxyOpPermitRejected(); + + /** + * Get the number of operations rejected due to lack of permits of each namespace. + * @return Number of operations rejected due to lack of permits of each namespace. + */ + String getProxyOpPermitRejectedPerNs(); + + /** + * Get the number of operations accepted of each namespace. + * @return Number of operations accepted of each namespace. + */ + String getProxyOpPermitAcceptedPerNs(); } diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/FederationRPCMetrics.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/FederationRPCMetrics.java index 1e6aa8050d92e..018bf0c60a9e4 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/FederationRPCMetrics.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/FederationRPCMetrics.java @@ -233,6 +233,12 @@ public String getRpcClientConnections() { return rpcServer.getRPCClient().getJSON(); } + @Override + public String getAvailableHandlerOnPerNs() { + return rpcServer.getRPCClient(). + getRouterRpcFairnessPolicyController().getAvailableHandlerOnPerNs(); + } + @Override public String getAsyncCallerPool() { return rpcServer.getRPCClient().getAsyncCallerPoolJson(); @@ -286,4 +292,14 @@ public void incrProxyOpPermitRejected() { public long getProxyOpPermitRejected() { return proxyOpPermitRejected.value(); } + + @Override + public String getProxyOpPermitRejectedPerNs() { + return rpcServer.getRPCClient().getRejectedPermitsPerNsJSON(); + } + + @Override + public String getProxyOpPermitAcceptedPerNs() { + return rpcServer.getRPCClient().getAcceptedPermitsPerNsJSON(); + } } diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/order/AvailableSpaceResolver.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/order/AvailableSpaceResolver.java index 88e20649506dc..591ac5b3c37ca 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/order/AvailableSpaceResolver.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/order/AvailableSpaceResolver.java @@ -39,7 +39,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; +import org.apache.hadoop.util.Preconditions; /** * Order the destinations based on available space. This resolver uses a diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterAdminServer.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterAdminServer.java index 0d322ca501674..deb933ad5adb8 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterAdminServer.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterAdminServer.java @@ -30,7 +30,7 @@ import java.util.Map; import java.util.Set; -import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; +import org.apache.hadoop.util.Preconditions; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcClient.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcClient.java index ea65245e71974..70c8880cec716 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcClient.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcClient.java @@ -47,6 +47,7 @@ import java.util.concurrent.BlockingQueue; import java.util.concurrent.Callable; import java.util.concurrent.CancellationException; +import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ExecutionException; import java.util.concurrent.Future; import java.util.concurrent.LinkedBlockingQueue; @@ -54,6 +55,7 @@ import java.util.concurrent.ThreadFactory; import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.LongAdder; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -62,7 +64,6 @@ import org.apache.hadoop.hdfs.client.HdfsClientConfigKeys; import org.apache.hadoop.hdfs.protocol.ExtendedBlock; import org.apache.hadoop.hdfs.protocol.SnapshotException; -import org.apache.hadoop.hdfs.server.federation.fairness.AbstractRouterRpcFairnessPolicyController; import org.apache.hadoop.hdfs.server.federation.fairness.RouterRpcFairnessPolicyController; import org.apache.hadoop.hdfs.server.federation.resolver.ActiveNamenodeResolver; import org.apache.hadoop.hdfs.server.federation.resolver.FederationNamenodeContext; @@ -134,6 +135,8 @@ public class RouterRpcClient { /** Fairness manager to control handlers assigned per NS. */ private RouterRpcFairnessPolicyController routerRpcFairnessPolicyController; + private Map rejectedPermitsPerNs = new ConcurrentHashMap<>(); + private Map acceptedPermitsPerNs = new ConcurrentHashMap<>(); /** * Create a router RPC client to manage remote procedure calls to NNs. @@ -319,6 +322,23 @@ public String getAsyncCallerPoolJson() { return JSON.toString(info); } + /** + * JSON representation of the rejected permits for each nameservice. + * + * @return String representation of the rejected permits for each nameservice. + */ + public String getRejectedPermitsPerNsJSON() { + return JSON.toString(rejectedPermitsPerNs); + } + + /** + * JSON representation of the accepted permits for each nameservice. + * + * @return String representation of the accepted permits for each nameservice. + */ + public String getAcceptedPermitsPerNsJSON() { + return JSON.toString(acceptedPermitsPerNs); + } /** * Get ClientProtocol proxy client for a NameNode. Each combination of user + * NN must use a unique proxy client. Previously created clients are cached @@ -1537,19 +1557,22 @@ private String getNameserviceForBlockPoolId(final String bpId) private void acquirePermit( final String nsId, final UserGroupInformation ugi, final RemoteMethod m) throws IOException { - if (routerRpcFairnessPolicyController != null - && !routerRpcFairnessPolicyController.acquirePermit(nsId)) { - // Throw StandByException, - // Clients could fail over and try another router. - if (rpcMonitor != null) { - rpcMonitor.getRPCMetrics().incrProxyOpPermitRejected(); + if (routerRpcFairnessPolicyController != null) { + if (!routerRpcFairnessPolicyController.acquirePermit(nsId)) { + // Throw StandByException, + // Clients could fail over and try another router. + if (rpcMonitor != null) { + rpcMonitor.getRPCMetrics().incrProxyOpPermitRejected(); + } + incrRejectedPermitForNs(nsId); + LOG.debug("Permit denied for ugi: {} for method: {}", + ugi, m.getMethodName()); + String msg = + "Router " + router.getRouterId() + + " is overloaded for NS: " + nsId; + throw new StandbyException(msg); } - LOG.debug("Permit denied for ugi: {} for method: {}", - ugi, m.getMethodName()); - String msg = - "Router " + router.getRouterId() + - " is overloaded for NS: " + nsId; - throw new StandbyException(msg); + incrAcceptedPermitForNs(nsId); } } @@ -1571,9 +1594,26 @@ private void releasePermit( } @VisibleForTesting - public AbstractRouterRpcFairnessPolicyController + public RouterRpcFairnessPolicyController getRouterRpcFairnessPolicyController() { - return (AbstractRouterRpcFairnessPolicyController - )routerRpcFairnessPolicyController; + return routerRpcFairnessPolicyController; + } + + private void incrRejectedPermitForNs(String ns) { + rejectedPermitsPerNs.computeIfAbsent(ns, k -> new LongAdder()).increment(); + } + + public Long getRejectedPermitForNs(String ns) { + return rejectedPermitsPerNs.containsKey(ns) ? + rejectedPermitsPerNs.get(ns).longValue() : 0L; + } + + private void incrAcceptedPermitForNs(String ns) { + acceptedPermitsPerNs.computeIfAbsent(ns, k -> new LongAdder()).increment(); + } + + public Long getAcceptedPermitForNs(String ns) { + return acceptedPermitsPerNs.containsKey(ns) ? + acceptedPermitsPerNs.get(ns).longValue() : 0L; } } diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterWebHdfsMethods.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterWebHdfsMethods.java index cd233e0bed7a2..a812f198f5e57 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterWebHdfsMethods.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterWebHdfsMethods.java @@ -72,6 +72,8 @@ import org.apache.hadoop.hdfs.web.resources.RenewerParam; import org.apache.hadoop.hdfs.web.resources.ReplicationParam; import org.apache.hadoop.hdfs.web.resources.SnapshotNameParam; +import org.apache.hadoop.hdfs.web.resources.SnapshotDiffStartPathParam; +import org.apache.hadoop.hdfs.web.resources.SnapshotDiffIndexParam; import org.apache.hadoop.hdfs.web.resources.StartAfterParam; import org.apache.hadoop.hdfs.web.resources.StoragePolicyParam; import org.apache.hadoop.hdfs.web.resources.StorageSpaceQuotaParam; @@ -335,6 +337,8 @@ protected Response get( final FsActionParam fsAction, final SnapshotNameParam snapshotName, final OldSnapshotNameParam oldSnapshotName, + final SnapshotDiffStartPathParam snapshotDiffStartPath, + final SnapshotDiffIndexParam snapshotDiffIndex, final TokenKindParam tokenKind, final TokenServiceParam tokenService, final NoRedirectParam noredirectParam, @@ -383,6 +387,7 @@ protected Response get( return super.get(ugi, delegation, username, doAsUser, fullpath, op, offset, length, renewer, bufferSize, xattrNames, xattrEncoding, excludeDatanodes, fsAction, snapshotName, oldSnapshotName, + snapshotDiffStartPath, snapshotDiffIndex, tokenKind, tokenService, noredirectParam, startAfter); } default: diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/site/markdown/HDFSRouterFederation.md b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/site/markdown/HDFSRouterFederation.md index 151289fa7f7d8..de92f86d63c52 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/site/markdown/HDFSRouterFederation.md +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/site/markdown/HDFSRouterFederation.md @@ -500,7 +500,6 @@ Isolation and dedicated assignment of RPC handlers across all configured downstr | Property | Default | Description| |:---- |:---- |:---- | -| dfs.federation.router.fairness.enable | `false` | If `true`, dedicated RPC handlers will be assigned to each nameservice based on the fairness assignment policy configured. | | dfs.federation.router.fairness.policy.controller.class | `org.apache.hadoop.hdfs.server.federation.fairness.NoRouterRpcFairnessPolicyController` | Default handler allocation model to be used if isolation feature is enabled. Recommend to use `org.apache.hadoop.hdfs.server.federation.fairness.StaticRouterRpcFairnessPolicyController` to fully use the feature. | | dfs.federation.router.fairness.handler.count.*EXAMPLENAMESERVICE* | | Dedicated handler assigned to a specific nameservice. If none is specified equal allocation is done across all nameservices. | | dfs.federation.router.fairness.handler.count.concurrent | | Dedicated handler assigned to fan out calls such as `renewLease`. | diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/MiniRouterDFSCluster.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/MiniRouterDFSCluster.java index 8a7a03e018b95..ac6ecd4398cba 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/MiniRouterDFSCluster.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/MiniRouterDFSCluster.java @@ -86,6 +86,8 @@ import org.apache.hadoop.hdfs.server.federation.resolver.NamenodeStatusReport; import org.apache.hadoop.hdfs.server.federation.router.Router; import org.apache.hadoop.hdfs.server.federation.router.RouterClient; +import org.apache.hadoop.hdfs.server.federation.router.RouterRpcClient; +import org.apache.hadoop.hdfs.server.federation.router.RouterRpcServer; import org.apache.hadoop.hdfs.server.namenode.FSImage; import org.apache.hadoop.hdfs.server.namenode.NameNode; import org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider; @@ -267,6 +269,14 @@ public DFSClient getClient() throws IOException, URISyntaxException { public Configuration getConf() { return conf; } + + public RouterRpcServer getRouterRpcServer() { + return router.getRpcServer(); + } + + public RouterRpcClient getRouterRpcClient() { + return getRouterRpcServer().getRPCClient(); + } } /** diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/fairness/TestRouterHandlersFairness.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/fairness/TestRouterHandlersFairness.java index c3fc324255c08..2d27c66e37ee0 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/fairness/TestRouterHandlersFairness.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/fairness/TestRouterHandlersFairness.java @@ -41,8 +41,7 @@ import org.slf4j.LoggerFactory; /** - * Test the Router handlers fairness control rejects - * requests when the handlers are overloaded. + * Test the Router handlers fairness control rejects and accepts requests. */ public class TestRouterHandlersFairness { @@ -126,6 +125,12 @@ private void startLoadTest(final boolean isConcurrent, final boolean fairness) throws Exception { RouterContext routerContext = cluster.getRandomRouter(); + URI address = routerContext.getFileSystemURI(); + Configuration conf = new HdfsConfiguration(); + final int numOps = 10; + AtomicInteger overloadException = new AtomicInteger(); + + // Test when handlers are overloaded if (fairness) { if (isConcurrent) { LOG.info("Taking fanout lock first"); @@ -142,41 +147,14 @@ private void startLoadTest(final boolean isConcurrent, final boolean fairness) } } } - URI address = routerContext.getFileSystemURI(); - Configuration conf = new HdfsConfiguration(); - final int numOps = 10; - final AtomicInteger overloadException = new AtomicInteger(); + int originalRejectedPermits = getTotalRejectedPermits(routerContext); - for (int i = 0; i < numOps; i++) { - DFSClient routerClient = null; - try { - routerClient = new DFSClient(address, conf); - String clientName = routerClient.getClientName(); - ClientProtocol routerProto = routerClient.getNamenode(); - if (isConcurrent) { - invokeConcurrent(routerProto, clientName); - } else { - invokeSequential(routerProto); - } - } catch (RemoteException re) { - IOException ioe = re.unwrapRemoteException(); - assertTrue("Wrong exception: " + ioe, - ioe instanceof StandbyException); - assertExceptionContains("is overloaded for NS", ioe); - overloadException.incrementAndGet(); - } catch (Throwable e) { - throw e; - } finally { - if (routerClient != null) { - try { - routerClient.close(); - } catch (IOException e) { - LOG.error("Cannot close the client"); - } - } - } - overloadException.get(); - } + // |- All calls should fail since permits not released + innerCalls(address, numOps, isConcurrent, conf, overloadException); + + int latestRejectedPermits = getTotalRejectedPermits(routerContext); + assertEquals(latestRejectedPermits - originalRejectedPermits, + overloadException.get()); if (fairness) { assertTrue(overloadException.get() > 0); @@ -197,6 +175,17 @@ private void startLoadTest(final boolean isConcurrent, final boolean fairness) assertEquals("Number of failed RPCs without fairness configured", 0, overloadException.get()); } + + // Test when handlers are not overloaded + int originalAcceptedPermits = getTotalAcceptedPermits(routerContext); + overloadException = new AtomicInteger(); + + // |- All calls should succeed since permits not acquired + innerCalls(address, numOps, isConcurrent, conf, overloadException); + + int latestAcceptedPermits = getTotalAcceptedPermits(routerContext); + assertEquals(latestAcceptedPermits - originalAcceptedPermits, numOps); + assertEquals(overloadException.get(), 0); } private void invokeSequential(ClientProtocol routerProto) throws IOException { @@ -208,4 +197,59 @@ private void invokeConcurrent(ClientProtocol routerProto, String clientName) routerProto.renewLease(clientName); } + private int getTotalRejectedPermits(RouterContext routerContext) { + int totalRejectedPermits = 0; + for (String ns : cluster.getNameservices()) { + totalRejectedPermits += routerContext.getRouterRpcClient() + .getRejectedPermitForNs(ns); + } + totalRejectedPermits += routerContext.getRouterRpcClient() + .getRejectedPermitForNs(RouterRpcFairnessConstants.CONCURRENT_NS); + return totalRejectedPermits; + } + + private int getTotalAcceptedPermits(RouterContext routerContext) { + int totalAcceptedPermits = 0; + for (String ns : cluster.getNameservices()) { + totalAcceptedPermits += routerContext.getRouterRpcClient() + .getAcceptedPermitForNs(ns); + } + totalAcceptedPermits += routerContext.getRouterRpcClient() + .getAcceptedPermitForNs(RouterRpcFairnessConstants.CONCURRENT_NS); + return totalAcceptedPermits; + } + + private void innerCalls(URI address, int numOps, boolean isConcurrent, + Configuration conf, AtomicInteger overloadException) throws IOException { + for (int i = 0; i < numOps; i++) { + DFSClient routerClient = null; + try { + routerClient = new DFSClient(address, conf); + String clientName = routerClient.getClientName(); + ClientProtocol routerProto = routerClient.getNamenode(); + if (isConcurrent) { + invokeConcurrent(routerProto, clientName); + } else { + invokeSequential(routerProto); + } + } catch (RemoteException re) { + IOException ioe = re.unwrapRemoteException(); + assertTrue("Wrong exception: " + ioe, + ioe instanceof StandbyException); + assertExceptionContains("is overloaded for NS", ioe); + overloadException.incrementAndGet(); + } catch (Throwable e) { + throw e; + } finally { + if (routerClient != null) { + try { + routerClient.close(); + } catch (IOException e) { + LOG.error("Cannot close the client"); + } + } + } + overloadException.get(); + } + } } diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/fairness/TestRouterRpcFairnessPolicyController.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/fairness/TestRouterRpcFairnessPolicyController.java index 8e816643ac5bb..9b301382d9596 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/fairness/TestRouterRpcFairnessPolicyController.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/fairness/TestRouterRpcFairnessPolicyController.java @@ -30,6 +30,7 @@ import static org.apache.hadoop.hdfs.server.federation.router.RBFConfigKeys.DFS_ROUTER_HANDLER_COUNT_KEY; import static org.apache.hadoop.hdfs.server.federation.router.RBFConfigKeys.DFS_ROUTER_MONITOR_NAMENODE; import static org.apache.hadoop.hdfs.server.federation.router.RBFConfigKeys.DFS_ROUTER_FAIR_HANDLER_COUNT_KEY_PREFIX; +import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; @@ -101,6 +102,26 @@ public void testAllocationErrorForLowDefaultHandlersPerNS() { verifyInstantiationError(conf, 1, 3); } + @Test + public void testGetAvailableHandlerOnPerNs() { + RouterRpcFairnessPolicyController routerRpcFairnessPolicyController + = getFairnessPolicyController(30); + assertEquals("{\"concurrent\":10,\"ns2\":10,\"ns1\":10}", + routerRpcFairnessPolicyController.getAvailableHandlerOnPerNs()); + routerRpcFairnessPolicyController.acquirePermit("ns1"); + assertEquals("{\"concurrent\":10,\"ns2\":10,\"ns1\":9}", + routerRpcFairnessPolicyController.getAvailableHandlerOnPerNs()); + } + + @Test + public void testGetAvailableHandlerOnPerNsForNoFairness() { + Configuration conf = new Configuration(); + RouterRpcFairnessPolicyController routerRpcFairnessPolicyController = + FederationUtil.newFairnessPolicyController(conf); + assertEquals("N/A", + routerRpcFairnessPolicyController.getAvailableHandlerOnPerNs()); + } + @Test public void testAllocationErrorForLowPreconfiguredHandlers() { Configuration conf = createConf(1); diff --git a/hadoop-hdfs-project/hadoop-hdfs/pom.xml b/hadoop-hdfs-project/hadoop-hdfs/pom.xml index ff4a8dc228aa6..863094cb0cb9c 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/pom.xml +++ b/hadoop-hdfs-project/hadoop-hdfs/pom.xml @@ -470,8 +470,8 @@ https://maven.apache.org/xsd/maven-4.0.0.xsd"> true Use hadoop-common provided VisibleForTesting rather than the one provided by Guava - org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting - com.google.common.annotations.VisibleForTesting + org.apache.hadoop.thirdparty.com.google.common.base.Preconditions + com.google.common.base.Preconditions diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java index 942929ac285e9..0526f1e4412dc 100755 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java @@ -727,6 +727,10 @@ public class DFSConfigKeys extends CommonConfigurationKeys { "dfs.datanode.metrics.logger.period.seconds"; public static final int DFS_DATANODE_METRICS_LOGGER_PERIOD_SECONDS_DEFAULT = 600; + public static final String DFS_NAMENODE_AUDIT_LOG_WITH_REMOTE_PORT_KEY = + "dfs.namenode.audit.log.with.remote.port"; + public static final boolean DFS_NAMENODE_AUDIT_LOG_WITH_REMOTE_PORT_DEFAULT = + false; /** * The maximum number of getBlocks RPCs data movement utilities can make to * a NameNode per second. Values <= 0 disable throttling. This affects @@ -1195,6 +1199,12 @@ public class DFSConfigKeys extends CommonConfigurationKeys { "dfs.namenode.available-space-block-placement-policy.balanced-space-preference-fraction"; public static final float DFS_NAMENODE_AVAILABLE_SPACE_BLOCK_PLACEMENT_POLICY_BALANCED_SPACE_PREFERENCE_FRACTION_DEFAULT = 0.6f; + public static final String + DFS_NAMENODE_AVAILABLE_SPACE_BLOCK_PLACEMENT_POLICY_BALANCED_SPACE_TOLERANCE_KEY = + "dfs.namenode.available-space-block-placement-policy.balanced-space-tolerance"; + public static final int + DFS_NAMENODE_AVAILABLE_SPACE_BLOCK_PLACEMENT_POLICY_BALANCED_SPACE_TOLERANCE_DEFAULT = + 5; public static final String DFS_NAMENODE_AVAILABLE_SPACE_RACK_FAULT_TOLERANT_BLOCK_PLACEMENT_POLICY_BALANCED_SPACE_PREFERENCE_FRACTION_KEY = "dfs.namenode.available-space-rack-fault-tolerant-block-placement-policy" @@ -1202,6 +1212,13 @@ public class DFSConfigKeys extends CommonConfigurationKeys { public static final float DFS_NAMENODE_AVAILABLE_SPACE_BLOCK_RACK_FAULT_TOLERANT_PLACEMENT_POLICY_BALANCED_SPACE_PREFERENCE_FRACTION_DEFAULT = 0.6f; + public static final String + DFS_NAMENODE_AVAILABLE_SPACE_RACK_FAULT_TOLERANT_BLOCK_PLACEMENT_POLICY_BALANCED_SPACE_TOLERANCE_KEY = + "dfs.namenode.available-space-rack-fault-tolerant-block-placement-policy" + + ".balanced-space-tolerance"; + public static final int + DFS_NAMENODE_AVAILABLE_SPACE_BLOCK_RACK_FAULT_TOLERANT_PLACEMENT_POLICY_BALANCED_SPACE_TOLERANCE_DEFAULT = + 5; public static final String DFS_NAMENODE_AVAILABLE_SPACE_BLOCK_PLACEMENT_POLICY_BALANCE_LOCAL_NODE_KEY = "dfs.namenode.available-space-block-placement-policy.balance-local-node"; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java index 968bad3bbe5e8..50c947b05941a 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java @@ -110,7 +110,7 @@ import org.apache.hadoop.classification.VisibleForTesting; import org.apache.hadoop.thirdparty.com.google.common.base.Joiner; -import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; +import org.apache.hadoop.util.Preconditions; import org.apache.hadoop.thirdparty.protobuf.BlockingService; @InterfaceAudience.Private @@ -310,7 +310,11 @@ public static String byteArray2PathString(final byte[][] components, // specifically not using StringBuilder to more efficiently build // string w/o excessive byte[] copies and charset conversions. final int range = offset + length; - Preconditions.checkPositionIndexes(offset, range, components.length); + if (offset < 0 || range < offset || range > components.length) { + throw new IndexOutOfBoundsException( + "Incorrect index [offset, range, size] [" + + offset + ", " + range + ", " + components.length + "]"); + } if (length == 0) { return ""; } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/HAUtil.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/HAUtil.java index 58034ddad4705..e1c2fcec7f285 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/HAUtil.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/HAUtil.java @@ -57,7 +57,7 @@ import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.thirdparty.com.google.common.base.Joiner; -import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; +import org.apache.hadoop.util.Preconditions; import org.apache.hadoop.util.Lists; import org.slf4j.LoggerFactory; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/net/DFSNetworkTopology.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/net/DFSNetworkTopology.java index 35d81201ad1d4..77e610434b49b 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/net/DFSNetworkTopology.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/net/DFSNetworkTopology.java @@ -18,7 +18,7 @@ package org.apache.hadoop.hdfs.net; import org.apache.hadoop.classification.VisibleForTesting; -import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; +import org.apache.hadoop.util.Preconditions; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.StorageType; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/net/DFSTopologyNodeImpl.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/net/DFSTopologyNodeImpl.java index 0d0d209d977f3..5f23a5b6b3e48 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/net/DFSTopologyNodeImpl.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/net/DFSTopologyNodeImpl.java @@ -18,7 +18,7 @@ package org.apache.hadoop.hdfs.net; import org.apache.hadoop.classification.VisibleForTesting; -import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; +import org.apache.hadoop.util.Preconditions; import org.apache.hadoop.fs.StorageType; import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeDescriptor; import org.apache.hadoop.net.InnerNode; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/BlockListAsLongs.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/BlockListAsLongs.java index eb7caead4bc0e..3e30a6309d903 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/BlockListAsLongs.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/BlockListAsLongs.java @@ -35,7 +35,7 @@ import org.apache.hadoop.hdfs.server.datanode.Replica; import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsVolumeSpi; import org.apache.hadoop.classification.VisibleForTesting; -import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; +import org.apache.hadoop.util.Preconditions; import org.apache.hadoop.thirdparty.protobuf.ByteString; import org.apache.hadoop.thirdparty.protobuf.CodedInputStream; import org.apache.hadoop.thirdparty.protobuf.CodedOutputStream; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/CacheDirective.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/CacheDirective.java index aef009a03a1dc..d03ad3d4d7ce6 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/CacheDirective.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/CacheDirective.java @@ -17,8 +17,6 @@ */ package org.apache.hadoop.hdfs.protocol; -import static org.apache.hadoop.thirdparty.com.google.common.base.Preconditions.checkNotNull; - import java.util.Date; import org.apache.hadoop.classification.InterfaceAudience; @@ -28,7 +26,7 @@ import org.apache.hadoop.util.IntrusiveCollection; import org.apache.hadoop.util.IntrusiveCollection.Element; -import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; +import org.apache.hadoop.util.Preconditions; /** * Namenode class that tracks state related to a cached path. @@ -63,7 +61,7 @@ public CacheDirective(long id, String path, short replication, long expiryTime) { Preconditions.checkArgument(id > 0); this.id = id; - this.path = checkNotNull(path); + this.path = Preconditions.checkNotNull(path); Preconditions.checkArgument(replication > 0); this.replication = replication; this.expiryTime = expiryTime; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/DatanodeProtocolServerSideTranslatorPB.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/DatanodeProtocolServerSideTranslatorPB.java index 9244b9fef8571..89cc6c79a1214 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/DatanodeProtocolServerSideTranslatorPB.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/DatanodeProtocolServerSideTranslatorPB.java @@ -61,7 +61,7 @@ import org.apache.hadoop.hdfs.server.protocol.StorageReport; import org.apache.hadoop.hdfs.server.protocol.VolumeFailureSummary; -import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; +import org.apache.hadoop.util.Preconditions; import org.apache.hadoop.thirdparty.protobuf.RpcController; import org.apache.hadoop.thirdparty.protobuf.ServiceException; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/AsyncLoggerSet.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/AsyncLoggerSet.java index c1771a68c07cd..624e574024c0b 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/AsyncLoggerSet.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/AsyncLoggerSet.java @@ -36,7 +36,7 @@ import org.apache.hadoop.classification.VisibleForTesting; import org.apache.hadoop.thirdparty.com.google.common.base.Joiner; -import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; +import org.apache.hadoop.util.Preconditions; import org.apache.hadoop.thirdparty.com.google.common.collect.ImmutableList; import org.apache.hadoop.thirdparty.com.google.common.collect.Maps; import org.apache.hadoop.thirdparty.com.google.common.util.concurrent.ListenableFuture; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/IPCLoggerChannel.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/IPCLoggerChannel.java index 4688f8db9c74f..548525ba012b8 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/IPCLoggerChannel.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/IPCLoggerChannel.java @@ -58,7 +58,7 @@ import org.apache.hadoop.util.StopWatch; import org.apache.hadoop.classification.VisibleForTesting; -import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; +import org.apache.hadoop.util.Preconditions; import org.apache.hadoop.thirdparty.com.google.common.net.InetAddresses; import org.apache.hadoop.thirdparty.com.google.common.util.concurrent.FutureCallback; import org.apache.hadoop.thirdparty.com.google.common.util.concurrent.Futures; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/QuorumCall.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/QuorumCall.java index e2a169aeb3c5f..c3ad872f30d0d 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/QuorumCall.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/QuorumCall.java @@ -30,7 +30,7 @@ import org.apache.hadoop.util.Timer; import org.apache.hadoop.thirdparty.com.google.common.base.Joiner; -import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; +import org.apache.hadoop.util.Preconditions; import org.apache.hadoop.thirdparty.com.google.common.collect.Maps; import org.apache.hadoop.thirdparty.com.google.common.util.concurrent.FutureCallback; import org.apache.hadoop.thirdparty.com.google.common.util.concurrent.Futures; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/QuorumException.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/QuorumException.java index 1f60e3d468821..65b227ccea8d4 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/QuorumException.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/QuorumException.java @@ -23,7 +23,7 @@ import org.apache.hadoop.util.StringUtils; import org.apache.hadoop.thirdparty.com.google.common.base.Joiner; -import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; +import org.apache.hadoop.util.Preconditions; /** * Exception thrown when too many exceptions occur while gathering diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/QuorumJournalManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/QuorumJournalManager.java index 4f8893952bd61..e25485e6cd78e 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/QuorumJournalManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/QuorumJournalManager.java @@ -61,7 +61,7 @@ import org.apache.hadoop.classification.VisibleForTesting; import org.apache.hadoop.thirdparty.com.google.common.base.Joiner; -import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; +import org.apache.hadoop.util.Preconditions; import org.apache.hadoop.thirdparty.protobuf.TextFormat; /** diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/SegmentRecoveryComparator.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/SegmentRecoveryComparator.java index 4b2a518ac0dff..38006a499d97e 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/SegmentRecoveryComparator.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/SegmentRecoveryComparator.java @@ -23,7 +23,7 @@ import org.apache.hadoop.hdfs.qjournal.protocol.QJournalProtocolProtos.PrepareRecoveryResponseProto; import org.apache.hadoop.hdfs.qjournal.protocol.QJournalProtocolProtos.SegmentStateProto; -import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; +import org.apache.hadoop.util.Preconditions; import org.apache.hadoop.thirdparty.com.google.common.collect.ComparisonChain; import org.apache.hadoop.thirdparty.com.google.common.primitives.Booleans; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/Journal.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/Journal.java index 296d4af59a496..6b9b40871816b 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/Journal.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/Journal.java @@ -73,7 +73,7 @@ import org.apache.hadoop.classification.VisibleForTesting; import org.apache.hadoop.thirdparty.com.google.common.base.Charsets; -import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; +import org.apache.hadoop.util.Preconditions; import org.apache.hadoop.thirdparty.com.google.common.collect.ImmutableList; import org.apache.hadoop.thirdparty.protobuf.TextFormat; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/JournalNode.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/JournalNode.java index a20cd218521f9..970078eeea9a1 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/JournalNode.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/JournalNode.java @@ -18,7 +18,7 @@ package org.apache.hadoop.hdfs.qjournal.server; import org.apache.hadoop.classification.VisibleForTesting; -import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; +import org.apache.hadoop.util.Preconditions; import org.apache.hadoop.thirdparty.com.google.common.base.Strings; import org.apache.hadoop.thirdparty.com.google.common.collect.Maps; import org.apache.hadoop.util.Lists; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/security/token/block/BlockTokenSecretManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/security/token/block/BlockTokenSecretManager.java index 7517ebb3f1853..18cdeaebfda82 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/security/token/block/BlockTokenSecretManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/security/token/block/BlockTokenSecretManager.java @@ -46,7 +46,7 @@ import org.apache.hadoop.util.Timer; import org.apache.hadoop.classification.VisibleForTesting; -import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; +import org.apache.hadoop.util.Preconditions; import org.apache.hadoop.thirdparty.com.google.common.collect.HashMultiset; import org.apache.hadoop.thirdparty.com.google.common.collect.Multiset; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/security/token/delegation/DelegationTokenSecretManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/security/token/delegation/DelegationTokenSecretManager.java index aef2306a55c15..f97c2f2c80b71 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/security/token/delegation/DelegationTokenSecretManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/security/token/delegation/DelegationTokenSecretManager.java @@ -50,7 +50,7 @@ import org.apache.hadoop.security.token.delegation.AbstractDelegationTokenSecretManager; import org.apache.hadoop.security.token.delegation.DelegationKey; -import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; +import org.apache.hadoop.util.Preconditions; import org.apache.hadoop.thirdparty.protobuf.ByteString; /** diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Balancer.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Balancer.java index 664ac62aeaef9..02004f337c10a 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Balancer.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Balancer.java @@ -17,7 +17,6 @@ */ package org.apache.hadoop.hdfs.server.balancer; -import static org.apache.hadoop.thirdparty.com.google.common.base.Preconditions.checkArgument; import static org.apache.hadoop.hdfs.protocol.BlockType.CONTIGUOUS; import java.io.IOException; @@ -75,7 +74,7 @@ import org.apache.hadoop.util.Tool; import org.apache.hadoop.util.ToolRunner; -import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; +import org.apache.hadoop.util.Preconditions; /**

The balancer is a tool that balances disk space usage on an HDFS cluster * when some datanodes become full or when new empty nodes join the cluster. @@ -998,7 +997,7 @@ static BalancerParameters parse(String[] args) { try { for(int i = 0; i < args.length; i++) { if ("-threshold".equalsIgnoreCase(args[i])) { - checkArgument(++i < args.length, + Preconditions.checkArgument(++i < args.length, "Threshold value is missing: args = " + Arrays.toString(args)); try { double threshold = Double.parseDouble(args[i]); @@ -1015,7 +1014,7 @@ static BalancerParameters parse(String[] args) { throw e; } } else if ("-policy".equalsIgnoreCase(args[i])) { - checkArgument(++i < args.length, + Preconditions.checkArgument(++i < args.length, "Policy value is missing: args = " + Arrays.toString(args)); try { b.setBalancingPolicy(BalancingPolicy.parse(args[i])); @@ -1036,7 +1035,7 @@ static BalancerParameters parse(String[] args) { i = processHostList(args, i, "source", sourceNodes); b.setSourceNodes(sourceNodes); } else if ("-blockpools".equalsIgnoreCase(args[i])) { - checkArgument( + Preconditions.checkArgument( ++i < args.length, "blockpools value is missing: args = " + Arrays.toString(args)); @@ -1045,7 +1044,7 @@ static BalancerParameters parse(String[] args) { + blockpools.toString()); b.setBlockpools(blockpools); } else if ("-idleiterations".equalsIgnoreCase(args[i])) { - checkArgument(++i < args.length, + Preconditions.checkArgument(++i < args.length, "idleiterations value is missing: args = " + Arrays .toString(args)); int maxIdleIteration = Integer.parseInt(args[i]); @@ -1061,7 +1060,7 @@ static BalancerParameters parse(String[] args) { b.setRunAsService(true); LOG.info("Balancer will run as a long running service"); } else if ("-hotBlockTimeInterval".equalsIgnoreCase(args[i])) { - checkArgument(++i < args.length, + Preconditions.checkArgument(++i < args.length, "hotBlockTimeInterval value is missing: args = " + Arrays.toString(args)); long hotBlockTimeInterval = Long.parseLong(args[i]); @@ -1077,7 +1076,7 @@ static BalancerParameters parse(String[] args) { + Arrays.toString(args)); } } - checkArgument(excludedNodes == null || includedNodes == null, + Preconditions.checkArgument(excludedNodes == null || includedNodes == null, "-exclude and -include options cannot be specified together."); } catch(RuntimeException e) { printUsage(System.err); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Dispatcher.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Dispatcher.java index 90fd7e6fa1218..bc7cad54166f9 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Dispatcher.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Dispatcher.java @@ -85,7 +85,7 @@ import org.apache.hadoop.util.Time; import org.apache.hadoop.classification.VisibleForTesting; -import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; +import org.apache.hadoop.util.Preconditions; /** Dispatching block replica moves between datanodes. */ @InterfaceAudience.Private diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/NameNodeConnector.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/NameNodeConnector.java index 750c351e81d6a..ce0fb968bbb9f 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/NameNodeConnector.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/NameNodeConnector.java @@ -31,7 +31,7 @@ import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicLong; -import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; +import org.apache.hadoop.util.Preconditions; import org.apache.hadoop.thirdparty.com.google.common.util.concurrent.RateLimiter; import org.apache.hadoop.ha.HAServiceProtocol; import org.apache.hadoop.hdfs.DFSConfigKeys; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/AvailableSpaceBlockPlacementPolicy.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/AvailableSpaceBlockPlacementPolicy.java index 6926c3e01c448..9e349423daa77 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/AvailableSpaceBlockPlacementPolicy.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/AvailableSpaceBlockPlacementPolicy.java @@ -20,6 +20,8 @@ import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_AVAILABLE_SPACE_BLOCK_PLACEMENT_POLICY_BALANCED_SPACE_PREFERENCE_FRACTION_KEY; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_AVAILABLE_SPACE_BLOCK_PLACEMENT_POLICY_BALANCED_SPACE_PREFERENCE_FRACTION_DEFAULT; +import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_AVAILABLE_SPACE_BLOCK_PLACEMENT_POLICY_BALANCED_SPACE_TOLERANCE_DEFAULT; +import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_AVAILABLE_SPACE_BLOCK_PLACEMENT_POLICY_BALANCED_SPACE_TOLERANCE_KEY; import java.util.Collection; import java.util.EnumMap; @@ -27,7 +29,7 @@ import java.util.Random; import java.util.Set; -import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; +import org.apache.hadoop.util.Preconditions; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.hadoop.conf.Configuration; @@ -47,6 +49,8 @@ public class AvailableSpaceBlockPlacementPolicy extends private static final Random RAND = new Random(); private int balancedPreference = (int) (100 * DFS_NAMENODE_AVAILABLE_SPACE_BLOCK_PLACEMENT_POLICY_BALANCED_SPACE_PREFERENCE_FRACTION_DEFAULT); + private int balancedSpaceTolerance = + DFS_NAMENODE_AVAILABLE_SPACE_BLOCK_PLACEMENT_POLICY_BALANCED_SPACE_TOLERANCE_DEFAULT; private boolean optimizeLocal; @Override @@ -59,9 +63,14 @@ public void initialize(Configuration conf, FSClusterStats stats, DFS_NAMENODE_AVAILABLE_SPACE_BLOCK_PLACEMENT_POLICY_BALANCED_SPACE_PREFERENCE_FRACTION_DEFAULT); LOG.info("Available space block placement policy initialized: " - + DFSConfigKeys.DFS_NAMENODE_AVAILABLE_SPACE_BLOCK_PLACEMENT_POLICY_BALANCED_SPACE_PREFERENCE_FRACTION_KEY + + DFS_NAMENODE_AVAILABLE_SPACE_BLOCK_PLACEMENT_POLICY_BALANCED_SPACE_PREFERENCE_FRACTION_KEY + " = " + balancedPreferencePercent); + balancedSpaceTolerance = + conf.getInt( + DFS_NAMENODE_AVAILABLE_SPACE_BLOCK_PLACEMENT_POLICY_BALANCED_SPACE_TOLERANCE_KEY, + DFS_NAMENODE_AVAILABLE_SPACE_BLOCK_PLACEMENT_POLICY_BALANCED_SPACE_TOLERANCE_DEFAULT); + optimizeLocal = conf.getBoolean( DFSConfigKeys.DFS_NAMENODE_AVAILABLE_SPACE_BLOCK_PLACEMENT_POLICY_BALANCE_LOCAL_NODE_KEY, DFSConfigKeys.DFS_NAMENODE_AVAILABLE_SPACE_BLOCK_PLACEMENT_POLICY_BALANCE_LOCAL_NODE_DEFAULT); @@ -77,6 +86,16 @@ public void initialize(Configuration conf, FSClusterStats stats, + " is less than 0.5 so datanodes with more used percent will" + " receive more block allocations."); } + + if (balancedSpaceTolerance > 20 || balancedSpaceTolerance < 0) { + LOG.warn("The value of " + + DFS_NAMENODE_AVAILABLE_SPACE_BLOCK_PLACEMENT_POLICY_BALANCED_SPACE_TOLERANCE_KEY + + " is invalid, Current value is " + balancedSpaceTolerance + ", Default value " + + DFS_NAMENODE_AVAILABLE_SPACE_BLOCK_PLACEMENT_POLICY_BALANCED_SPACE_TOLERANCE_DEFAULT + + " will be used instead."); + balancedSpaceTolerance = + DFS_NAMENODE_AVAILABLE_SPACE_BLOCK_PLACEMENT_POLICY_BALANCED_SPACE_TOLERANCE_DEFAULT; + } balancedPreference = (int) (100 * balancedPreferencePercent); } @@ -183,7 +202,7 @@ private DatanodeDescriptor select(DatanodeDescriptor a, DatanodeDescriptor b, protected int compareDataNode(final DatanodeDescriptor a, final DatanodeDescriptor b, boolean isBalanceLocal) { if (a.equals(b) - || Math.abs(a.getDfsUsedPercent() - b.getDfsUsedPercent()) < 5 || (( + || Math.abs(a.getDfsUsedPercent() - b.getDfsUsedPercent()) < balancedSpaceTolerance || (( isBalanceLocal && a.getDfsUsedPercent() < 50))) { return 0; } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/AvailableSpaceRackFaultTolerantBlockPlacementPolicy.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/AvailableSpaceRackFaultTolerantBlockPlacementPolicy.java index e673347de630d..85de2fee3e2f1 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/AvailableSpaceRackFaultTolerantBlockPlacementPolicy.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/AvailableSpaceRackFaultTolerantBlockPlacementPolicy.java @@ -18,7 +18,7 @@ package org.apache.hadoop.hdfs.server.blockmanagement; -import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; +import org.apache.hadoop.util.Preconditions; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.StorageType; import org.apache.hadoop.hdfs.DFSConfigKeys; @@ -32,7 +32,9 @@ import java.util.Random; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_AVAILABLE_SPACE_BLOCK_RACK_FAULT_TOLERANT_PLACEMENT_POLICY_BALANCED_SPACE_PREFERENCE_FRACTION_DEFAULT; +import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_AVAILABLE_SPACE_BLOCK_RACK_FAULT_TOLERANT_PLACEMENT_POLICY_BALANCED_SPACE_TOLERANCE_DEFAULT; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_AVAILABLE_SPACE_RACK_FAULT_TOLERANT_BLOCK_PLACEMENT_POLICY_BALANCED_SPACE_PREFERENCE_FRACTION_KEY; +import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_AVAILABLE_SPACE_RACK_FAULT_TOLERANT_BLOCK_PLACEMENT_POLICY_BALANCED_SPACE_TOLERANCE_KEY; /** * Space balanced rack fault tolerant block placement policy. @@ -45,7 +47,8 @@ public class AvailableSpaceRackFaultTolerantBlockPlacementPolicy private static final Random RAND = new Random(); private int balancedPreference = (int) (100 * DFS_NAMENODE_AVAILABLE_SPACE_BLOCK_RACK_FAULT_TOLERANT_PLACEMENT_POLICY_BALANCED_SPACE_PREFERENCE_FRACTION_DEFAULT); - + private int balancedSpaceTolerance = + DFS_NAMENODE_AVAILABLE_SPACE_BLOCK_RACK_FAULT_TOLERANT_PLACEMENT_POLICY_BALANCED_SPACE_TOLERANCE_DEFAULT; @Override public void initialize(Configuration conf, FSClusterStats stats, NetworkTopology clusterMap, Host2NodesMap host2datanodeMap) { @@ -54,6 +57,10 @@ public void initialize(Configuration conf, FSClusterStats stats, DFS_NAMENODE_AVAILABLE_SPACE_RACK_FAULT_TOLERANT_BLOCK_PLACEMENT_POLICY_BALANCED_SPACE_PREFERENCE_FRACTION_KEY, DFS_NAMENODE_AVAILABLE_SPACE_BLOCK_RACK_FAULT_TOLERANT_PLACEMENT_POLICY_BALANCED_SPACE_PREFERENCE_FRACTION_DEFAULT); + balancedSpaceTolerance = conf.getInt( + DFS_NAMENODE_AVAILABLE_SPACE_RACK_FAULT_TOLERANT_BLOCK_PLACEMENT_POLICY_BALANCED_SPACE_TOLERANCE_KEY, + DFS_NAMENODE_AVAILABLE_SPACE_BLOCK_RACK_FAULT_TOLERANT_PLACEMENT_POLICY_BALANCED_SPACE_TOLERANCE_DEFAULT); + LOG.info("Available space rack fault tolerant block placement policy " + "initialized: " + DFSConfigKeys.DFS_NAMENODE_AVAILABLE_SPACE_RACK_FAULT_TOLERANT_BLOCK_PLACEMENT_POLICY_BALANCED_SPACE_PREFERENCE_FRACTION_KEY @@ -70,6 +77,18 @@ public void initialize(Configuration conf, FSClusterStats stats, + " is less than 0.5 so datanodes with more used percent will" + " receive more block allocations."); } + + + if (balancedSpaceTolerance > 20 || balancedSpaceTolerance < 0) { + LOG.warn("The value of " + + DFS_NAMENODE_AVAILABLE_SPACE_RACK_FAULT_TOLERANT_BLOCK_PLACEMENT_POLICY_BALANCED_SPACE_TOLERANCE_KEY + + " is invalid, Current value is " + balancedSpaceTolerance + ", Default value " + + DFS_NAMENODE_AVAILABLE_SPACE_BLOCK_RACK_FAULT_TOLERANT_PLACEMENT_POLICY_BALANCED_SPACE_TOLERANCE_DEFAULT + + " will be used instead."); + balancedSpaceTolerance = + DFS_NAMENODE_AVAILABLE_SPACE_BLOCK_RACK_FAULT_TOLERANT_PLACEMENT_POLICY_BALANCED_SPACE_TOLERANCE_DEFAULT; + } + balancedPreference = (int) (100 * balancedPreferencePercent); } @@ -118,7 +137,7 @@ private DatanodeDescriptor select(DatanodeDescriptor a, protected int compareDataNode(final DatanodeDescriptor a, final DatanodeDescriptor b) { if (a.equals(b) - || Math.abs(a.getDfsUsedPercent() - b.getDfsUsedPercent()) < 5) { + || Math.abs(a.getDfsUsedPercent() - b.getDfsUsedPercent()) < balancedSpaceTolerance) { return 0; } return a.getDfsUsedPercent() < b.getDfsUsedPercent() ? -1 : 1; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockIdManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockIdManager.java index df14107c322dd..a79e4c594ac2b 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockIdManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockIdManager.java @@ -18,7 +18,7 @@ package org.apache.hadoop.hdfs.server.blockmanagement; import org.apache.hadoop.classification.VisibleForTesting; -import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; +import org.apache.hadoop.util.Preconditions; import org.apache.hadoop.hdfs.protocol.Block; import org.apache.hadoop.hdfs.protocol.BlockType; import org.apache.hadoop.hdfs.protocol.HdfsConstants; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfo.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfo.java index b8047a8d08015..81a559e0f04c8 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfo.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfo.java @@ -22,7 +22,7 @@ import java.util.LinkedList; import java.util.List; -import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; +import org.apache.hadoop.util.Preconditions; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.fs.StorageType; import org.apache.hadoop.hdfs.protocol.Block; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfoContiguous.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfoContiguous.java index d68b72d6c6277..651f39f2b69a3 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfoContiguous.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfoContiguous.java @@ -17,7 +17,7 @@ */ package org.apache.hadoop.hdfs.server.blockmanagement; -import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; +import org.apache.hadoop.util.Preconditions; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.fs.StorageType; import org.apache.hadoop.hdfs.protocol.Block; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfoStriped.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfoStriped.java index 53368c98a4820..4b8d092935a00 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfoStriped.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfoStriped.java @@ -18,7 +18,7 @@ package org.apache.hadoop.hdfs.server.blockmanagement; import org.apache.hadoop.classification.VisibleForTesting; -import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; +import org.apache.hadoop.util.Preconditions; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.hdfs.protocol.Block; import org.apache.hadoop.hdfs.protocol.BlockType; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java index 69e9ca8e7f397..3252535ca4876 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java @@ -126,7 +126,7 @@ import org.apache.hadoop.util.Time; import org.apache.hadoop.classification.VisibleForTesting; -import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; +import org.apache.hadoop.util.Preconditions; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -2778,7 +2778,7 @@ public boolean processReport(final DatanodeID nodeID, + "storage report for {} from datanode {}", strBlockReportId, storageInfo.getStorageID(), - nodeID.getDatanodeUuid()); + nodeID); processFirstBlockReport(storageInfo, newReport); } else { // Block reports for provided storage are not diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManagerSafeMode.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManagerSafeMode.java index 6fac6dc2abec2..d825c5ecb31e8 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManagerSafeMode.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManagerSafeMode.java @@ -36,7 +36,7 @@ import org.apache.hadoop.util.Daemon; import org.apache.hadoop.classification.VisibleForTesting; -import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; +import org.apache.hadoop.util.Preconditions; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicy.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicy.java index 07df43b83dcf8..2a212d6261598 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicy.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicy.java @@ -25,7 +25,7 @@ import java.util.Map; import java.util.Set; -import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; +import org.apache.hadoop.util.Preconditions; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hdfs.AddBlockFlag; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java index c16fe59f596e6..3ea232258aa2b 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java @@ -26,7 +26,7 @@ import java.util.*; import java.util.concurrent.TimeUnit; -import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; +import org.apache.hadoop.util.Preconditions; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hdfs.AddBlockFlag; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockReportLeaseManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockReportLeaseManager.java index 3560a36dde1bc..f45daac142c86 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockReportLeaseManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockReportLeaseManager.java @@ -17,7 +17,7 @@ */ package org.apache.hadoop.hdfs.server.blockmanagement; -import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; +import org.apache.hadoop.util.Preconditions; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hdfs.DFSConfigKeys; import org.apache.hadoop.util.Time; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockStoragePolicySuite.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockStoragePolicySuite.java index 419dcfcddf1b9..35d3b0a20f8fc 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockStoragePolicySuite.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockStoragePolicySuite.java @@ -18,7 +18,7 @@ package org.apache.hadoop.hdfs.server.blockmanagement; import org.apache.hadoop.classification.VisibleForTesting; -import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; +import org.apache.hadoop.util.Preconditions; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.StorageType; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockToMarkCorrupt.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockToMarkCorrupt.java index 3ce5ef07acdca..bcbd94cd3cc7d 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockToMarkCorrupt.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockToMarkCorrupt.java @@ -19,7 +19,7 @@ import static org.apache.hadoop.hdfs.server.blockmanagement.CorruptReplicasMap.Reason; -import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; +import org.apache.hadoop.util.Preconditions; import org.apache.hadoop.hdfs.protocol.Block; /** diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/CacheReplicationMonitor.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/CacheReplicationMonitor.java index eab58124cb11c..4cc404f55c5d6 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/CacheReplicationMonitor.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/CacheReplicationMonitor.java @@ -55,7 +55,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; +import org.apache.hadoop.util.Preconditions; /** * Scans the namesystem, scheduling blocks to be cached as appropriate. diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeAdminDefaultMonitor.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeAdminDefaultMonitor.java index dfa7bf4e8381b..a217c9978e822 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeAdminDefaultMonitor.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeAdminDefaultMonitor.java @@ -17,7 +17,7 @@ */ package org.apache.hadoop.hdfs.server.blockmanagement; -import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; +import org.apache.hadoop.util.Preconditions; import org.apache.hadoop.hdfs.DFSConfigKeys; import org.apache.hadoop.hdfs.protocol.DatanodeID; import org.apache.hadoop.hdfs.server.namenode.INode; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeAdminManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeAdminManager.java index fecec597f5a5f..42b6ddd8c78fb 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeAdminManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeAdminManager.java @@ -17,7 +17,6 @@ */ package org.apache.hadoop.hdfs.server.blockmanagement; -import static org.apache.hadoop.thirdparty.com.google.common.base.Preconditions.checkArgument; import static org.apache.hadoop.util.Time.monotonicNow; import java.util.Queue; @@ -30,6 +29,7 @@ import org.apache.hadoop.hdfs.DFSConfigKeys; import org.apache.hadoop.hdfs.server.namenode.NameNode; import org.apache.hadoop.hdfs.server.namenode.Namesystem; +import org.apache.hadoop.util.Preconditions; import org.apache.hadoop.util.ReflectionUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -105,7 +105,7 @@ void activate(Configuration conf) { DFSConfigKeys.DFS_NAMENODE_DECOMMISSION_INTERVAL_KEY, DFSConfigKeys.DFS_NAMENODE_DECOMMISSION_INTERVAL_DEFAULT, TimeUnit.SECONDS); - checkArgument(intervalSecs >= 0, "Cannot set a negative " + + Preconditions.checkArgument(intervalSecs >= 0, "Cannot set a negative " + "value for " + DFSConfigKeys.DFS_NAMENODE_DECOMMISSION_INTERVAL_KEY); int blocksPerInterval = conf.getInt( @@ -122,7 +122,7 @@ void activate(Configuration conf) { DFSConfigKeys.DFS_NAMENODE_DECOMMISSION_BLOCKS_PER_INTERVAL_KEY); } - checkArgument(blocksPerInterval > 0, + Preconditions.checkArgument(blocksPerInterval > 0, "Must set a positive value for " + DFSConfigKeys.DFS_NAMENODE_DECOMMISSION_BLOCKS_PER_INTERVAL_KEY); @@ -130,8 +130,8 @@ void activate(Configuration conf) { DFSConfigKeys.DFS_NAMENODE_DECOMMISSION_MAX_CONCURRENT_TRACKED_NODES, DFSConfigKeys .DFS_NAMENODE_DECOMMISSION_MAX_CONCURRENT_TRACKED_NODES_DEFAULT); - checkArgument(maxConcurrentTrackedNodes >= 0, "Cannot set a negative " + - "value for " + Preconditions.checkArgument(maxConcurrentTrackedNodes >= 0, + "Cannot set a negative value for " + DFSConfigKeys.DFS_NAMENODE_DECOMMISSION_MAX_CONCURRENT_TRACKED_NODES); Class cls = null; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java index 23bcc98b7688f..d673c47ea0d32 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java @@ -24,7 +24,7 @@ import org.apache.commons.lang3.StringUtils; import org.apache.hadoop.classification.VisibleForTesting; -import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; +import org.apache.hadoop.util.Preconditions; import org.apache.hadoop.thirdparty.com.google.common.net.InetAddresses; import org.apache.hadoop.fs.StorageType; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/HostSet.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/HostSet.java index af0c92df5e272..d12e5fbae1362 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/HostSet.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/HostSet.java @@ -18,7 +18,7 @@ package org.apache.hadoop.hdfs.server.blockmanagement; -import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; +import org.apache.hadoop.util.Preconditions; import org.apache.hadoop.thirdparty.com.google.common.collect.HashMultimap; import org.apache.hadoop.thirdparty.com.google.common.collect.Multimap; import org.apache.hadoop.thirdparty.com.google.common.collect.UnmodifiableIterator; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/HdfsServerConstants.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/HdfsServerConstants.java index 58e9d5fdd9557..8e343f998a626 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/HdfsServerConstants.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/HdfsServerConstants.java @@ -31,7 +31,7 @@ import org.apache.hadoop.hdfs.server.namenode.FSDirectory; import org.apache.hadoop.hdfs.server.namenode.MetaRecoveryContext; -import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; +import org.apache.hadoop.util.Preconditions; import org.apache.hadoop.hdfs.server.namenode.NameNodeLayoutVersion; import org.apache.hadoop.util.StringUtils; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/JspHelper.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/JspHelper.java index 2f249655eed8e..4265c288e88d9 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/JspHelper.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/JspHelper.java @@ -198,6 +198,9 @@ public static String getRemoteAddr(HttpServletRequest request) { return remoteAddr; } + public static int getRemotePort(HttpServletRequest request) { + return request.getRemotePort(); + } /** * Expected user name should be a short name. diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/Storage.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/Storage.java index 83a82566f6c3e..7a4535f33898c 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/Storage.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/Storage.java @@ -54,7 +54,7 @@ import org.apache.hadoop.util.VersionInfo; import org.apache.hadoop.thirdparty.com.google.common.base.Charsets; -import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; +import org.apache.hadoop.util.Preconditions; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/Util.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/Util.java index 816e278ccf56b..5039db6ceb488 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/Util.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/Util.java @@ -54,7 +54,7 @@ import org.apache.hadoop.util.Time; import org.apache.hadoop.hdfs.web.URLConnectionFactory; -import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; +import org.apache.hadoop.util.Preconditions; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPOfferService.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPOfferService.java index 0e9a1f755855e..3c63160f28362 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPOfferService.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPOfferService.java @@ -18,7 +18,7 @@ package org.apache.hadoop.hdfs.server.datanode; import org.apache.hadoop.classification.VisibleForTesting; -import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; +import org.apache.hadoop.util.Preconditions; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.fs.StorageType; @@ -378,7 +378,7 @@ void verifyAndSetNamespaceInfo(BPServiceActor actor, NamespaceInfo nsInfo) if(nsInfo.getState() == HAServiceState.ACTIVE && bpServiceToActive == null) { - LOG.info("Acknowledging ACTIVE Namenode during handshake" + actor); + LOG.info("Acknowledging ACTIVE Namenode during handshake {}", actor); bpServiceToActive = actor; } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockChecksumHelper.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockChecksumHelper.java index 265267da887d2..eb2bfeb78698f 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockChecksumHelper.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockChecksumHelper.java @@ -17,7 +17,7 @@ */ package org.apache.hadoop.hdfs.server.datanode; -import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; +import org.apache.hadoop.util.Preconditions; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.hdfs.DFSUtilClient; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockPoolManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockPoolManager.java index e34c7ec8bae5d..1139500c22a19 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockPoolManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockPoolManager.java @@ -33,7 +33,7 @@ import org.apache.hadoop.classification.VisibleForTesting; import org.apache.hadoop.thirdparty.com.google.common.base.Joiner; -import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; +import org.apache.hadoop.util.Preconditions; import org.apache.hadoop.thirdparty.com.google.common.collect.Maps; import org.slf4j.Logger; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockPoolSliceStorage.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockPoolSliceStorage.java index 8dad8009297cd..a82ed0045a6a3 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockPoolSliceStorage.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockPoolSliceStorage.java @@ -48,7 +48,7 @@ import org.apache.hadoop.util.Lists; import org.apache.hadoop.classification.VisibleForTesting; -import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; +import org.apache.hadoop.util.Preconditions; /** * Manages storage for the set of BlockPoolSlices which share a particular diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockRecoveryWorker.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockRecoveryWorker.java index d575b592bd443..e4861f9774870 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockRecoveryWorker.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockRecoveryWorker.java @@ -19,7 +19,7 @@ import org.apache.hadoop.classification.VisibleForTesting; import org.apache.hadoop.thirdparty.com.google.common.base.Joiner; -import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; +import org.apache.hadoop.util.Preconditions; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hdfs.protocol.DatanodeID; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockScanner.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockScanner.java index 53da12f774293..69e3db11bdf73 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockScanner.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockScanner.java @@ -35,7 +35,7 @@ import org.apache.hadoop.classification.VisibleForTesting; import org.apache.hadoop.hdfs.protocol.ExtendedBlock; import org.apache.hadoop.hdfs.server.datanode.VolumeScanner.ScanResultHandler; -import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; +import org.apache.hadoop.util.Preconditions; import org.apache.hadoop.thirdparty.com.google.common.util.concurrent.Uninterruptibles; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.conf.Configuration; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockSender.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockSender.java index d3cdee11381ca..172a245b3525a 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockSender.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockSender.java @@ -57,7 +57,7 @@ import static org.apache.hadoop.io.nativeio.NativeIO.POSIX.POSIX_FADV_SEQUENTIAL; import org.apache.hadoop.classification.VisibleForTesting; -import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; +import org.apache.hadoop.util.Preconditions; import org.slf4j.Logger; /** diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java index 4625c67873d66..0bb59de95ae5e 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java @@ -215,7 +215,7 @@ import org.apache.hadoop.classification.VisibleForTesting; import org.apache.hadoop.thirdparty.com.google.common.base.Joiner; -import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; +import org.apache.hadoop.util.Preconditions; import org.apache.hadoop.thirdparty.com.google.common.cache.CacheBuilder; import org.apache.hadoop.thirdparty.com.google.common.cache.CacheLoader; import org.apache.hadoop.thirdparty.com.google.common.cache.LoadingCache; @@ -1383,7 +1383,7 @@ boolean areCacheReportsDisabledForTests() { /** * This method starts the data node with the specified conf. * - * If conf's CONFIG_PROPERTY_SIMULATED property is set + * If conf's DFS_DATANODE_FSDATASET_FACTORY_KEY property is set * then a simulated storage based data node is created. * * @param dataDirectories - only for a non-simulated storage data node diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataXceiver.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataXceiver.java index 1554ace5e801b..8150b7974a159 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataXceiver.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataXceiver.java @@ -18,7 +18,7 @@ package org.apache.hadoop.hdfs.server.datanode; import org.apache.hadoop.classification.VisibleForTesting; -import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; +import org.apache.hadoop.util.Preconditions; import org.apache.hadoop.thirdparty.protobuf.ByteString; import javax.crypto.SecretKey; import org.apache.commons.logging.Log; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataXceiverServer.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataXceiverServer.java index 3eabb0102ae03..77fa70e3cc54f 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataXceiverServer.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataXceiverServer.java @@ -36,7 +36,7 @@ import org.apache.hadoop.util.Daemon; import org.apache.hadoop.classification.VisibleForTesting; -import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; +import org.apache.hadoop.util.Preconditions; import org.slf4j.Logger; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DiskBalancer.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DiskBalancer.java index d46448324ecc2..b516d1325d4e2 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DiskBalancer.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DiskBalancer.java @@ -19,7 +19,7 @@ package org.apache.hadoop.hdfs.server.datanode; import org.apache.hadoop.classification.VisibleForTesting; -import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; +import org.apache.hadoop.util.Preconditions; import org.apache.commons.codec.digest.DigestUtils; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.conf.Configuration; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/ShortCircuitRegistry.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/ShortCircuitRegistry.java index 960811a2e0769..0728ac4b4e55c 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/ShortCircuitRegistry.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/ShortCircuitRegistry.java @@ -47,7 +47,7 @@ import org.apache.hadoop.hdfs.shortcircuit.DfsClientShmManager; import org.apache.hadoop.thirdparty.com.google.common.base.Joiner; -import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; +import org.apache.hadoop.util.Preconditions; import org.apache.hadoop.thirdparty.com.google.common.collect.HashMultimap; /** diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/VolumeScanner.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/VolumeScanner.java index 5212bc6cbdf92..5ab5b49e1a8aa 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/VolumeScanner.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/VolumeScanner.java @@ -31,7 +31,7 @@ import java.util.concurrent.TimeUnit; import org.apache.hadoop.classification.VisibleForTesting; -import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; +import org.apache.hadoop.util.Preconditions; import org.apache.hadoop.thirdparty.com.google.common.cache.Cache; import org.apache.hadoop.thirdparty.com.google.common.cache.CacheBuilder; import org.apache.hadoop.hdfs.protocol.Block; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/checker/AbstractFuture.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/checker/AbstractFuture.java index 6ae4f01299ceb..4855c5de85fde 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/checker/AbstractFuture.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/checker/AbstractFuture.java @@ -22,13 +22,13 @@ import org.apache.hadoop.thirdparty.com.google.common.annotations.Beta; import org.apache.hadoop.thirdparty.com.google.common.annotations.GwtCompatible; -import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; -import static org.apache.hadoop.thirdparty.com.google.common.base.Preconditions.checkNotNull; import org.apache.hadoop.thirdparty.com.google.common.util.concurrent.Futures; import org.apache.hadoop.thirdparty.com.google.common.util.concurrent.ListeningExecutorService; import org.apache.hadoop.thirdparty.com.google.common.util.concurrent.ListenableFuture; import org.apache.hadoop.thirdparty.com.google.common.util.concurrent.SettableFuture; import org.apache.hadoop.thirdparty.com.google.common.util.concurrent.Uninterruptibles; +import org.apache.hadoop.util.Preconditions; + import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; import static java.util.concurrent.atomic.AtomicReferenceFieldUpdater .newUpdater; @@ -290,7 +290,7 @@ public synchronized Throwable fillInStackTrace() { final Throwable exception; Failure(Throwable exception) { - this.exception = checkNotNull(exception); + this.exception = Preconditions.checkNotNull(exception); } } @@ -679,8 +679,8 @@ protected final boolean wasInterrupted() { */ @Override public void addListener(Runnable listener, Executor executor) { - checkNotNull(listener, "Runnable was null."); - checkNotNull(executor, "Executor was null."); + Preconditions.checkNotNull(listener, "Runnable was null."); + Preconditions.checkNotNull(executor, "Executor was null."); Listener oldHead = listeners; if (oldHead != Listener.TOMBSTONE) { Listener newNode = new Listener(listener, executor); @@ -737,7 +737,7 @@ protected boolean set(@Nullable V value) { * @return true if the attempt was accepted, completing the {@code Future} */ protected boolean setException(Throwable throwable) { - Object valueToSet = new Failure(checkNotNull(throwable)); + Object valueToSet = new Failure(Preconditions.checkNotNull(throwable)); if (ATOMIC_HELPER.casValue(this, null, valueToSet)) { complete(this); return true; @@ -771,7 +771,7 @@ protected boolean setException(Throwable throwable) { */ @Beta protected boolean setFuture(ListenableFuture future) { - checkNotNull(future); + Preconditions.checkNotNull(future); Object localValue = value; if (localValue == null) { if (future.isDone()) { @@ -1097,7 +1097,7 @@ public sun.misc.Unsafe run() throws Exception { } public static void throwIfUnchecked(Throwable throwable) { - checkNotNull(throwable); + Preconditions.checkNotNull(throwable); if (throwable instanceof RuntimeException) { throw (RuntimeException) throwable; } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/checker/DatasetVolumeChecker.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/checker/DatasetVolumeChecker.java index a58c536117d46..f8f6450ea5221 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/checker/DatasetVolumeChecker.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/checker/DatasetVolumeChecker.java @@ -19,7 +19,7 @@ package org.apache.hadoop.hdfs.server.datanode.checker; import org.apache.hadoop.classification.VisibleForTesting; -import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; +import org.apache.hadoop.util.Preconditions; import org.apache.hadoop.thirdparty.com.google.common.util.concurrent.FutureCallback; import org.apache.hadoop.thirdparty.com.google.common.util.concurrent.Futures; import org.apache.hadoop.thirdparty.com.google.common.util.concurrent.ListenableFuture; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/checker/TimeoutFuture.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/checker/TimeoutFuture.java index d014e499f912e..6bb2c7a84163a 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/checker/TimeoutFuture.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/checker/TimeoutFuture.java @@ -20,7 +20,7 @@ */ package org.apache.hadoop.hdfs.server.datanode.checker; -import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; +import org.apache.hadoop.util.Preconditions; import org.apache.hadoop.thirdparty.com.google.common.util.concurrent.ListenableFuture; import org.apache.hadoop.hdfs.server.datanode.checker.AbstractFuture; import org.slf4j.Logger; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/ErasureCodingWorker.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/ErasureCodingWorker.java index 9fb5d9e708ca2..0f145d6cdd7b2 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/ErasureCodingWorker.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/ErasureCodingWorker.java @@ -17,7 +17,7 @@ */ package org.apache.hadoop.hdfs.server.datanode.erasurecode; -import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; +import org.apache.hadoop.util.Preconditions; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hdfs.DFSConfigKeys; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedReader.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedReader.java index f8998849baf84..940cb71172805 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedReader.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedReader.java @@ -18,7 +18,7 @@ package org.apache.hadoop.hdfs.server.datanode.erasurecode; import java.util.concurrent.TimeUnit; -import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; +import org.apache.hadoop.util.Preconditions; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hdfs.DFSConfigKeys; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedWriter.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedWriter.java index 683b63fa149af..590ecb46eed52 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedWriter.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedWriter.java @@ -17,7 +17,7 @@ */ package org.apache.hadoop.hdfs.server.datanode.erasurecode; -import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; +import org.apache.hadoop.util.Preconditions; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.StorageType; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetCache.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetCache.java index 417e26647466c..5ca755954fced 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetCache.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetCache.java @@ -24,7 +24,7 @@ import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_CACHE_REVOCATION_POLLING_MS_DEFAULT; import org.apache.hadoop.io.IOUtils; -import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; +import org.apache.hadoop.util.Preconditions; import org.apache.hadoop.thirdparty.com.google.common.util.concurrent.ThreadFactoryBuilder; import java.io.FileInputStream; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java index 081d755eb603d..8d37bda166091 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java @@ -125,7 +125,7 @@ import org.apache.hadoop.util.Time; import org.apache.hadoop.util.Timer; -import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; +import org.apache.hadoop.util.Preconditions; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetUtil.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetUtil.java index 39e47db7174a2..b7686c512963a 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetUtil.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetUtil.java @@ -36,7 +36,7 @@ import java.util.Arrays; import com.fasterxml.jackson.databind.util.ByteBufferBackedInputStream; -import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; +import org.apache.hadoop.util.Preconditions; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hdfs.protocol.Block; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsVolumeImpl.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsVolumeImpl.java index 459137ea3c293..8f15d8a70932e 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsVolumeImpl.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsVolumeImpl.java @@ -89,7 +89,7 @@ import com.fasterxml.jackson.databind.ObjectWriter; import org.apache.hadoop.classification.VisibleForTesting; import org.apache.hadoop.thirdparty.com.google.common.base.Joiner; -import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; +import org.apache.hadoop.util.Preconditions; import org.apache.hadoop.thirdparty.com.google.common.util.concurrent.ThreadFactoryBuilder; /** diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/MappableBlockLoader.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/MappableBlockLoader.java index a35c6f2a31bb4..5d1b21f4a0b93 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/MappableBlockLoader.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/MappableBlockLoader.java @@ -18,7 +18,7 @@ package org.apache.hadoop.hdfs.server.datanode.fsdataset.impl; -import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; +import org.apache.hadoop.util.Preconditions; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.hdfs.ExtendedBlockId; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/NativePmemMappableBlockLoader.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/NativePmemMappableBlockLoader.java index 6a5810349ea48..526eb55593d4d 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/NativePmemMappableBlockLoader.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/NativePmemMappableBlockLoader.java @@ -18,7 +18,7 @@ package org.apache.hadoop.hdfs.server.datanode.fsdataset.impl; -import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; +import org.apache.hadoop.util.Preconditions; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.hdfs.ExtendedBlockId; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/RamDiskReplicaTracker.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/RamDiskReplicaTracker.java index f7b12ff179941..1103468d3c8b9 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/RamDiskReplicaTracker.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/RamDiskReplicaTracker.java @@ -18,7 +18,7 @@ package org.apache.hadoop.hdfs.server.datanode.fsdataset.impl; -import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; +import org.apache.hadoop.util.Preconditions; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/webhdfs/WebHdfsHandler.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/webhdfs/WebHdfsHandler.java index 2f901da165ff4..f2ced88aaa30e 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/webhdfs/WebHdfsHandler.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/webhdfs/WebHdfsHandler.java @@ -17,7 +17,7 @@ */ package org.apache.hadoop.hdfs.server.datanode.web.webhdfs; -import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; +import org.apache.hadoop.util.Preconditions; import io.netty.buffer.Unpooled; import io.netty.channel.ChannelFutureListener; import io.netty.channel.ChannelHandlerContext; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/command/CancelCommand.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/command/CancelCommand.java index 6889f5f92e389..2707b9412bc3b 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/command/CancelCommand.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/command/CancelCommand.java @@ -19,7 +19,7 @@ package org.apache.hadoop.hdfs.server.diskbalancer.command; -import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; +import org.apache.hadoop.util.Preconditions; import org.apache.commons.cli.CommandLine; import org.apache.commons.cli.HelpFormatter; import org.apache.commons.codec.digest.DigestUtils; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/command/Command.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/command/Command.java index b78a86bf499e3..c90b77e98d2e8 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/command/Command.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/command/Command.java @@ -52,7 +52,7 @@ import org.slf4j.LoggerFactory; import org.apache.hadoop.classification.VisibleForTesting; -import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; +import org.apache.hadoop.util.Preconditions; import java.io.Closeable; import java.io.IOException; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/command/ExecuteCommand.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/command/ExecuteCommand.java index 78b16e999f84e..ff5389b43d002 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/command/ExecuteCommand.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/command/ExecuteCommand.java @@ -19,7 +19,7 @@ package org.apache.hadoop.hdfs.server.diskbalancer.command; -import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; +import org.apache.hadoop.util.Preconditions; import org.apache.commons.cli.CommandLine; import org.apache.commons.cli.HelpFormatter; import org.apache.commons.codec.digest.DigestUtils; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/command/HelpCommand.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/command/HelpCommand.java index e36628edf0eb2..ec180538a9843 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/command/HelpCommand.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/command/HelpCommand.java @@ -19,7 +19,7 @@ package org.apache.hadoop.hdfs.server.diskbalancer.command; -import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; +import org.apache.hadoop.util.Preconditions; import org.apache.commons.cli.CommandLine; import org.apache.commons.cli.HelpFormatter; import org.apache.hadoop.conf.Configuration; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/command/PlanCommand.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/command/PlanCommand.java index e9f9f33e71535..3732de86aa0fc 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/command/PlanCommand.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/command/PlanCommand.java @@ -17,7 +17,7 @@ */ package org.apache.hadoop.hdfs.server.diskbalancer.command; -import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; +import org.apache.hadoop.util.Preconditions; import org.apache.hadoop.thirdparty.com.google.common.base.Throwables; import org.apache.commons.cli.CommandLine; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/command/QueryCommand.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/command/QueryCommand.java index 520e80f3974c7..6a89e248f5967 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/command/QueryCommand.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/command/QueryCommand.java @@ -19,7 +19,7 @@ package org.apache.hadoop.hdfs.server.diskbalancer.command; -import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; +import org.apache.hadoop.util.Preconditions; import org.apache.commons.cli.CommandLine; import org.apache.commons.cli.HelpFormatter; import org.apache.hadoop.conf.Configuration; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/command/ReportCommand.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/command/ReportCommand.java index f93a909418332..e26f0a113b33f 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/command/ReportCommand.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/command/ReportCommand.java @@ -34,7 +34,7 @@ import org.apache.hadoop.hdfs.tools.DiskBalancerCLI; import org.apache.hadoop.util.Lists; -import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; +import org.apache.hadoop.util.Preconditions; /** diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/connectors/DBNameNodeConnector.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/connectors/DBNameNodeConnector.java index b7bb3f02dce9d..f9bcd5e018065 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/connectors/DBNameNodeConnector.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/connectors/DBNameNodeConnector.java @@ -17,7 +17,7 @@ package org.apache.hadoop.hdfs.server.diskbalancer.connectors; -import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; +import org.apache.hadoop.util.Preconditions; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.hadoop.conf.Configuration; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/connectors/JsonNodeConnector.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/connectors/JsonNodeConnector.java index 268c055a354ac..1cc82253f9885 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/connectors/JsonNodeConnector.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/connectors/JsonNodeConnector.java @@ -19,7 +19,7 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.ObjectReader; -import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; +import org.apache.hadoop.util.Preconditions; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/datamodel/DiskBalancerCluster.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/datamodel/DiskBalancerCluster.java index 0e405ff7bd3c5..c801f36ea5205 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/datamodel/DiskBalancerCluster.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/datamodel/DiskBalancerCluster.java @@ -21,7 +21,7 @@ import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.ObjectReader; -import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; +import org.apache.hadoop.util.Preconditions; import org.apache.commons.io.FileUtils; import org.slf4j.Logger; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/datamodel/DiskBalancerDataNode.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/datamodel/DiskBalancerDataNode.java index fce858aaca01b..fe9edf6678ac5 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/datamodel/DiskBalancerDataNode.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/datamodel/DiskBalancerDataNode.java @@ -17,7 +17,7 @@ package org.apache.hadoop.hdfs.server.diskbalancer.datamodel; -import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; +import org.apache.hadoop.util.Preconditions; import java.util.HashMap; import java.util.Map; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/datamodel/DiskBalancerVolumeSet.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/datamodel/DiskBalancerVolumeSet.java index bcce012ff84b9..f59f4fc9e3fe3 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/datamodel/DiskBalancerVolumeSet.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/datamodel/DiskBalancerVolumeSet.java @@ -21,7 +21,7 @@ import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import com.fasterxml.jackson.annotation.JsonProperty; -import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; +import org.apache.hadoop.util.Preconditions; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/planner/GreedyPlanner.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/planner/GreedyPlanner.java index 0ed56afb39a5a..59b908671791a 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/planner/GreedyPlanner.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/planner/GreedyPlanner.java @@ -17,7 +17,7 @@ package org.apache.hadoop.hdfs.server.diskbalancer.planner; -import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; +import org.apache.hadoop.util.Preconditions; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.hadoop.hdfs.server.diskbalancer.datamodel diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/planner/NodePlan.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/planner/NodePlan.java index 72df5abe6bcaa..39a7c57bca2cd 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/planner/NodePlan.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/planner/NodePlan.java @@ -21,7 +21,7 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.ObjectReader; import com.fasterxml.jackson.databind.ObjectWriter; -import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; +import org.apache.hadoop.util.Preconditions; import java.io.IOException; import java.util.LinkedList; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/AclFeature.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/AclFeature.java index 2dfe50742dac2..9c67d3da43ab9 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/AclFeature.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/AclFeature.java @@ -24,7 +24,6 @@ import org.apache.hadoop.fs.permission.AclEntry; import org.apache.hadoop.hdfs.util.ReferenceCountMap.ReferenceCounter; -import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; import org.apache.hadoop.thirdparty.com.google.common.collect.ImmutableList; /** @@ -56,8 +55,9 @@ int getEntriesSize() { * @throws IndexOutOfBoundsException if pos out of bound */ int getEntryAt(int pos) { - Preconditions.checkPositionIndex(pos, entries.length, - "Invalid position for AclEntry"); + if (pos < 0 || pos > entries.length) { + throw new IndexOutOfBoundsException("Invalid position for AclEntry"); + } return entries[pos]; } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/AuditLogger.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/AuditLogger.java index 614eb63d055a7..49dcb8c0a3757 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/AuditLogger.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/AuditLogger.java @@ -17,14 +17,13 @@ */ package org.apache.hadoop.hdfs.server.namenode; -import java.net.InetAddress; -import java.security.Principal; - import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileStatus; +import java.net.InetAddress; + /** * Interface defining an audit logger. */ diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/BackupImage.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/BackupImage.java index 48a3416922baf..504df6068ef3c 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/BackupImage.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/BackupImage.java @@ -31,7 +31,7 @@ import org.apache.hadoop.util.Lists; import org.apache.hadoop.util.StringUtils; -import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; +import org.apache.hadoop.util.Preconditions; /** * Extension of FSImage for the backup node. diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/CachePool.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/CachePool.java index dda4789b74e49..8016a1ec11a30 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/CachePool.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/CachePool.java @@ -32,7 +32,7 @@ import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.util.IntrusiveCollection; -import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; +import org.apache.hadoop.util.Preconditions; /** * A CachePool describes a set of cache resources being managed by the NameNode. diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ContentSummaryComputationContext.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ContentSummaryComputationContext.java index e304baf652843..61a34828f14dd 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ContentSummaryComputationContext.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ContentSummaryComputationContext.java @@ -17,7 +17,7 @@ */ package org.apache.hadoop.hdfs.server.namenode; -import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; +import org.apache.hadoop.util.Preconditions; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.fs.permission.FsAction; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EditLogBackupInputStream.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EditLogBackupInputStream.java index d17fd06bc882c..a77075fa4f2e8 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EditLogBackupInputStream.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EditLogBackupInputStream.java @@ -21,7 +21,7 @@ import java.io.ByteArrayInputStream; import java.io.IOException; -import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; +import org.apache.hadoop.util.Preconditions; import org.apache.hadoop.hdfs.server.common.HdfsServerConstants; /** diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EditLogFileInputStream.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EditLogFileInputStream.java index 459cb5be5db67..ba4f32fd2154d 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EditLogFileInputStream.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EditLogFileInputStream.java @@ -46,7 +46,7 @@ import org.apache.hadoop.security.authentication.client.AuthenticationException; import org.apache.hadoop.classification.VisibleForTesting; -import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; +import org.apache.hadoop.util.Preconditions; import org.apache.hadoop.thirdparty.com.google.common.base.Throwables; /** diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EditsDoubleBuffer.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EditsDoubleBuffer.java index ccc233efcbafe..9577a528923ac 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EditsDoubleBuffer.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EditsDoubleBuffer.java @@ -32,7 +32,7 @@ import org.apache.hadoop.io.DataOutputBuffer; import org.apache.hadoop.io.IOUtils; -import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; +import org.apache.hadoop.util.Preconditions; /** * A double-buffer for edits. New edits are written into the first buffer diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EncryptionZoneManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EncryptionZoneManager.java index de6e49d747aa5..62a81f4064eec 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EncryptionZoneManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EncryptionZoneManager.java @@ -53,7 +53,7 @@ import org.apache.hadoop.util.Lists; import org.apache.hadoop.classification.VisibleForTesting; -import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; +import org.apache.hadoop.util.Preconditions; import org.apache.hadoop.thirdparty.com.google.common.util.concurrent.ThreadFactoryBuilder; import org.apache.hadoop.thirdparty.protobuf.InvalidProtocolBufferException; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ErasureCodingPolicyManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ErasureCodingPolicyManager.java index 7dddfeda21c42..8338c4fb091a8 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ErasureCodingPolicyManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ErasureCodingPolicyManager.java @@ -18,7 +18,7 @@ package org.apache.hadoop.hdfs.server.namenode; import org.apache.hadoop.classification.VisibleForTesting; -import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; +import org.apache.hadoop.util.Preconditions; import org.apache.hadoop.HadoopIllegalArgumentException; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.conf.Configuration; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirAclOp.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirAclOp.java index 6ced588e96882..b17c459cc993a 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirAclOp.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirAclOp.java @@ -17,8 +17,6 @@ */ package org.apache.hadoop.hdfs.server.namenode; -import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; - import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.permission.AclEntry; import org.apache.hadoop.fs.permission.AclEntryScope; @@ -228,10 +226,12 @@ private static void unprotectedRemoveAcl(FSDirectory fsd, INodesInPath iip) int groupEntryIndex = Collections.binarySearch( featureEntries, groupEntryKey, AclTransformation.ACL_ENTRY_COMPARATOR); - Preconditions.checkPositionIndex(groupEntryIndex, featureEntries.size(), - "Invalid group entry index after binary-searching inode: " + - inode.getFullPathName() + "(" + inode.getId() + ") " - + "with featureEntries:" + featureEntries); + if (groupEntryIndex < 0 || groupEntryIndex > featureEntries.size()) { + throw new IndexOutOfBoundsException( + "Invalid group entry index after binary-searching inode: " + + inode.getFullPathName() + "(" + inode.getId() + ") " + + "with featureEntries:" + featureEntries); + } FsAction groupPerm = featureEntries.get(groupEntryIndex).getPermission(); FsPermission newPerm = new FsPermission(perm.getUserAction(), groupPerm, perm.getOtherAction(), perm.getStickyBit()); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirAppendOp.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirAppendOp.java index 7e90d4bafa8bb..ba00e8ae936a0 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirAppendOp.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirAppendOp.java @@ -40,7 +40,7 @@ import org.apache.hadoop.hdfs.server.namenode.NameNodeLayoutVersion.Feature; import org.apache.hadoop.ipc.RetriableException; -import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; +import org.apache.hadoop.util.Preconditions; /** * Helper class to perform append operation. diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirConcatOp.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirConcatOp.java index 04ae358c67afb..ea5ac38aa8659 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirConcatOp.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirConcatOp.java @@ -17,7 +17,7 @@ */ package org.apache.hadoop.hdfs.server.namenode; -import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; +import org.apache.hadoop.util.Preconditions; import org.apache.hadoop.HadoopIllegalArgumentException; import org.apache.hadoop.fs.permission.FsAction; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirEncryptionZoneOp.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirEncryptionZoneOp.java index 0969679be175c..2971af1829809 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirEncryptionZoneOp.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirEncryptionZoneOp.java @@ -54,7 +54,7 @@ import org.apache.hadoop.util.Lists; import org.apache.hadoop.util.Time; -import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; +import org.apache.hadoop.util.Preconditions; import org.apache.hadoop.thirdparty.protobuf.InvalidProtocolBufferException; import static org.apache.hadoop.hdfs.server.common.HdfsServerConstants.CRYPTO_XATTR_ENCRYPTION_ZONE; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirErasureCodingOp.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirErasureCodingOp.java index 4815dfce986ac..6628b56a132e0 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirErasureCodingOp.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirErasureCodingOp.java @@ -34,7 +34,7 @@ import org.apache.hadoop.security.AccessControlException; import org.apache.hadoop.util.Lists; -import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; +import org.apache.hadoop.util.Preconditions; import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirMkdirOp.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirMkdirOp.java index da324fb46738a..862880d95b2d3 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirMkdirOp.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirMkdirOp.java @@ -18,7 +18,7 @@ package org.apache.hadoop.hdfs.server.namenode; import org.apache.hadoop.fs.permission.FsCreateModes; -import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; +import org.apache.hadoop.util.Preconditions; import org.apache.hadoop.fs.FileAlreadyExistsException; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.ParentNotDirectoryException; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirRenameOp.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirRenameOp.java index 8f6fcdc1d1f02..cc0d29e953572 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirRenameOp.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirRenameOp.java @@ -17,7 +17,7 @@ */ package org.apache.hadoop.hdfs.server.namenode; -import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; +import org.apache.hadoop.util.Preconditions; import org.apache.hadoop.fs.FileAlreadyExistsException; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.InvalidPathException; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirStatAndListingOp.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirStatAndListingOp.java index 8aff179358896..236e308f4101e 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirStatAndListingOp.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirStatAndListingOp.java @@ -18,7 +18,7 @@ package org.apache.hadoop.hdfs.server.namenode; -import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; +import org.apache.hadoop.util.Preconditions; import org.apache.hadoop.fs.ContentSummary; import org.apache.hadoop.fs.DirectoryListingStartAfterNotFoundException; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirWriteFileOp.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirWriteFileOp.java index 0d9c6aeeb9c45..7b7f4a0f9c070 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirWriteFileOp.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirWriteFileOp.java @@ -17,7 +17,7 @@ */ package org.apache.hadoop.hdfs.server.namenode; -import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; +import org.apache.hadoop.util.Preconditions; import org.apache.hadoop.HadoopIllegalArgumentException; import org.apache.hadoop.fs.XAttrSetFlag; import org.apache.hadoop.hdfs.AddBlockFlag; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirXAttrOp.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirXAttrOp.java index 4151e016f4261..6e6ade291ce27 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirXAttrOp.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirXAttrOp.java @@ -35,7 +35,7 @@ import org.apache.hadoop.util.Lists; import org.apache.hadoop.classification.VisibleForTesting; -import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; +import org.apache.hadoop.util.Preconditions; import java.io.FileNotFoundException; import java.io.IOException; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java index d89b087c46061..4d86e52f74d52 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java @@ -22,7 +22,7 @@ import org.apache.hadoop.classification.VisibleForTesting; import org.apache.hadoop.thirdparty.com.google.common.base.Joiner; -import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; +import org.apache.hadoop.util.Preconditions; import org.apache.hadoop.thirdparty.protobuf.InvalidProtocolBufferException; import org.apache.hadoop.HadoopIllegalArgumentException; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java index aa6a0a001bad1..53c663307d738 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java @@ -112,7 +112,7 @@ import org.apache.hadoop.util.Lists; import org.apache.hadoop.classification.VisibleForTesting; -import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; +import org.apache.hadoop.util.Preconditions; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogAsync.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogAsync.java index ad03a85636d7e..115e9485fa0a9 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogAsync.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogAsync.java @@ -37,7 +37,7 @@ import org.apache.hadoop.ipc.Server; import org.apache.hadoop.util.ExitUtil; import org.apache.hadoop.classification.VisibleForTesting; -import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; +import org.apache.hadoop.util.Preconditions; class FSEditLogAsync extends FSEditLog implements Runnable { static final Logger LOG = LoggerFactory.getLogger(FSEditLog.class); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java index aea928d4cf63a..a065fe6c0cfa8 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java @@ -118,7 +118,7 @@ import org.apache.hadoop.classification.VisibleForTesting; import org.apache.hadoop.thirdparty.com.google.common.base.Joiner; -import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; +import org.apache.hadoop.util.Preconditions; import static org.apache.hadoop.log.LogThrottlingHelper.LogAction; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogOp.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogOp.java index 3ab0a9facd3df..8bddc6741a1aa 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogOp.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogOp.java @@ -140,7 +140,7 @@ import org.apache.hadoop.classification.VisibleForTesting; import org.apache.hadoop.thirdparty.com.google.common.base.Joiner; -import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; +import org.apache.hadoop.util.Preconditions; import org.apache.hadoop.thirdparty.com.google.common.collect.ImmutableMap; /** diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java index f7749ce7e231b..b350ef1f98c3e 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java @@ -77,7 +77,7 @@ import org.apache.hadoop.util.Time; import org.apache.hadoop.thirdparty.com.google.common.base.Joiner; -import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; +import org.apache.hadoop.util.Preconditions; /** * FSImage handles checkpointing and logging of the namespace edits. diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormat.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormat.java index 2ce2c645e1511..7e679296e25c5 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormat.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormat.java @@ -76,7 +76,7 @@ import org.apache.hadoop.util.StringUtils; import org.apache.hadoop.classification.VisibleForTesting; -import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; +import org.apache.hadoop.util.Preconditions; /** * Contains inner classes for reading or writing the on-disk format for diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormatPBINode.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormatPBINode.java index 0a69c99cab810..3f0c9faa97c9a 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormatPBINode.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormatPBINode.java @@ -72,7 +72,7 @@ import org.apache.hadoop.hdfs.util.EnumCounters; import org.apache.hadoop.hdfs.util.ReadOnlyList; -import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; +import org.apache.hadoop.util.Preconditions; import org.apache.hadoop.thirdparty.com.google.common.collect.ImmutableList; import org.apache.hadoop.thirdparty.protobuf.ByteString; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageSerialization.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageSerialization.java index 3d75cebf729d3..404f2c73ad3a2 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageSerialization.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageSerialization.java @@ -53,7 +53,7 @@ import org.xml.sax.ContentHandler; import org.xml.sax.SAXException; -import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; +import org.apache.hadoop.util.Preconditions; /** * Static utility functions for serializing various pieces of data in the correct diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java index 7ad90ac1a93ca..27dcf01ee92c5 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java @@ -24,12 +24,16 @@ import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_CALLER_CONTEXT_ENABLED_KEY; import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_CALLER_CONTEXT_MAX_SIZE_DEFAULT; import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_CALLER_CONTEXT_MAX_SIZE_KEY; +import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_CALLER_CONTEXT_SEPARATOR_DEFAULT; +import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_CALLER_CONTEXT_SEPARATOR_KEY; import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_CALLER_CONTEXT_SIGNATURE_MAX_SIZE_DEFAULT; import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_CALLER_CONTEXT_SIGNATURE_MAX_SIZE_KEY; import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.IO_FILE_BUFFER_SIZE_DEFAULT; import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.IO_FILE_BUFFER_SIZE_KEY; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_BLOCK_SIZE_DEFAULT; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_BLOCK_SIZE_KEY; +import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_AUDIT_LOG_WITH_REMOTE_PORT_DEFAULT; +import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_AUDIT_LOG_WITH_REMOTE_PORT_KEY; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_STORAGE_POLICY_ENABLED_DEFAULT; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_STORAGE_POLICY_PERMISSIONS_SUPERUSER_ONLY_DEFAULT; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_STORAGE_POLICY_PERMISSIONS_SUPERUSER_ONLY_KEY; @@ -343,7 +347,7 @@ import org.apache.hadoop.classification.VisibleForTesting; import org.apache.hadoop.thirdparty.com.google.common.base.Charsets; -import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; +import org.apache.hadoop.util.Preconditions; import org.apache.hadoop.thirdparty.com.google.common.collect.ImmutableMap; import org.apache.hadoop.thirdparty.com.google.common.util.concurrent.ThreadFactoryBuilder; @@ -397,6 +401,9 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean, @Metric final MutableRatesWithAggregation detailedLockHoldTimeMetrics = registry.newRatesWithAggregation("detailedLockHoldTimeMetrics"); + private static final String CLIENT_PORT_STR = "clientPort"; + private final String contextFieldSeparator; + boolean isAuditEnabled() { return (!isDefaultAuditLogger || auditLog.isInfoEnabled()) && !auditLoggers.isEmpty(); @@ -411,7 +418,7 @@ private void logAuditEvent(boolean succeeded, String cmd, String src, String dst, FileStatus stat) throws IOException { if (isAuditEnabled() && isExternalInvocation()) { logAuditEvent(succeeded, Server.getRemoteUser(), Server.getRemoteIp(), - cmd, src, dst, stat); + cmd, src, dst, stat); } } @@ -442,6 +449,9 @@ private void logAuditEvent(boolean succeeded, for (AuditLogger logger : auditLoggers) { if (logger instanceof HdfsAuditLogger) { HdfsAuditLogger hdfsLogger = (HdfsAuditLogger) logger; + if (auditLogWithRemotePort) { + appendClientPortToCallerContextIfAbsent(); + } hdfsLogger.logAuditEvent(succeeded, ugiStr, addr, cmd, src, dst, status, CallerContext.getCurrent(), ugi, dtSecretManager); } else { @@ -450,6 +460,25 @@ private void logAuditEvent(boolean succeeded, } } + private void appendClientPortToCallerContextIfAbsent() { + final CallerContext ctx = CallerContext.getCurrent(); + if (isClientPortInfoAbsent(CLIENT_PORT_STR + ":" + Server.getRemotePort(), + ctx)) { + String origContext = ctx == null ? null : ctx.getContext(); + byte[] origSignature = ctx == null ? null : ctx.getSignature(); + CallerContext.setCurrent( + new CallerContext.Builder(origContext, contextFieldSeparator) + .append(CLIENT_PORT_STR, String.valueOf(Server.getRemotePort())) + .setSignature(origSignature) + .build()); + } + } + + private boolean isClientPortInfoAbsent(String clientPortInfo, CallerContext ctx){ + return ctx == null || ctx.getContext() == null + || !ctx.getContext().contains(clientPortInfo); + } + /** * Logger for audit events, noting successful FSNamesystem operations. Emits * to FSNamesystem.audit at INFO. Each event causes a set of tab-separated @@ -501,6 +530,7 @@ private void logAuditEvent(boolean succeeded, // underlying logger is disabled, and avoid some unnecessary work. private final boolean isDefaultAuditLogger; private final List auditLoggers; + private final boolean auditLogWithRemotePort; /** The namespace tree. */ FSDirectory dir; @@ -833,6 +863,12 @@ static FSNamesystem loadFromDisk(Configuration conf) throws IOException { LOG.info("Enabling async auditlog"); enableAsyncAuditLog(conf); } + auditLogWithRemotePort = + conf.getBoolean(DFS_NAMENODE_AUDIT_LOG_WITH_REMOTE_PORT_KEY, + DFS_NAMENODE_AUDIT_LOG_WITH_REMOTE_PORT_DEFAULT); + this.contextFieldSeparator = + conf.get(HADOOP_CALLER_CONTEXT_SEPARATOR_KEY, + HADOOP_CALLER_CONTEXT_SEPARATOR_DEFAULT); fsLock = new FSNamesystemLock(conf, detailedLockHoldTimeMetrics); cond = fsLock.newWriteLockCondition(); cpLock = new ReentrantLock(); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSPermissionChecker.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSPermissionChecker.java index 324cd5d441201..c7430e38cd07b 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSPermissionChecker.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSPermissionChecker.java @@ -23,7 +23,7 @@ import java.util.List; import java.util.Stack; -import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; +import org.apache.hadoop.util.Preconditions; import org.apache.hadoop.ipc.CallerContext; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -273,31 +273,41 @@ void checkPermission(INodesInPath inodesInPath, boolean doCheckOwner, AccessControlEnforcer enforcer = getAccessControlEnforcer(); String opType = operationType.get(); - if (this.authorizeWithContext && opType != null) { - INodeAttributeProvider.AuthorizationContext.Builder builder = - new INodeAttributeProvider.AuthorizationContext.Builder(); - builder.fsOwner(fsOwner). - supergroup(supergroup). - callerUgi(callerUgi). - inodeAttrs(inodeAttrs). - inodes(inodes). - pathByNameArr(components). - snapshotId(snapshotId). - path(path). - ancestorIndex(ancestorIndex). - doCheckOwner(doCheckOwner). - ancestorAccess(ancestorAccess). - parentAccess(parentAccess). - access(access). - subAccess(subAccess). - ignoreEmptyDir(ignoreEmptyDir). - operationName(opType). - callerContext(CallerContext.getCurrent()); - enforcer.checkPermissionWithContext(builder.build()); - } else { - enforcer.checkPermission(fsOwner, supergroup, callerUgi, inodeAttrs, - inodes, components, snapshotId, path, ancestorIndex, doCheckOwner, - ancestorAccess, parentAccess, access, subAccess, ignoreEmptyDir); + try { + if (this.authorizeWithContext && opType != null) { + INodeAttributeProvider.AuthorizationContext.Builder builder = + new INodeAttributeProvider.AuthorizationContext.Builder(); + builder.fsOwner(fsOwner). + supergroup(supergroup). + callerUgi(callerUgi). + inodeAttrs(inodeAttrs). + inodes(inodes). + pathByNameArr(components). + snapshotId(snapshotId). + path(path). + ancestorIndex(ancestorIndex). + doCheckOwner(doCheckOwner). + ancestorAccess(ancestorAccess). + parentAccess(parentAccess). + access(access). + subAccess(subAccess). + ignoreEmptyDir(ignoreEmptyDir). + operationName(opType). + callerContext(CallerContext.getCurrent()); + enforcer.checkPermissionWithContext(builder.build()); + } else { + enforcer.checkPermission(fsOwner, supergroup, callerUgi, inodeAttrs, + inodes, components, snapshotId, path, ancestorIndex, doCheckOwner, + ancestorAccess, parentAccess, access, subAccess, ignoreEmptyDir); + } + } catch (AccessControlException ace) { + Class exceptionClass = ace.getClass(); + if (exceptionClass.equals(AccessControlException.class) + || exceptionClass.equals(TraverseAccessControlException.class)) { + throw ace; + } + // Only form a new ACE for subclasses which come from external enforcers + throw new AccessControlException(ace); } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSTreeTraverser.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSTreeTraverser.java index a90dc27a54fa8..c1d26f40c0826 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSTreeTraverser.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSTreeTraverser.java @@ -34,7 +34,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; +import org.apache.hadoop.util.Preconditions; /** * FSTreeTraverser traverse directory recursively and process files diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FileJournalManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FileJournalManager.java index fab8f7e43531a..42a5d2d7385bf 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FileJournalManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FileJournalManager.java @@ -48,7 +48,7 @@ import org.apache.hadoop.classification.VisibleForTesting; import org.apache.hadoop.thirdparty.com.google.common.base.Joiner; -import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; +import org.apache.hadoop.util.Preconditions; import org.apache.hadoop.thirdparty.com.google.common.collect.ComparisonChain; /** diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FsckServlet.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FsckServlet.java index e5c02e81dccfc..059b6531242e8 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FsckServlet.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FsckServlet.java @@ -48,9 +48,9 @@ public void doGet(HttpServletRequest request, HttpServletResponse response @SuppressWarnings("unchecked") final Map pmap = request.getParameterMap(); final PrintWriter out = response.getWriter(); - final InetAddress remoteAddress = + final InetAddress remoteAddress = InetAddress.getByName(request.getRemoteAddr()); - final ServletContext context = getServletContext(); + final ServletContext context = getServletContext(); final Configuration conf = NameNodeHttpServer.getConfFromContext(context); final UserGroupInformation ugi = getUGI(request, conf); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/HdfsAuditLogger.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/HdfsAuditLogger.java index 0a355d0ec3e77..5c0a34ce76582 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/HdfsAuditLogger.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/HdfsAuditLogger.java @@ -17,8 +17,6 @@ */ package org.apache.hadoop.hdfs.server.namenode; -import java.net.InetAddress; - import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.fs.FileStatus; @@ -26,6 +24,8 @@ import org.apache.hadoop.ipc.CallerContext; import org.apache.hadoop.security.UserGroupInformation; +import java.net.InetAddress; + /** * Extension of {@link AuditLogger}. */ diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INode.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INode.java index dacde02a8a9ad..fbb1c44be6c0a 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INode.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INode.java @@ -18,7 +18,7 @@ package org.apache.hadoop.hdfs.server.namenode; import org.apache.hadoop.classification.VisibleForTesting; -import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; +import org.apache.hadoop.util.Preconditions; import org.apache.hadoop.thirdparty.com.google.common.collect.ImmutableMap; import org.apache.hadoop.thirdparty.com.google.common.collect.Maps; import org.apache.hadoop.classification.InterfaceAudience; @@ -994,14 +994,14 @@ public static class ReclaimContext { /** * @param bsps - * block storage policy suite to calculate intended storage type - * usage + * block storage policy suite to calculate intended storage type + * usage * @param collectedBlocks -* blocks collected from the descents for further block -* deletion/update will be added to the given map. + * blocks collected from the descents for further block + * deletion/update will be added to the given map. * @param removedINodes -* INodes collected from the descents for further cleaning up of - * @param removedUCFiles + * INodes collected from the descents for further cleaning up of + * @param removedUCFiles INodes whose leases need to be released */ public ReclaimContext( BlockStoragePolicySuite bsps, BlocksMapUpdateInfo collectedBlocks, diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeDirectory.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeDirectory.java index c0a54e596366d..e375bc1e25ddf 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeDirectory.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeDirectory.java @@ -43,7 +43,7 @@ import org.apache.hadoop.hdfs.util.ReadOnlyList; import org.apache.hadoop.classification.VisibleForTesting; -import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; +import org.apache.hadoop.util.Preconditions; import org.apache.hadoop.security.AccessControlException; import static org.apache.hadoop.hdfs.protocol.HdfsConstants.BLOCK_STORAGE_POLICY_ID_UNSPECIFIED; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeDirectoryAttributes.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeDirectoryAttributes.java index 5e5c4b4b81fb7..05b78ccc6f53d 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeDirectoryAttributes.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeDirectoryAttributes.java @@ -22,7 +22,7 @@ import org.apache.hadoop.fs.StorageType; import org.apache.hadoop.hdfs.util.EnumCounters; -import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; +import org.apache.hadoop.util.Preconditions; /** * The attributes of an inode. diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java index 2b6bc061ab6c3..aa2b95d2ea608 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java @@ -59,7 +59,7 @@ import static org.apache.hadoop.io.erasurecode.ErasureCodeConstants.REPLICATION_POLICY_ID; import org.apache.hadoop.classification.VisibleForTesting; -import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; +import org.apache.hadoop.util.Preconditions; /** I-node for closed file. */ @InterfaceAudience.Private diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeMap.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeMap.java index f35949fdcdbed..01709832e9a3e 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeMap.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeMap.java @@ -26,7 +26,7 @@ import org.apache.hadoop.util.GSet; import org.apache.hadoop.util.LightWeightGSet; -import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; +import org.apache.hadoop.util.Preconditions; /** * Storing all the {@link INode}s and maintaining the mapping between INode ID diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeReference.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeReference.java index ccb4b87a0e8de..6e655f7a13408 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeReference.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeReference.java @@ -30,7 +30,7 @@ import org.apache.hadoop.hdfs.server.namenode.snapshot.DirectoryWithSnapshotFeature; import org.apache.hadoop.hdfs.server.namenode.snapshot.Snapshot; -import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; +import org.apache.hadoop.util.Preconditions; import org.apache.hadoop.hdfs.server.namenode.visitor.NamespaceVisitor; import org.apache.hadoop.security.AccessControlException; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeReferenceValidation.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeReferenceValidation.java index 9241644dcb9c2..b7fff697bd841 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeReferenceValidation.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeReferenceValidation.java @@ -17,7 +17,7 @@ */ package org.apache.hadoop.hdfs.server.namenode; -import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; +import org.apache.hadoop.util.Preconditions; import org.apache.hadoop.hdfs.server.namenode.FsImageValidation.Util; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeWithAdditionalFields.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeWithAdditionalFields.java index a2c7804a83521..febd6f48ef4f7 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeWithAdditionalFields.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeWithAdditionalFields.java @@ -24,7 +24,7 @@ import org.apache.hadoop.hdfs.util.LongBitFormat; import org.apache.hadoop.util.LightWeightGSet.LinkedElement; -import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; +import org.apache.hadoop.util.Preconditions; /** * {@link INode} with additional fields including id, name, permission, diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodesInPath.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodesInPath.java index c2cdd48d4952a..832d84a50f28f 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodesInPath.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodesInPath.java @@ -27,7 +27,7 @@ import org.apache.hadoop.hdfs.server.namenode.snapshot.DirectoryWithSnapshotFeature; import org.apache.hadoop.hdfs.server.namenode.snapshot.Snapshot; -import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; +import org.apache.hadoop.util.Preconditions; import static org.apache.hadoop.hdfs.server.namenode.snapshot.Snapshot.CURRENT_STATE_ID; import static org.apache.hadoop.hdfs.server.namenode.snapshot.Snapshot.ID_INTEGER_COMPARATOR; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ImageServlet.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ImageServlet.java index f85f434e799e5..442c1aba95b1c 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ImageServlet.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ImageServlet.java @@ -65,7 +65,7 @@ import org.apache.hadoop.util.StringUtils; import org.apache.hadoop.classification.VisibleForTesting; -import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; +import org.apache.hadoop.util.Preconditions; /** * This class is used in Namesystem's jetty to retrieve/upload a file diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/JournalSet.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/JournalSet.java index 8457f4186f394..4729cd99f305d 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/JournalSet.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/JournalSet.java @@ -45,7 +45,7 @@ import org.apache.hadoop.util.Sets; import org.apache.hadoop.classification.VisibleForTesting; -import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; +import org.apache.hadoop.util.Preconditions; /** * Manages a collection of Journals. None of the methods are synchronized, it is diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/LeaseManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/LeaseManager.java index 4caacc35c2fda..b21a34a932af9 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/LeaseManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/LeaseManager.java @@ -51,7 +51,7 @@ import org.apache.hadoop.util.Time; import org.apache.hadoop.classification.VisibleForTesting; -import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; +import org.apache.hadoop.util.Preconditions; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NNStorage.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NNStorage.java index ece11a360d003..ca8b6fc752065 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NNStorage.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NNStorage.java @@ -59,7 +59,7 @@ import org.eclipse.jetty.util.ajax.JSON; import org.apache.hadoop.classification.VisibleForTesting; -import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; +import org.apache.hadoop.util.Preconditions; /** * NNStorage is responsible for management of the StorageDirectories used by diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NNStorageRetentionManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NNStorageRetentionManager.java index a4758bf695f94..b81cfccd75734 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NNStorageRetentionManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NNStorageRetentionManager.java @@ -38,7 +38,7 @@ import org.apache.hadoop.hdfs.util.MD5FileUtils; import org.apache.hadoop.util.Lists; -import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; +import org.apache.hadoop.util.Preconditions; import org.apache.hadoop.thirdparty.com.google.common.collect.ComparisonChain; /** diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NNUpgradeUtil.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NNUpgradeUtil.java index 8086b60637dae..4e9ebec1d8529 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NNUpgradeUtil.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NNUpgradeUtil.java @@ -34,7 +34,7 @@ import org.apache.hadoop.hdfs.server.common.Storage.StorageDirectory; import org.apache.hadoop.hdfs.server.common.StorageInfo; -import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; +import org.apache.hadoop.util.Preconditions; public abstract class NNUpgradeUtil { diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java index 5e601cdd9a803..2c5c27ebfce77 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java @@ -19,7 +19,7 @@ import org.apache.hadoop.classification.VisibleForTesting; import org.apache.hadoop.thirdparty.com.google.common.base.Joiner; -import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; +import org.apache.hadoop.util.Preconditions; import java.util.Set; import org.apache.commons.logging.Log; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/RedundantEditLogInputStream.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/RedundantEditLogInputStream.java index 15f799ab215ba..5c046cdce0ad3 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/RedundantEditLogInputStream.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/RedundantEditLogInputStream.java @@ -26,7 +26,7 @@ import org.apache.hadoop.hdfs.server.common.HdfsServerConstants; import org.apache.hadoop.io.IOUtils; -import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; +import org.apache.hadoop.util.Preconditions; import org.apache.hadoop.thirdparty.com.google.common.primitives.Longs; import org.apache.hadoop.log.LogThrottlingHelper; import org.apache.hadoop.log.LogThrottlingHelper.LogAction; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ReencryptionHandler.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ReencryptionHandler.java index da5413ab5e4e8..1a60879a970fc 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ReencryptionHandler.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ReencryptionHandler.java @@ -18,7 +18,7 @@ package org.apache.hadoop.hdfs.server.namenode; import org.apache.hadoop.classification.VisibleForTesting; -import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; +import org.apache.hadoop.util.Preconditions; import org.apache.hadoop.thirdparty.com.google.common.util.concurrent.ThreadFactoryBuilder; import org.apache.hadoop.classification.InterfaceAudience; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ReencryptionUpdater.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ReencryptionUpdater.java index 2650141718776..43684a7accccd 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ReencryptionUpdater.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ReencryptionUpdater.java @@ -32,7 +32,7 @@ import org.slf4j.LoggerFactory; import org.apache.hadoop.classification.VisibleForTesting; -import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; +import org.apache.hadoop.util.Preconditions; import java.io.IOException; import java.util.Arrays; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/SaveNamespaceContext.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/SaveNamespaceContext.java index 35a7cd2f643cc..ae6bef783cacc 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/SaveNamespaceContext.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/SaveNamespaceContext.java @@ -26,7 +26,7 @@ import org.apache.hadoop.hdfs.server.common.Storage.StorageDirectory; import org.apache.hadoop.hdfs.util.Canceler; -import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; +import org.apache.hadoop.util.Preconditions; /** * Context for an ongoing SaveNamespace operation. This class diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/SecondaryNameNode.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/SecondaryNameNode.java index 0163e7311d809..74730522d863a 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/SecondaryNameNode.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/SecondaryNameNode.java @@ -78,7 +78,7 @@ import org.apache.hadoop.util.Time; import org.apache.hadoop.classification.VisibleForTesting; -import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; +import org.apache.hadoop.util.Preconditions; import org.apache.hadoop.util.VersionInfo; import javax.management.ObjectName; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/XAttrFormat.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/XAttrFormat.java index af1025ab457ec..4d46e691df217 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/XAttrFormat.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/XAttrFormat.java @@ -25,7 +25,7 @@ import org.apache.hadoop.fs.XAttr; import org.apache.hadoop.hdfs.XAttrHelper; -import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; +import org.apache.hadoop.util.Preconditions; import org.apache.hadoop.thirdparty.com.google.common.primitives.Ints; import org.apache.hadoop.hdfs.util.LongBitFormat; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/XAttrPermissionFilter.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/XAttrPermissionFilter.java index da760e9ddd020..22c616067a6cd 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/XAttrPermissionFilter.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/XAttrPermissionFilter.java @@ -26,7 +26,7 @@ import org.apache.hadoop.security.AccessControlException; import org.apache.hadoop.util.Lists; -import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; +import org.apache.hadoop.util.Preconditions; import static org.apache.hadoop.hdfs.server.common.HdfsServerConstants.SECURITY_XATTR_UNREADABLE_BY_SUPERUSER; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/BootstrapStandby.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/BootstrapStandby.java index 3f273cb5e75e6..7a9ce46b1159f 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/BootstrapStandby.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/BootstrapStandby.java @@ -71,7 +71,7 @@ import org.apache.hadoop.util.ToolRunner; import org.apache.hadoop.thirdparty.com.google.common.base.Joiner; -import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; +import org.apache.hadoop.util.Preconditions; /** * Tool which allows the standby node's storage directories to be bootstrapped diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/EditLogTailer.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/EditLogTailer.java index cdea5eac04d33..4a5bd0f7ecb61 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/EditLogTailer.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/EditLogTailer.java @@ -59,7 +59,7 @@ import static org.apache.hadoop.util.ExitUtil.terminate; import org.apache.hadoop.classification.VisibleForTesting; -import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; +import org.apache.hadoop.util.Preconditions; /** diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/RemoteNameNodeInfo.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/RemoteNameNodeInfo.java index 1ff251d808c8c..2960c95c1cd95 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/RemoteNameNodeInfo.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/RemoteNameNodeInfo.java @@ -30,8 +30,6 @@ import org.apache.hadoop.hdfs.HAUtil; import org.apache.hadoop.hdfs.server.namenode.NameNode; -import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; - /** * Information about a single remote NameNode */ diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/StandbyCheckpointer.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/StandbyCheckpointer.java index 89df652f23e51..ec848668d2561 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/StandbyCheckpointer.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/StandbyCheckpointer.java @@ -49,7 +49,7 @@ import org.apache.hadoop.util.Lists; import org.apache.hadoop.classification.VisibleForTesting; -import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; +import org.apache.hadoop.util.Preconditions; import org.apache.hadoop.thirdparty.com.google.common.util.concurrent.ThreadFactoryBuilder; import org.slf4j.Logger; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/AbstractINodeDiff.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/AbstractINodeDiff.java index df052f171afa8..b1f15dbd1b331 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/AbstractINodeDiff.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/AbstractINodeDiff.java @@ -24,7 +24,7 @@ import org.apache.hadoop.hdfs.server.namenode.INodeAttributes; import org.apache.hadoop.hdfs.server.namenode.snapshot.SnapshotFSImageFormat.ReferenceMap; -import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; +import org.apache.hadoop.util.Preconditions; /** * The difference of an inode between in two snapshots. diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/DiffListBySkipList.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/DiffListBySkipList.java index dedc1e49d341f..ecdad34360937 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/DiffListBySkipList.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/DiffListBySkipList.java @@ -17,7 +17,7 @@ */ package org.apache.hadoop.hdfs.server.namenode.snapshot; -import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; +import org.apache.hadoop.util.Preconditions; import org.apache.hadoop.hdfs.server.namenode.INodeDirectory; import org.apache.hadoop.hdfs.server.namenode.snapshot. DirectoryWithSnapshotFeature.DirectoryDiff; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/DirectorySnapshottableFeature.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/DirectorySnapshottableFeature.java index 1d6b0e5a33159..c181c05d0811b 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/DirectorySnapshottableFeature.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/DirectorySnapshottableFeature.java @@ -49,7 +49,7 @@ import org.apache.hadoop.util.Time; import org.apache.hadoop.classification.VisibleForTesting; -import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; +import org.apache.hadoop.util.Preconditions; /** * A directory with this feature is a snapshottable directory, where snapshots diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/DirectoryWithSnapshotFeature.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/DirectoryWithSnapshotFeature.java index 2c0c8146f4044..3893f42fde2de 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/DirectoryWithSnapshotFeature.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/DirectoryWithSnapshotFeature.java @@ -17,7 +17,7 @@ */ package org.apache.hadoop.hdfs.server.namenode.snapshot; -import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; +import org.apache.hadoop.util.Preconditions; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.hdfs.server.blockmanagement.BlockStoragePolicySuite; import org.apache.hadoop.hdfs.server.namenode.*; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/FSImageFormatPBSnapshot.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/FSImageFormatPBSnapshot.java index a6e2d5debb13b..f1a21cce45a4b 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/FSImageFormatPBSnapshot.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/FSImageFormatPBSnapshot.java @@ -82,7 +82,7 @@ import org.apache.hadoop.hdfs.server.namenode.XAttrFeature; import org.apache.hadoop.hdfs.util.EnumCounters; -import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; +import org.apache.hadoop.util.Preconditions; import org.apache.hadoop.thirdparty.protobuf.ByteString; @InterfaceAudience.Private diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotDiffInfo.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotDiffInfo.java index 58dd2cf0a3f04..74d06c883b8d7 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotDiffInfo.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotDiffInfo.java @@ -33,7 +33,7 @@ import org.apache.hadoop.hdfs.server.namenode.INodeReference; import org.apache.hadoop.hdfs.server.namenode.snapshot.DirectoryWithSnapshotFeature.ChildrenDiff; -import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; +import org.apache.hadoop.util.Preconditions; import org.apache.hadoop.thirdparty.com.google.common.primitives.SignedBytes; import org.apache.hadoop.util.ChunkedArrayList; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotDiffListingInfo.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotDiffListingInfo.java index 4b03c4f0d53a1..e20a46e1b75f9 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotDiffListingInfo.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotDiffListingInfo.java @@ -29,7 +29,7 @@ import org.apache.hadoop.hdfs.server.namenode.INodeReference; import org.apache.hadoop.hdfs.server.namenode.snapshot.DirectoryWithSnapshotFeature.ChildrenDiff; -import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; +import org.apache.hadoop.util.Preconditions; import org.apache.hadoop.util.ChunkedArrayList; /** diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotFSImageFormat.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotFSImageFormat.java index b43c45854bbfd..21642da9c2463 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotFSImageFormat.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotFSImageFormat.java @@ -40,7 +40,7 @@ import org.apache.hadoop.hdfs.tools.snapshot.SnapshotDiff; import org.apache.hadoop.hdfs.util.ReadOnlyList; -import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; +import org.apache.hadoop.util.Preconditions; /** * A helper class defining static methods for reading/writing snapshot related diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotManager.java index d6c2263e5766e..7d192db362c65 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotManager.java @@ -65,7 +65,7 @@ import org.apache.hadoop.metrics2.util.MBeans; import org.apache.hadoop.util.Lists; -import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; +import org.apache.hadoop.util.Preconditions; import org.apache.hadoop.util.Time; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/sps/StoragePolicySatisfier.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/sps/StoragePolicySatisfier.java index 686644f3ec019..47596019af4ed 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/sps/StoragePolicySatisfier.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/sps/StoragePolicySatisfier.java @@ -58,7 +58,7 @@ import org.slf4j.LoggerFactory; import org.apache.hadoop.classification.VisibleForTesting; -import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; +import org.apache.hadoop.util.Preconditions; /** * Setting storagePolicy on a file after the file write will only update the new diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/top/TopAuditLogger.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/top/TopAuditLogger.java index 93eea6068c2fb..a43eb62d72b41 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/top/TopAuditLogger.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/top/TopAuditLogger.java @@ -19,7 +19,7 @@ import java.net.InetAddress; -import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; +import org.apache.hadoop.util.Preconditions; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileStatus; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/top/TopConf.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/top/TopConf.java index e78e41957d107..d0c0b6958cc27 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/top/TopConf.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/top/TopConf.java @@ -23,7 +23,7 @@ import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hdfs.DFSConfigKeys; -import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; +import org.apache.hadoop.util.Preconditions; /** * This class is a common place for NNTop configuration. diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/top/window/RollingWindowManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/top/window/RollingWindowManager.java index 3edc29e47d393..8015641e1f1bd 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/top/window/RollingWindowManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/top/window/RollingWindowManager.java @@ -29,7 +29,7 @@ import java.util.concurrent.ConcurrentHashMap; import org.apache.hadoop.hdfs.server.namenode.top.TopConf; -import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; +import org.apache.hadoop.util.Preconditions; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hdfs.DFSConfigKeys; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/visitor/NamespacePrintVisitor.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/visitor/NamespacePrintVisitor.java index ad7a6d9b6b68d..3dcc9e628dc07 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/visitor/NamespacePrintVisitor.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/visitor/NamespacePrintVisitor.java @@ -17,7 +17,7 @@ */ package org.apache.hadoop.hdfs.server.namenode.visitor; -import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; +import org.apache.hadoop.util.Preconditions; import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfo; import org.apache.hadoop.hdfs.server.namenode.DirectoryWithQuotaFeature; import org.apache.hadoop.hdfs.server.namenode.FSNamesystem; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/web/resources/NamenodeWebHdfsMethods.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/web/resources/NamenodeWebHdfsMethods.java index 3f2028e7c000c..5910a80700ec6 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/web/resources/NamenodeWebHdfsMethods.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/web/resources/NamenodeWebHdfsMethods.java @@ -85,6 +85,7 @@ import org.apache.hadoop.hdfs.protocol.HdfsFileStatus; import org.apache.hadoop.hdfs.protocol.LocatedBlocks; import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport; +import org.apache.hadoop.hdfs.protocol.SnapshotDiffReportListing; import org.apache.hadoop.hdfs.protocol.SnapshottableDirectoryStatus; import org.apache.hadoop.hdfs.protocol.SnapshotStatus; import org.apache.hadoop.hdfs.protocolPB.PBHelperClient; @@ -132,6 +133,7 @@ public class NamenodeWebHdfsMethods { private String scheme; private Principal userPrincipal; private String remoteAddr; + private int remotePort; private @Context ServletContext context; private @Context HttpServletResponse response; @@ -146,6 +148,7 @@ public NamenodeWebHdfsMethods(@Context HttpServletRequest request) { // get the remote address, if coming in via a trusted proxy server then // the address with be that of the proxied client remoteAddr = JspHelper.getRemoteAddr(request); + remotePort = JspHelper.getRemotePort(request); supportEZ = Boolean.valueOf(request.getHeader(WebHdfsFileSystem.EZ_HEADER)); } @@ -224,6 +227,10 @@ public String getHostAddress() { return getRemoteAddr(); } @Override + public int getRemotePort() { + return getRemotePortFromJSPHelper(); + } + @Override public InetAddress getHostInetAddress() { try { return InetAddress.getByName(getHostAddress()); @@ -254,6 +261,10 @@ protected String getRemoteAddr() { return remoteAddr; } + protected int getRemotePortFromJSPHelper() { + return remotePort; + } + protected void queueExternalCall(ExternalCall call) throws IOException, InterruptedException { final NameNode namenode = (NameNode)context.getAttribute("name.node"); @@ -1037,6 +1048,10 @@ public Response getRoot( final SnapshotNameParam snapshotName, @QueryParam(OldSnapshotNameParam.NAME) @DefaultValue(OldSnapshotNameParam.DEFAULT) final OldSnapshotNameParam oldSnapshotName, + @QueryParam(SnapshotDiffStartPathParam.NAME) @DefaultValue(SnapshotDiffStartPathParam.DEFAULT) + final SnapshotDiffStartPathParam snapshotDiffStartPath, + @QueryParam(SnapshotDiffIndexParam.NAME) @DefaultValue(SnapshotDiffIndexParam.DEFAULT) + final SnapshotDiffIndexParam snapshotDiffIndex, @QueryParam(TokenKindParam.NAME) @DefaultValue(TokenKindParam.DEFAULT) final TokenKindParam tokenKind, @QueryParam(TokenServiceParam.NAME) @DefaultValue(TokenServiceParam.DEFAULT) @@ -1048,7 +1063,9 @@ public Response getRoot( ) throws IOException, InterruptedException { return get(ugi, delegation, username, doAsUser, ROOT, op, offset, length, renewer, bufferSize, xattrNames, xattrEncoding, excludeDatanodes, - fsAction, snapshotName, oldSnapshotName, tokenKind, tokenService, + fsAction, snapshotName, oldSnapshotName, + snapshotDiffStartPath, snapshotDiffIndex, + tokenKind, tokenService, noredirect, startAfter); } @@ -1088,6 +1105,10 @@ public Response get( final SnapshotNameParam snapshotName, @QueryParam(OldSnapshotNameParam.NAME) @DefaultValue(OldSnapshotNameParam.DEFAULT) final OldSnapshotNameParam oldSnapshotName, + @QueryParam(SnapshotDiffStartPathParam.NAME) @DefaultValue(SnapshotDiffStartPathParam.DEFAULT) + final SnapshotDiffStartPathParam snapshotDiffStartPath, + @QueryParam(SnapshotDiffIndexParam.NAME) @DefaultValue(SnapshotDiffIndexParam.DEFAULT) + final SnapshotDiffIndexParam snapshotDiffIndex, @QueryParam(TokenKindParam.NAME) @DefaultValue(TokenKindParam.DEFAULT) final TokenKindParam tokenKind, @QueryParam(TokenServiceParam.NAME) @DefaultValue(TokenServiceParam.DEFAULT) @@ -1108,6 +1129,7 @@ public Response run() throws IOException, URISyntaxException { return get(ugi, delegation, username, doAsUser, path.getAbsolutePath(), op, offset, length, renewer, bufferSize, xattrNames, xattrEncoding, excludeDatanodes, fsAction, snapshotName, oldSnapshotName, + snapshotDiffStartPath, snapshotDiffIndex, tokenKind, tokenService, noredirect, startAfter); } }); @@ -1137,6 +1159,8 @@ protected Response get( final FsActionParam fsAction, final SnapshotNameParam snapshotName, final OldSnapshotNameParam oldSnapshotName, + final SnapshotDiffStartPathParam snapshotDiffStartPath, + final SnapshotDiffIndexParam snapshotDiffIndex, final TokenKindParam tokenKind, final TokenServiceParam tokenService, final NoRedirectParam noredirectParam, @@ -1335,6 +1359,14 @@ protected Response get( final String js = JsonUtil.toJsonString(diffReport); return Response.ok(js).type(MediaType.APPLICATION_JSON).build(); } + case GETSNAPSHOTDIFFLISTING: { + SnapshotDiffReportListing diffReport = cp.getSnapshotDiffReportListing( + fullpath, oldSnapshotName.getValue(), snapshotName.getValue(), + DFSUtilClient.string2Bytes(snapshotDiffStartPath.getValue()), + snapshotDiffIndex.getValue()); + final String js = JsonUtil.toJsonString(diffReport); + return Response.ok(js).type(MediaType.APPLICATION_JSON).build(); + } case GETSNAPSHOTTABLEDIRECTORYLIST: { SnapshottableDirectoryStatus[] snapshottableDirectoryList = cp.getSnapshottableDirListing(); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/BlocksWithLocations.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/BlocksWithLocations.java index f60d748dc9f56..21fc09b183b49 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/BlocksWithLocations.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/BlocksWithLocations.java @@ -17,7 +17,7 @@ */ package org.apache.hadoop.hdfs.server.protocol; -import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; +import org.apache.hadoop.util.Preconditions; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.fs.StorageType; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/NamespaceInfo.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/NamespaceInfo.java index 8b94a11594146..93125a22dbd24 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/NamespaceInfo.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/NamespaceInfo.java @@ -31,7 +31,7 @@ import org.apache.hadoop.util.VersionInfo; import org.apache.hadoop.classification.VisibleForTesting; -import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; +import org.apache.hadoop.util.Preconditions; /** * NamespaceInfo is returned by the name-node in reply diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/RemoteEditLogManifest.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/RemoteEditLogManifest.java index 391078f558509..8cd7ebd1cefac 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/RemoteEditLogManifest.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/RemoteEditLogManifest.java @@ -21,7 +21,7 @@ import java.util.List; import org.apache.hadoop.thirdparty.com.google.common.base.Joiner; -import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; +import org.apache.hadoop.util.Preconditions; import org.apache.hadoop.hdfs.server.common.HdfsServerConstants; /** diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/AdminHelper.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/AdminHelper.java index f619fb40f55d9..36f4f026ce5f4 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/AdminHelper.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/AdminHelper.java @@ -18,7 +18,7 @@ */ package org.apache.hadoop.hdfs.tools; -import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; +import org.apache.hadoop.util.Preconditions; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSAdmin.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSAdmin.java index 4f57040867588..dec8bef24b4df 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSAdmin.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSAdmin.java @@ -105,7 +105,7 @@ import org.apache.hadoop.util.StringUtils; import org.apache.hadoop.util.ToolRunner; -import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; +import org.apache.hadoop.util.Preconditions; /** * This class provides some DFS administrative access shell commands. diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSHAAdmin.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSHAAdmin.java index 15c63732f7a69..4d0d56c05a75c 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSHAAdmin.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSHAAdmin.java @@ -23,7 +23,7 @@ import java.util.Collection; import java.util.Map; -import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; +import org.apache.hadoop.util.Preconditions; import org.apache.hadoop.thirdparty.com.google.common.collect.ImmutableMap; import org.apache.hadoop.thirdparty.com.google.common.collect.ImmutableSortedMap; import org.apache.commons.cli.CommandLine; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DebugAdmin.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DebugAdmin.java index f5967e15a63d0..32e8248adc692 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DebugAdmin.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DebugAdmin.java @@ -24,15 +24,41 @@ import java.io.FileInputStream; import java.io.FileNotFoundException; import java.io.IOException; +import java.net.InetSocketAddress; import java.net.URI; import java.net.URISyntaxException; import java.nio.ByteBuffer; import java.nio.channels.FileChannel; import java.nio.file.Files; +import java.util.ArrayList; import java.util.LinkedList; import java.util.List; +import java.util.concurrent.CompletionService; +import java.util.concurrent.ExecutorCompletionService; +import java.util.concurrent.Future; +import java.util.concurrent.LinkedBlockingQueue; import java.util.concurrent.TimeUnit; +import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.hdfs.BlockReader; +import org.apache.hadoop.hdfs.DFSClient; +import org.apache.hadoop.hdfs.DFSConfigKeys; +import org.apache.hadoop.hdfs.client.impl.BlockReaderRemote; +import org.apache.hadoop.hdfs.net.Peer; +import org.apache.hadoop.hdfs.protocol.DatanodeInfo; +import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy; +import org.apache.hadoop.hdfs.protocol.ExtendedBlock; +import org.apache.hadoop.hdfs.protocol.LocatedBlock; +import org.apache.hadoop.hdfs.protocol.LocatedBlocks; +import org.apache.hadoop.hdfs.protocol.LocatedStripedBlock; +import org.apache.hadoop.hdfs.security.token.block.BlockTokenIdentifier; +import org.apache.hadoop.hdfs.server.datanode.CachingStrategy; +import org.apache.hadoop.hdfs.util.StripedBlockUtil; +import org.apache.hadoop.io.erasurecode.CodecUtil; +import org.apache.hadoop.io.erasurecode.ErasureCoderOptions; +import org.apache.hadoop.io.erasurecode.rawcoder.RawErasureEncoder; +import org.apache.hadoop.net.NetUtils; +import org.apache.hadoop.security.token.Token; import org.apache.hadoop.thirdparty.com.google.common.util.concurrent.Uninterruptibles; import org.apache.hadoop.HadoopIllegalArgumentException; import org.apache.hadoop.classification.InterfaceAudience; @@ -69,6 +95,7 @@ public class DebugAdmin extends Configured implements Tool { new VerifyMetaCommand(), new ComputeMetaCommand(), new RecoverLeaseCommand(), + new VerifyECCommand(), new HelpCommand() }; @@ -387,6 +414,209 @@ int run(List args) throws IOException { } } + /** + * The command for verifying the correctness of erasure coding on an erasure coded file. + */ + private class VerifyECCommand extends DebugCommand { + private DFSClient client; + private int dataBlkNum; + private int parityBlkNum; + private int cellSize; + private boolean useDNHostname; + private CachingStrategy cachingStrategy; + private int stripedReadBufferSize; + private CompletionService readService; + private RawErasureEncoder encoder; + private BlockReader[] blockReaders; + + + VerifyECCommand() { + super("verifyEC", + "verifyEC -file ", + " Verify HDFS erasure coding on all block groups of the file."); + } + + int run(List args) throws IOException { + if (args.size() < 2) { + System.out.println(usageText); + System.out.println(helpText + System.lineSeparator()); + return 1; + } + String file = StringUtils.popOptionWithArgument("-file", args); + Path path = new Path(file); + DistributedFileSystem dfs = AdminHelper.getDFS(getConf()); + this.client = dfs.getClient(); + + FileStatus fileStatus; + try { + fileStatus = dfs.getFileStatus(path); + } catch (FileNotFoundException e) { + System.err.println("File " + file + " does not exist."); + return 1; + } + + if (!fileStatus.isFile()) { + System.err.println("File " + file + " is not a regular file."); + return 1; + } + if (!dfs.isFileClosed(path)) { + System.err.println("File " + file + " is not closed."); + return 1; + } + this.useDNHostname = getConf().getBoolean(DFSConfigKeys.DFS_DATANODE_USE_DN_HOSTNAME, + DFSConfigKeys.DFS_DATANODE_USE_DN_HOSTNAME_DEFAULT); + this.cachingStrategy = CachingStrategy.newDefaultStrategy(); + this.stripedReadBufferSize = getConf().getInt( + DFSConfigKeys.DFS_DN_EC_RECONSTRUCTION_STRIPED_READ_BUFFER_SIZE_KEY, + DFSConfigKeys.DFS_DN_EC_RECONSTRUCTION_STRIPED_READ_BUFFER_SIZE_DEFAULT); + + LocatedBlocks locatedBlocks = client.getLocatedBlocks(file, 0, fileStatus.getLen()); + if (locatedBlocks.getErasureCodingPolicy() == null) { + System.err.println("File " + file + " is not erasure coded."); + return 1; + } + ErasureCodingPolicy ecPolicy = locatedBlocks.getErasureCodingPolicy(); + this.dataBlkNum = ecPolicy.getNumDataUnits(); + this.parityBlkNum = ecPolicy.getNumParityUnits(); + this.cellSize = ecPolicy.getCellSize(); + this.encoder = CodecUtil.createRawEncoder(getConf(), ecPolicy.getCodecName(), + new ErasureCoderOptions( + ecPolicy.getNumDataUnits(), ecPolicy.getNumParityUnits())); + int blockNum = dataBlkNum + parityBlkNum; + this.readService = new ExecutorCompletionService<>( + DFSUtilClient.getThreadPoolExecutor(blockNum, blockNum, 60, + new LinkedBlockingQueue<>(), "read-", false)); + this.blockReaders = new BlockReader[dataBlkNum + parityBlkNum]; + + for (LocatedBlock locatedBlock : locatedBlocks.getLocatedBlocks()) { + System.out.println("Checking EC block group: blk_" + locatedBlock.getBlock().getBlockId()); + LocatedStripedBlock blockGroup = (LocatedStripedBlock) locatedBlock; + + try { + verifyBlockGroup(blockGroup); + System.out.println("Status: OK"); + } catch (Exception e) { + System.err.println("Status: ERROR, message: " + e.getMessage()); + return 1; + } finally { + closeBlockReaders(); + } + } + System.out.println("\nAll EC block group status: OK"); + return 0; + } + + private void verifyBlockGroup(LocatedStripedBlock blockGroup) throws Exception { + final LocatedBlock[] indexedBlocks = StripedBlockUtil.parseStripedBlockGroup(blockGroup, + cellSize, dataBlkNum, parityBlkNum); + + int blockNumExpected = Math.min(dataBlkNum, + (int) ((blockGroup.getBlockSize() - 1) / cellSize + 1)) + parityBlkNum; + if (blockGroup.getBlockIndices().length < blockNumExpected) { + throw new Exception("Block group is under-erasure-coded."); + } + + long maxBlockLen = 0L; + DataChecksum checksum = null; + for (int i = 0; i < dataBlkNum + parityBlkNum; i++) { + LocatedBlock block = indexedBlocks[i]; + if (block == null) { + blockReaders[i] = null; + continue; + } + if (block.getBlockSize() > maxBlockLen) { + maxBlockLen = block.getBlockSize(); + } + BlockReader blockReader = createBlockReader(block.getBlock(), + block.getLocations()[0], block.getBlockToken()); + if (checksum == null) { + checksum = blockReader.getDataChecksum(); + } else { + assert checksum.equals(blockReader.getDataChecksum()); + } + blockReaders[i] = blockReader; + } + assert checksum != null; + int bytesPerChecksum = checksum.getBytesPerChecksum(); + int bufferSize = stripedReadBufferSize < bytesPerChecksum ? bytesPerChecksum : + stripedReadBufferSize - stripedReadBufferSize % bytesPerChecksum; + final ByteBuffer[] buffers = new ByteBuffer[dataBlkNum + parityBlkNum]; + final ByteBuffer[] outputs = new ByteBuffer[parityBlkNum]; + for (int i = 0; i < dataBlkNum + parityBlkNum; i++) { + buffers[i] = ByteBuffer.allocate(bufferSize); + } + for (int i = 0; i < parityBlkNum; i++) { + outputs[i] = ByteBuffer.allocate(bufferSize); + } + long positionInBlock = 0L; + while (positionInBlock < maxBlockLen) { + final int toVerifyLen = (int) Math.min(bufferSize, maxBlockLen - positionInBlock); + List> futures = new ArrayList<>(dataBlkNum + parityBlkNum); + for (int i = 0; i < dataBlkNum + parityBlkNum; i++) { + final int fi = i; + futures.add(this.readService.submit(() -> { + BlockReader blockReader = blockReaders[fi]; + ByteBuffer buffer = buffers[fi]; + buffer.clear(); + buffer.limit(toVerifyLen); + int readLen = 0; + if (blockReader != null) { + int toRead = buffer.remaining(); + while (readLen < toRead) { + int nread = blockReader.read(buffer); + if (nread <= 0) { + break; + } + readLen += nread; + } + } + while (buffer.hasRemaining()) { + buffer.put((byte) 0); + } + buffer.flip(); + return readLen; + })); + } + for (int i = 0; i < dataBlkNum + parityBlkNum; i++) { + futures.get(i).get(1, TimeUnit.MINUTES); + } + ByteBuffer[] inputs = new ByteBuffer[dataBlkNum]; + System.arraycopy(buffers, 0, inputs, 0, dataBlkNum); + for (int i = 0; i < parityBlkNum; i++) { + outputs[i].clear(); + outputs[i].limit(toVerifyLen); + } + this.encoder.encode(inputs, outputs); + for (int i = 0; i < parityBlkNum; i++) { + if (!buffers[dataBlkNum + i].equals(outputs[i])) { + throw new Exception("EC compute result not match."); + } + } + positionInBlock += toVerifyLen; + } + } + + private BlockReader createBlockReader(ExtendedBlock block, DatanodeInfo dnInfo, + Token token) throws IOException { + InetSocketAddress dnAddress = NetUtils.createSocketAddr(dnInfo.getXferAddr(useDNHostname)); + Peer peer = client.newConnectedPeer(dnAddress, token, dnInfo); + return BlockReaderRemote.newBlockReader( + "dummy", block, token, 0, + block.getNumBytes(), true, "", peer, dnInfo, + null, cachingStrategy, -1, getConf()); + } + + private void closeBlockReaders() { + for (int i = 0; i < blockReaders.length; i++) { + if (blockReaders[i] != null) { + IOUtils.closeStream(blockReaders[i]); + blockReaders[i] = null; + } + } + } + + } + /** * The command for getting help about other commands. */ @@ -459,9 +689,9 @@ private void printUsage() { if (!command.name.equals("help")) { System.out.println(command.usageText); } - System.out.println(); - ToolRunner.printGenericCommandUsage(System.out); } + System.out.println(); + ToolRunner.printGenericCommandUsage(System.out); } public static void main(String[] argsArray) throws Exception { diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/NNHAServiceTarget.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/NNHAServiceTarget.java index 78a1bc08f1471..cb3a1e4a83bfb 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/NNHAServiceTarget.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/NNHAServiceTarget.java @@ -33,7 +33,7 @@ import org.apache.hadoop.hdfs.server.namenode.NameNode; import org.apache.hadoop.net.NetUtils; -import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; +import org.apache.hadoop.util.Preconditions; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMESERVICES; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/FSImageLoader.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/FSImageLoader.java index af459617667cc..26d8c0ff949b1 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/FSImageLoader.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/FSImageLoader.java @@ -52,7 +52,7 @@ import org.apache.hadoop.io.IOUtils; import org.apache.hadoop.util.LimitInputStream; -import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; +import org.apache.hadoop.util.Preconditions; import org.apache.hadoop.thirdparty.com.google.common.collect.ImmutableList; import org.apache.hadoop.thirdparty.com.google.common.collect.Maps; import org.apache.hadoop.thirdparty.protobuf.CodedInputStream; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/FileDistributionCalculator.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/FileDistributionCalculator.java index 54b183b7b6965..fbeea0f673c0e 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/FileDistributionCalculator.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/FileDistributionCalculator.java @@ -33,7 +33,7 @@ import org.apache.hadoop.util.LimitInputStream; import org.apache.hadoop.util.StringUtils; -import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; +import org.apache.hadoop.util.Preconditions; /** * This is the tool for analyzing file sizes in the namespace image. In order to diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/OfflineImageReconstructor.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/OfflineImageReconstructor.java index 9ad4b090649b2..1aedcaf65cf18 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/OfflineImageReconstructor.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/OfflineImageReconstructor.java @@ -17,7 +17,7 @@ */ package org.apache.hadoop.hdfs.tools.offlineImageViewer; -import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; +import org.apache.hadoop.util.Preconditions; import static org.apache.hadoop.hdfs.server.namenode.FSImageFormatPBINode.ACL_ENTRY_NAME_MASK; import static org.apache.hadoop.hdfs.server.namenode.FSImageFormatPBINode.ACL_ENTRY_NAME_OFFSET; import static org.apache.hadoop.hdfs.server.namenode.FSImageFormatPBINode.ACL_ENTRY_SCOPE_OFFSET; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/PBImageCorruptionDetector.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/PBImageCorruptionDetector.java index 737e7384b9a7c..28c450701b846 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/PBImageCorruptionDetector.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/PBImageCorruptionDetector.java @@ -17,7 +17,7 @@ */ package org.apache.hadoop.hdfs.tools.offlineImageViewer; -import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; +import org.apache.hadoop.util.Preconditions; import org.apache.hadoop.hdfs.server.namenode.FsImageProto; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/PBImageTextWriter.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/PBImageTextWriter.java index ef0b168658c68..08fe7fb943c15 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/PBImageTextWriter.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/PBImageTextWriter.java @@ -66,7 +66,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; +import org.apache.hadoop.util.Preconditions; import org.apache.hadoop.thirdparty.com.google.common.collect.ImmutableList; /** diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/util/Diff.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/util/Diff.java index 21a7bb58750b8..298e645d43224 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/util/Diff.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/util/Diff.java @@ -17,7 +17,7 @@ */ package org.apache.hadoop.hdfs.util; -import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; +import org.apache.hadoop.util.Preconditions; import java.util.ArrayList; import java.util.Collections; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/util/EnumCounters.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/util/EnumCounters.java index fc3039af70ec5..9ebfa665e1c1d 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/util/EnumCounters.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/util/EnumCounters.java @@ -17,7 +17,7 @@ */ package org.apache.hadoop.hdfs.util; -import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; +import org.apache.hadoop.util.Preconditions; import org.apache.commons.lang3.ArrayUtils; import java.util.Arrays; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/util/EnumDoubles.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/util/EnumDoubles.java index 9012ceab3e198..dde7297f56a6f 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/util/EnumDoubles.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/util/EnumDoubles.java @@ -19,7 +19,7 @@ import java.util.Arrays; -import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; +import org.apache.hadoop.util.Preconditions; /** * Similar to {@link EnumCounters} except that the value type is double. diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/JsonUtil.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/JsonUtil.java index 307b0e8440fe2..1744b3dde056e 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/JsonUtil.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/JsonUtil.java @@ -577,6 +577,58 @@ private static Object toJsonMap( return m; } + public static String toJsonString(SnapshotDiffReportListing diffReport) { + return toJsonString(SnapshotDiffReportListing.class.getSimpleName(), + toJsonMap(diffReport)); + } + + private static Object toJsonMap(SnapshotDiffReportListing diffReport) { + final Map m = new TreeMap(); + m.put("lastPath", DFSUtilClient.bytes2String(diffReport.getLastPath())); + m.put("lastIndex", diffReport.getLastIndex()); + m.put("isFromEarlier", diffReport.getIsFromEarlier()); + + Object[] modifyList = new Object[diffReport.getModifyList().size()]; + for (int i = 0; i < diffReport.getModifyList().size(); i++) { + modifyList[i] = toJsonMap(diffReport.getModifyList().get(i)); + } + m.put("modifyList", modifyList); + + Object[] createList = new Object[diffReport.getCreateList().size()]; + for (int i = 0; i < diffReport.getCreateList().size(); i++) { + createList[i] = toJsonMap(diffReport.getCreateList().get(i)); + } + m.put("createList", createList); + + Object[] deleteList = new Object[diffReport.getDeleteList().size()]; + for (int i = 0; i < diffReport.getDeleteList().size(); i++) { + deleteList[i] = toJsonMap(diffReport.getDeleteList().get(i)); + } + m.put("deleteList", deleteList); + + return m; + } + + private static Object toJsonMap( + SnapshotDiffReportListing.DiffReportListingEntry diffReportEntry) { + final Map m = new TreeMap(); + m.put("dirId", diffReportEntry.getDirId()); + m.put("fileId", diffReportEntry.getFileId()); + + if (diffReportEntry.getSourcePath() != null) { + m.put("sourcePath", + DFSUtilClient.byteArray2String(diffReportEntry.getSourcePath())); + } + + if (diffReportEntry.getTargetPath() != null) { + m.put("targetPath", + DFSUtilClient.byteArray2String(diffReportEntry.getTargetPath())); + } + + m.put("isReference", diffReportEntry.isReference()); + return m; + } + public static String toJsonString( SnapshottableDirectoryStatus[] snapshottableDirectoryList) { if (snapshottableDirectoryList == null) { diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml index a49137bf582cc..7bcbccd81728c 100755 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml @@ -5056,6 +5056,14 @@ + + dfs.namenode.audit.log.with.remote.port + false + + If true, adds a port of RPC call to callerContext for all audit log events. + + + dfs.namenode.available-space-block-placement-policy.balanced-space-preference-fraction 0.6 @@ -5067,6 +5075,18 @@ + + dfs.namenode.available-space-block-placement-policy.balanced-space-tolerance + 5 + + Only used when the dfs.block.replicator.classname is set to + org.apache.hadoop.hdfs.server.blockmanagement.AvailableSpaceBlockPlacementPolicy. + Special value between 0 and 20, inclusive. if the value is set beyond the scope, + this value will be set as 5 by default, Increases tolerance of + placing blocks on Datanodes with similar disk space used. + + + dfs.namenode.available-space-block-placement-policy.balance-local-node @@ -5092,7 +5112,17 @@ high load increases as the value reaches near 0. - + + dfs.namenode.available-space-rack-fault-tolerant-block-placement-policy.balanced-space-tolerance + 5 + + Only used when the dfs.block.replicator.classname is set to + org.apache.hadoop.hdfs.server.blockmanagement.AvailableSpaceRackFaultTolerantBlockPlacementPolicy. + Special value between 0 and 20, inclusive. if the value is set beyond the scope, + this value will be set as 5 by default, Increases tolerance of + placing blocks on Datanodes with similar disk space used. + + dfs.namenode.backup.dnrpc-address diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSCommands.md b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSCommands.md index a1509d2d01d1d..df333ef51ecc8 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSCommands.md +++ b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSCommands.md @@ -708,6 +708,16 @@ Usage: `hdfs debug recoverLease -path [-retries ]` Recover the lease on the specified path. The path must reside on an HDFS file system. The default number of retries is 1. +### `verifyEC` + +Usage: `hdfs debug verifyEC -file ` + +| COMMAND\_OPTION | Description | +|:---- |:---- | +| [`-file` *EC-file*] | HDFS EC file to be verified. | + +Verify the correctness of erasure coding on an erasure coded file. + dfsadmin with ViewFsOverloadScheme ---------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HdfsBlockPlacementPolicies.md b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HdfsBlockPlacementPolicies.md index 4550f0441b62e..cc664ec4987a9 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HdfsBlockPlacementPolicies.md +++ b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HdfsBlockPlacementPolicies.md @@ -119,6 +119,16 @@ The AvailableSpaceBlockPlacementPolicy is a space balanced block placement polic + +dfs.namenode.available-space-block-placement-policy.balanced-space-tolerance +5 + + Special value between 0 and 20, inclusive. if the value is set beyond the scope, + this value will be set as 5 by default, Increases tolerance of + placing blocks on Datanodes with similar disk space used. + + + dfs.namenode.available-space-block-placement-policy.balance-local-node @@ -160,6 +170,18 @@ amongst maximum number of racks possible and at the same time will try to choose high load increases as the value reaches near 0. + + + dfs.namenode.available-space-rack-fault-tolerant-block-placement-policy.balanced-space-tolerance + 5 + + Only used when the dfs.block.replicator.classname is set to + org.apache.hadoop.hdfs.server.blockmanagement.AvailableSpaceRackFaultTolerantBlockPlacementPolicy. + Special value between 0 and 20, inclusive. if the value is set beyond the scope, + this value will be set as 5 by default, Increases tolerance of + placing blocks on Datanodes with similar disk space used. + + ``` For more details check [HDFS-15288](https://issues.apache.org/jira/browse/HDFS-15288) \ No newline at end of file diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/TestEnhancedByteBufferAccess.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/TestEnhancedByteBufferAccess.java index 99b1ddbbc1130..7cf216b45508d 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/TestEnhancedByteBufferAccess.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/TestEnhancedByteBufferAccess.java @@ -71,7 +71,7 @@ import org.junit.BeforeClass; import org.junit.Test; -import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; +import org.apache.hadoop.util.Preconditions; import java.util.function.Supplier; /** diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/viewfs/TestViewFileSystemOverloadSchemeHdfsFileSystemContract.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/viewfs/TestViewFileSystemOverloadSchemeHdfsFileSystemContract.java index dcfa051c3902d..3d0e705e6a98c 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/viewfs/TestViewFileSystemOverloadSchemeHdfsFileSystemContract.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/viewfs/TestViewFileSystemOverloadSchemeHdfsFileSystemContract.java @@ -103,7 +103,7 @@ protected String getDefaultWorkingDirectory() { } @Override - @Test + @Test(timeout = 60000) public void testAppend() throws IOException { AppendTestUtil.testAppend(fs, new Path("/append/f")); } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/viewfs/TestViewFileSystemOverloadSchemeWithHdfsScheme.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/viewfs/TestViewFileSystemOverloadSchemeWithHdfsScheme.java index 650a4722798e8..76eee6d727c06 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/viewfs/TestViewFileSystemOverloadSchemeWithHdfsScheme.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/viewfs/TestViewFileSystemOverloadSchemeWithHdfsScheme.java @@ -496,7 +496,7 @@ private boolean isFallBackExist(Configuration config) { * When InnerCache disabled, all matching ViewFileSystemOverloadScheme * initialized scheme file systems would not use FileSystem cache. */ - @Test(timeout = 3000) + @Test(timeout = 30000) public void testViewFsOverloadSchemeWithNoInnerCacheAndHdfsTargets() throws Exception { final Path hdfsTargetPath = new Path(defaultFSURI + HDFS_USER_FOLDER); @@ -523,7 +523,7 @@ public void testViewFsOverloadSchemeWithNoInnerCacheAndHdfsTargets() * initialized scheme file systems should continue to take advantage of * FileSystem cache. */ - @Test(timeout = 3000) + @Test(timeout = 30000) public void testViewFsOverloadSchemeWithNoInnerCacheAndLocalSchemeTargets() throws Exception { final Path localTragetPath = new Path(localTargetDir.toURI()); @@ -545,7 +545,7 @@ public void testViewFsOverloadSchemeWithNoInnerCacheAndLocalSchemeTargets() /** * Tests the rename with nfly mount link. */ - @Test(timeout = 3000) + @Test(timeout = 30000) public void testNflyRename() throws Exception { final Path hdfsTargetPath1 = new Path(defaultFSURI + HDFS_USER_FOLDER); final Path hdfsTargetPath2 = new Path(defaultFSURI + HDFS_USER_FOLDER + 1); @@ -577,7 +577,7 @@ public void testNflyRename() throws Exception { /** * Tests the write and read contents with nfly mount link. */ - @Test(timeout = 3000) + @Test(timeout = 30000) public void testNflyWriteRead() throws Exception { final Path hdfsTargetPath1 = new Path(defaultFSURI + HDFS_USER_FOLDER); final Path hdfsTargetPath2 = new Path(defaultFSURI + HDFS_USER_FOLDER + 1); @@ -604,7 +604,7 @@ public void testNflyWriteRead() throws Exception { * target file. 3. Tests the read works with repairOnRead flag. 4. Tests that * previously deleted file fully recovered and exists. */ - @Test(timeout = 3000) + @Test(timeout = 30000) public void testNflyRepair() throws Exception { final NflyFSystem.NflyKey repairKey = NflyFSystem.NflyKey.repairOnRead; final Path hdfsTargetPath1 = new Path(defaultFSURI + HDFS_USER_FOLDER); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java index f27f9bee9913a..594b46f6cacb0 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java @@ -72,7 +72,7 @@ import org.apache.hadoop.thirdparty.com.google.common.base.Charsets; import org.apache.hadoop.thirdparty.com.google.common.base.Joiner; -import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; +import org.apache.hadoop.util.Preconditions; import org.apache.hadoop.thirdparty.com.google.common.base.Strings; import java.util.function.Supplier; import org.apache.hadoop.thirdparty.com.google.common.collect.Maps; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/ErasureCodeBenchmarkThroughput.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/ErasureCodeBenchmarkThroughput.java index ba5a451d8c2f6..b2ef1b4ec3065 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/ErasureCodeBenchmarkThroughput.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/ErasureCodeBenchmarkThroughput.java @@ -17,7 +17,7 @@ */ package org.apache.hadoop.hdfs; -import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; +import org.apache.hadoop.util.Preconditions; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.conf.Configured; import org.apache.hadoop.fs.FSDataInputStream; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java index 04a7611032826..e4b6434b4860a 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java @@ -148,7 +148,7 @@ import org.apache.hadoop.util.ToolRunner; import org.apache.hadoop.thirdparty.com.google.common.base.Joiner; -import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; +import org.apache.hadoop.util.Preconditions; /** * This class creates a single-process DFS cluster for junit testing. diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSNNTopology.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSNNTopology.java index 31707d672c39a..0a8161c9c2efa 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSNNTopology.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSNNTopology.java @@ -22,7 +22,7 @@ import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; -import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; +import org.apache.hadoop.util.Preconditions; import org.apache.hadoop.util.Lists; /** diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestAppendSnapshotTruncate.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestAppendSnapshotTruncate.java index b4e9550e11828..8a702d571644f 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestAppendSnapshotTruncate.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestAppendSnapshotTruncate.java @@ -50,7 +50,7 @@ import org.junit.BeforeClass; import org.junit.Test; -import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; +import org.apache.hadoop.util.Preconditions; import org.slf4j.event.Level; /** diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStreamWithFailureBase.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStreamWithFailureBase.java index bbe991dacc781..7c2775aa8fdc1 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStreamWithFailureBase.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStreamWithFailureBase.java @@ -17,7 +17,7 @@ */ package org.apache.hadoop.hdfs; -import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; +import org.apache.hadoop.util.Preconditions; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.fs.Path; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestHDFSFileSystemContract.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestHDFSFileSystemContract.java index 6da46dec378d2..3a8528968dc6c 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestHDFSFileSystemContract.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestHDFSFileSystemContract.java @@ -63,7 +63,7 @@ protected String getDefaultWorkingDirectory() { return defaultWorkingDirectory; } - @Test + @Test(timeout = 60000) public void testAppend() throws IOException { AppendTestUtil.testAppend(fs, new Path("/testAppend/f")); } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestLeaseRecoveryStriped.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestLeaseRecoveryStriped.java index 5aa1f6383704f..cc3e5ed2c7328 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestLeaseRecoveryStriped.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestLeaseRecoveryStriped.java @@ -17,7 +17,7 @@ */ package org.apache.hadoop.hdfs; -import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; +import org.apache.hadoop.util.Preconditions; import java.util.function.Supplier; import org.apache.commons.lang3.builder.ToStringBuilder; import org.apache.hadoop.conf.Configuration; @@ -188,6 +188,24 @@ public void testLeaseRecovery() throws Exception { } } + @Test + public void testSafeLength() { + checkSafeLength(0, 0); // Length of: 0 + checkSafeLength(1024 * 1024, 6291456L); // Length of: 1 MiB + checkSafeLength(64 * 1024 * 1024, 402653184L); // Length of: 64 MiB + checkSafeLength(189729792, 1132462080L); // Length of: 189729792 + checkSafeLength(256 * 1024 * 1024, 1610612736L); // Length of: 256 MiB + checkSafeLength(517399040, 3101687808L); // Length of: 517399040 + checkSafeLength(1024 * 1024 * 1024, 6442450944L); // Length of: 1 GiB + } + + private void checkSafeLength(int blockLength, long expectedSafeLength) { + int[] blockLengths = new int[]{blockLength, blockLength, blockLength, blockLength, + blockLength, blockLength}; + long safeLength = new BlockLengths(ecPolicy, blockLengths).getSafeLength(); + Assert.assertEquals(expectedSafeLength, safeLength); + } + private void runTest(int[] blockLengths, long safeLength) throws Exception { writePartialBlocks(blockLengths); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestMiniDFSCluster.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestMiniDFSCluster.java index 74a8e44bf7b0b..74cfe9d673cb6 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestMiniDFSCluster.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestMiniDFSCluster.java @@ -42,7 +42,7 @@ import org.junit.Before; import org.junit.Test; -import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; +import org.apache.hadoop.util.Preconditions; /** * Tests MiniDFS cluster setup/teardown and isolation. diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestSnapshotCommands.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestSnapshotCommands.java index 32ac2980cd590..52bc12f4d8098 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestSnapshotCommands.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestSnapshotCommands.java @@ -224,7 +224,7 @@ public void testSnapshotCommandsWithURI()throws Exception { fs.delete(new Path("/Fully/QPath"), true); } - @Test (timeout=60000) + @Test (timeout=120000) public void testSnapshotDiff()throws Exception { Configuration config = new HdfsConfiguration(); Path snapDirPath = new Path(fs.getUri().toString() + "/snap_dir"); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/UpgradeUtilities.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/UpgradeUtilities.java index 2b90c92388f32..7ebf55f571cd2 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/UpgradeUtilities.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/UpgradeUtilities.java @@ -51,7 +51,7 @@ import org.apache.hadoop.hdfs.server.protocol.DatanodeStorage; import org.apache.hadoop.hdfs.server.protocol.NamenodeProtocols; -import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; +import org.apache.hadoop.util.Preconditions; import org.apache.hadoop.thirdparty.com.google.common.io.Files; import org.apache.hadoop.thirdparty.com.google.common.primitives.Bytes; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/client/TestQJMWithFaults.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/client/TestQJMWithFaults.java index 6cdbe2d31d9f3..519d94af10afa 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/client/TestQJMWithFaults.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/client/TestQJMWithFaults.java @@ -59,7 +59,7 @@ import org.mockito.invocation.InvocationOnMock; import org.mockito.stubbing.Answer; -import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; +import org.apache.hadoop.util.Preconditions; import org.apache.hadoop.thirdparty.com.google.common.collect.Maps; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManagerTestUtil.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManagerTestUtil.java index 2b8804c12cc6a..97c65556e7d15 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManagerTestUtil.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManagerTestUtil.java @@ -36,7 +36,7 @@ import org.apache.hadoop.test.Whitebox; import org.junit.Assert; -import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; +import org.apache.hadoop.util.Preconditions; public class BlockManagerTestUtil { public static void setNodeReplicationLimit(final BlockManager blockManager, diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestAvailableSpaceBlockPlacementPolicy.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestAvailableSpaceBlockPlacementPolicy.java index f58961eb9d072..af7690d681e49 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestAvailableSpaceBlockPlacementPolicy.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestAvailableSpaceBlockPlacementPolicy.java @@ -173,6 +173,52 @@ public void testChooseDataNode() { } } + @Test + public void testChooseSimilarDataNode() { + DatanodeDescriptor[] tolerateDataNodes; + DatanodeStorageInfo[] tolerateStorages; + int capacity = 3; + Collection allTolerateNodes = new ArrayList<>(capacity); + String[] ownerRackOfTolerateNodes = new String[capacity]; + for (int i = 0; i < capacity; i++) { + ownerRackOfTolerateNodes[i] = "rack"+i; + } + tolerateStorages = DFSTestUtil.createDatanodeStorageInfos(ownerRackOfTolerateNodes); + tolerateDataNodes = DFSTestUtil.toDatanodeDescriptor(tolerateStorages); + + Collections.addAll(allTolerateNodes, tolerateDataNodes); + final BlockManager bm = namenode.getNamesystem().getBlockManager(); + AvailableSpaceBlockPlacementPolicy toleratePlacementPolicy = + (AvailableSpaceBlockPlacementPolicy)bm.getBlockPlacementPolicy(); + + updateHeartbeatWithUsage(tolerateDataNodes[0], + 20 * HdfsServerConstants.MIN_BLOCKS_FOR_WRITE * blockSize, + 1 * HdfsServerConstants.MIN_BLOCKS_FOR_WRITE * blockSize, + HdfsServerConstants.MIN_BLOCKS_FOR_WRITE + * blockSize, 0L, 0L, 0L, 0, 0); + + updateHeartbeatWithUsage(tolerateDataNodes[1], + 11 * HdfsServerConstants.MIN_BLOCKS_FOR_WRITE * blockSize, + 1 * HdfsServerConstants.MIN_BLOCKS_FOR_WRITE * blockSize, + HdfsServerConstants.MIN_BLOCKS_FOR_WRITE + * blockSize, 0L, 0L, 0L, 0, 0); + + updateHeartbeatWithUsage(tolerateDataNodes[2], + 10 * HdfsServerConstants.MIN_BLOCKS_FOR_WRITE * blockSize, + 1 * HdfsServerConstants.MIN_BLOCKS_FOR_WRITE * blockSize, + HdfsServerConstants.MIN_BLOCKS_FOR_WRITE + * blockSize, 0L, 0L, 0L, 0, 0); + + assertTrue(toleratePlacementPolicy.compareDataNode(tolerateDataNodes[0], + tolerateDataNodes[1], false) == 0); + assertTrue(toleratePlacementPolicy.compareDataNode(tolerateDataNodes[1], + tolerateDataNodes[0], false) == 0); + assertTrue(toleratePlacementPolicy.compareDataNode(tolerateDataNodes[0], + tolerateDataNodes[2], false) == -1); + assertTrue(toleratePlacementPolicy.compareDataNode(tolerateDataNodes[2], + tolerateDataNodes[0], false) == 1); + } + @AfterClass public static void teardownCluster() { if (namenode != null) { diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestAvailableSpaceRackFaultTolerantBPP.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestAvailableSpaceRackFaultTolerantBPP.java index 179c6c6b6931c..4d249c8ef67cd 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestAvailableSpaceRackFaultTolerantBPP.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestAvailableSpaceRackFaultTolerantBPP.java @@ -41,6 +41,7 @@ import java.util.HashSet; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; /** * Tests AvailableSpaceRackFaultTolerant block placement policy. @@ -206,6 +207,52 @@ public void testMaxRackAllocation() { assertEquals(REPLICA, racks.size()); } + @Test + public void testChooseSimilarDataNode() { + DatanodeDescriptor[] tolerateDataNodes; + DatanodeStorageInfo[] tolerateStorages; + int capacity = 3; + Collection allTolerateNodes = new ArrayList<>(capacity); + String[] ownerRackOfTolerateNodes = new String[capacity]; + for (int i = 0; i < capacity; i++) { + ownerRackOfTolerateNodes[i] = "rack"+i; + } + tolerateStorages = DFSTestUtil.createDatanodeStorageInfos(ownerRackOfTolerateNodes); + tolerateDataNodes = DFSTestUtil.toDatanodeDescriptor(tolerateStorages); + + Collections.addAll(allTolerateNodes, tolerateDataNodes); + final BlockManager bm = namenode.getNamesystem().getBlockManager(); + AvailableSpaceRackFaultTolerantBlockPlacementPolicy toleratePlacementPolicy = + (AvailableSpaceRackFaultTolerantBlockPlacementPolicy)bm.getBlockPlacementPolicy(); + + updateHeartbeatWithUsage(tolerateDataNodes[0], + 20 * HdfsServerConstants.MIN_BLOCKS_FOR_WRITE * BLOCK_SIZE, + 1 * HdfsServerConstants.MIN_BLOCKS_FOR_WRITE * BLOCK_SIZE, + HdfsServerConstants.MIN_BLOCKS_FOR_WRITE + * BLOCK_SIZE, 0L, 0L, 0L, 0, 0); + + updateHeartbeatWithUsage(tolerateDataNodes[1], + 11 * HdfsServerConstants.MIN_BLOCKS_FOR_WRITE * BLOCK_SIZE, + 1 * HdfsServerConstants.MIN_BLOCKS_FOR_WRITE * BLOCK_SIZE, + HdfsServerConstants.MIN_BLOCKS_FOR_WRITE + * BLOCK_SIZE, 0L, 0L, 0L, 0, 0); + + updateHeartbeatWithUsage(tolerateDataNodes[2], + 10 * HdfsServerConstants.MIN_BLOCKS_FOR_WRITE * BLOCK_SIZE, + 1 * HdfsServerConstants.MIN_BLOCKS_FOR_WRITE * BLOCK_SIZE, + HdfsServerConstants.MIN_BLOCKS_FOR_WRITE + * BLOCK_SIZE, 0L, 0L, 0L, 0, 0); + + assertTrue(toleratePlacementPolicy.compareDataNode(tolerateDataNodes[0], + tolerateDataNodes[1]) == 0); + assertTrue(toleratePlacementPolicy.compareDataNode(tolerateDataNodes[1], + tolerateDataNodes[0]) == 0); + assertTrue(toleratePlacementPolicy.compareDataNode(tolerateDataNodes[0], + tolerateDataNodes[2]) == -1); + assertTrue(toleratePlacementPolicy.compareDataNode(tolerateDataNodes[2], + tolerateDataNodes[0]) == 1); + } + @AfterClass public static void teardownCluster() { if (namenode != null) { diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/FsDatasetTestUtils.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/FsDatasetTestUtils.java index 987b74df47b8b..0f710f9a3c395 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/FsDatasetTestUtils.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/FsDatasetTestUtils.java @@ -17,7 +17,7 @@ */ package org.apache.hadoop.hdfs.server.datanode; -import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; +import org.apache.hadoop.util.Preconditions; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.conf.Configuration; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/InternalDataNodeTestUtils.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/InternalDataNodeTestUtils.java index 30fee2fddd99b..2b04e2707a034 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/InternalDataNodeTestUtils.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/InternalDataNodeTestUtils.java @@ -46,7 +46,7 @@ import org.mockito.invocation.InvocationOnMock; import org.mockito.stubbing.Answer; -import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; +import org.apache.hadoop.util.Preconditions; import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.doAnswer; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImplTestUtils.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImplTestUtils.java index c4280bc97b4ea..67176d8a1a3e6 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImplTestUtils.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImplTestUtils.java @@ -18,7 +18,7 @@ package org.apache.hadoop.hdfs.server.datanode.fsdataset.impl; -import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; +import org.apache.hadoop.util.Preconditions; import org.apache.commons.io.FileExistsException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/LazyPersistTestCase.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/LazyPersistTestCase.java index 9774fea1c9a50..eaaa7b0c89227 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/LazyPersistTestCase.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/LazyPersistTestCase.java @@ -46,7 +46,7 @@ import java.util.concurrent.TimeoutException; import org.apache.hadoop.io.IOUtils; -import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; +import org.apache.hadoop.util.Preconditions; import org.apache.hadoop.hdfs.protocol.DatanodeInfo; import org.apache.hadoop.hdfs.protocol.HdfsConstants; import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfo; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestScrLazyPersistFiles.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestScrLazyPersistFiles.java index efc203b73000b..f5e5be65607a3 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestScrLazyPersistFiles.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestScrLazyPersistFiles.java @@ -17,7 +17,7 @@ */ package org.apache.hadoop.hdfs.server.datanode.fsdataset.impl; -import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; +import org.apache.hadoop.util.Preconditions; import org.apache.hadoop.fs.ChecksumException; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hdfs.ClientContext; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/diskbalancer/DiskBalancerTestUtil.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/diskbalancer/DiskBalancerTestUtil.java index 2a496fb0ec222..aa9b6f1c19c5c 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/diskbalancer/DiskBalancerTestUtil.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/diskbalancer/DiskBalancerTestUtil.java @@ -17,7 +17,7 @@ package org.apache.hadoop.hdfs.server.diskbalancer; -import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; +import org.apache.hadoop.util.Preconditions; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/diskbalancer/TestDiskBalancer.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/diskbalancer/TestDiskBalancer.java index aa2c5a7c8e3f9..073bb532ddf6d 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/diskbalancer/TestDiskBalancer.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/diskbalancer/TestDiskBalancer.java @@ -16,7 +16,7 @@ */ package org.apache.hadoop.hdfs.server.diskbalancer; -import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; +import org.apache.hadoop.util.Preconditions; import java.util.function.Supplier; import org.apache.commons.codec.digest.DigestUtils; import org.apache.hadoop.conf.Configuration; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/diskbalancer/TestDiskBalancerWithMockMover.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/diskbalancer/TestDiskBalancerWithMockMover.java index e9c74aedd08b5..2fab1410d02df 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/diskbalancer/TestDiskBalancerWithMockMover.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/diskbalancer/TestDiskBalancerWithMockMover.java @@ -19,7 +19,7 @@ package org.apache.hadoop.hdfs.server.diskbalancer; -import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; +import org.apache.hadoop.util.Preconditions; import java.util.function.Supplier; import org.apache.commons.codec.digest.DigestUtils; import org.slf4j.Logger; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/diskbalancer/command/TestDiskBalancerCommand.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/diskbalancer/command/TestDiskBalancerCommand.java index 1d6331d83c558..d0107b507ade9 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/diskbalancer/command/TestDiskBalancerCommand.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/diskbalancer/command/TestDiskBalancerCommand.java @@ -158,7 +158,7 @@ public void testSubmitPlanInNonRegularStatus() throws Exception { * Tests running multiple commands under on setup. This mainly covers * {@link org.apache.hadoop.hdfs.server.diskbalancer.command.Command#close} */ - @Test(timeout = 60000) + @Test(timeout = 120000) public void testRunMultipleCommandsUnderOneSetup() throws Exception { final int numDatanodes = 1; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/mover/TestStorageMover.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/mover/TestStorageMover.java index d95e76fc85294..94f7ef69c9bb3 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/mover/TestStorageMover.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/mover/TestStorageMover.java @@ -65,7 +65,7 @@ import org.junit.Assert; import org.junit.Test; -import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; +import org.apache.hadoop.util.Preconditions; import org.apache.hadoop.thirdparty.com.google.common.collect.Maps; /** diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/NNThroughputBenchmark.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/NNThroughputBenchmark.java index 7f6d572fceb39..62f1a507fd508 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/NNThroughputBenchmark.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/NNThroughputBenchmark.java @@ -27,7 +27,7 @@ import java.util.EnumSet; import java.util.List; -import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; +import org.apache.hadoop.util.Preconditions; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -1219,10 +1219,28 @@ void generateInputs(int[] ignore) throws IOException { private ExtendedBlock addBlocks(String fileName, String clientName) throws IOException { + DatanodeInfo[] excludeNodes = null; + DatanodeInfo[] dnInfos = clientProto.getDatanodeReport( + HdfsConstants.DatanodeReportType.LIVE); + if (dnInfos != null && dnInfos.length > 0) { + List tmpNodes = new ArrayList<>(); + String localHost = DNS.getDefaultHost("default", "default"); + for (DatanodeInfo dnInfo : dnInfos) { + if (!localHost.equals(dnInfo.getHostName()) || + (dnInfo.getXferPort() > datanodes.length)) { + tmpNodes.add(dnInfo); + } + } + + if (tmpNodes.size() > 0) { + excludeNodes = tmpNodes.toArray(new DatanodeInfo[tmpNodes.size()]); + } + } + ExtendedBlock prevBlock = null; for(int jdx = 0; jdx < blocksPerFile; jdx++) { LocatedBlock loc = addBlock(fileName, clientName, - prevBlock, null, HdfsConstants.GRANDFATHER_INODE_ID, null); + prevBlock, excludeNodes, HdfsConstants.GRANDFATHER_INODE_ID, null); prevBlock = loc.getBlock(); for(DatanodeInfo dnInfo : loc.getLocations()) { int dnIdx = dnInfo.getXferPort() - 1; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAuditLogger.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAuditLogger.java index aa2c7f689766f..c7e47561dcc7e 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAuditLogger.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAuditLogger.java @@ -55,6 +55,7 @@ import java.net.URI; import java.net.URISyntaxException; import java.util.List; +import java.util.regex.Pattern; import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_CALLER_CONTEXT_ENABLED_KEY; import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_CALLER_CONTEXT_MAX_SIZE_KEY; @@ -69,6 +70,7 @@ import static org.apache.hadoop.fs.permission.FsAction.READ_EXECUTE; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_ACLS_ENABLED_KEY; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_AUDIT_LOGGERS_KEY; +import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_AUDIT_LOG_WITH_REMOTE_PORT_KEY; import static org.apache.hadoop.hdfs.DFSConfigKeys.NNTOP_ENABLED_KEY; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; @@ -89,6 +91,20 @@ public class TestAuditLogger { } private static final short TEST_PERMISSION = (short) 0654; + private static final Pattern AUDIT_PATTERN = Pattern.compile( + ".*allowed=.*?\\s" + + "ugi=.*?\\s" + + "ip=/\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}\\s" + + "cmd=.*?\\ssrc=.*?\\sdst=null\\s" + + "perm=.*?"); + private static final Pattern AUDIT_WITH_PORT_PATTERN = Pattern.compile( + ".*allowed=.*?\\s" + + "ugi=.*?\\s" + + "ip=/\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}\\s" + + "cmd=.*?\\ssrc=.*?\\sdst=null\\s" + + "perm=.*?" + + "proto=.*?" + + "callerContext=.*?clientPort\\:(\\d{0,9}).*?"); @Before public void setup() { @@ -544,6 +560,45 @@ public void testBrokenLogger() throws IOException { } } + /** + * Test adding remote port to audit log. + */ + @Test + public void testAuditLogWithRemotePort() throws Exception { + // Audit log without remote port by default. + Configuration conf = new HdfsConfiguration(); + MiniDFSCluster cluster1 = new MiniDFSCluster.Builder(conf).build(); + try { + LogCapturer auditLog = LogCapturer.captureLogs(FSNamesystem.auditLog); + cluster1.waitClusterUp(); + FileSystem fs = cluster1.getFileSystem(); + long time = System.currentTimeMillis(); + fs.setTimes(new Path("/"), time, time); + assertTrue(AUDIT_PATTERN.matcher(auditLog.getOutput().trim()).matches()); + assertFalse(auditLog.getOutput().contains("clientPort")); + auditLog.clearOutput(); + } finally { + cluster1.shutdown(); + } + + // Audit log with remote port. + conf.setBoolean(DFS_NAMENODE_AUDIT_LOG_WITH_REMOTE_PORT_KEY, true); + conf.setBoolean(HADOOP_CALLER_CONTEXT_ENABLED_KEY, true); + MiniDFSCluster cluster2 = new MiniDFSCluster.Builder(conf).build(); + try { + LogCapturer auditLog = LogCapturer.captureLogs(FSNamesystem.auditLog); + cluster2.waitClusterUp(); + FileSystem fs = cluster2.getFileSystem(); + long time = System.currentTimeMillis(); + fs.setTimes(new Path("/"), time, time); + assertTrue(AUDIT_WITH_PORT_PATTERN.matcher( + auditLog.getOutput().trim()).matches()); + auditLog.clearOutput(); + } finally { + cluster2.shutdown(); + } + } + public static class DummyAuditLogger implements AuditLogger { static boolean initialized; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAuditLogs.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAuditLogs.java index 9fe7404eed8eb..2832782caf918 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAuditLogs.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAuditLogs.java @@ -92,9 +92,9 @@ public TestAuditLogs(boolean useAsyncLog, boolean useAsyncEdits) { // allowed=(true|false) ugi=name ip=/address cmd={cmd} src={path} dst=null perm=null static final Pattern auditPattern = Pattern.compile( "allowed=.*?\\s" + - "ugi=.*?\\s" + - "ip=/\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}\\s" + - "cmd=.*?\\ssrc=.*?\\sdst=null\\s" + + "ugi=.*?\\s" + + "ip=/\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}\\s" + + "cmd=.*?\\ssrc=.*?\\sdst=null\\s" + "perm=.*?"); static final Pattern successPattern = Pattern.compile( ".*allowed=true.*"); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFsck.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFsck.java index 049a785481616..ae1021da0db4d 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFsck.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFsck.java @@ -144,14 +144,14 @@ public class TestFsck { // allowed=true ugi=name ip=/address cmd=FSCK src=/ dst=null perm=null static final Pattern FSCK_PATTERN = Pattern.compile( "allowed=.*?\\s" + - "ugi=.*?\\s" + - "ip=/\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}\\s" + + "ugi=.*?\\s" + + "ip=/\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}\\s" + "cmd=fsck\\ssrc=\\/\\sdst=null\\s" + "perm=null\\s" + "proto=.*"); static final Pattern GET_FILE_INFO_PATTERN = Pattern.compile( "allowed=.*?\\s" + - "ugi=.*?\\s" + - "ip=/\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}\\s" + + "ugi=.*?\\s" + + "ip=/\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}\\s" + "cmd=getfileinfo\\ssrc=\\/\\sdst=null\\s" + "perm=null\\s" + "proto=.*"); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestINodeAttributeProvider.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestINodeAttributeProvider.java index 776a1981ce4e5..700b32f2895c6 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestINodeAttributeProvider.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestINodeAttributeProvider.java @@ -48,6 +48,8 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import static org.junit.Assert.fail; + public class TestINodeAttributeProvider { private static final Logger LOG = LoggerFactory.getLogger(TestINodeAttributeProvider.class); @@ -57,6 +59,15 @@ public class TestINodeAttributeProvider { private static final short HDFS_PERMISSION = 0777; private static final short PROVIDER_PERMISSION = 0770; private static boolean runPermissionCheck = false; + private static boolean shouldThrowAccessException = false; + + public static class MyAuthorizationProviderAccessException + extends AccessControlException { + + public MyAuthorizationProviderAccessException() { + super(); + } + }; public static class MyAuthorizationProvider extends INodeAttributeProvider { @@ -82,6 +93,9 @@ public void checkPermission(String fsOwner, String supergroup, ancestorAccess, parentAccess, access, subAccess, ignoreEmptyDir); } CALLED.add("checkPermission|" + ancestorAccess + "|" + parentAccess + "|" + access); + if (shouldThrowAccessException) { + throw new MyAuthorizationProviderAccessException(); + } } @Override @@ -96,6 +110,9 @@ public void checkPermissionWithContext( CALLED.add("checkPermission|" + authzContext.getAncestorAccess() + "|" + authzContext.getParentAccess() + "|" + authzContext .getAccess()); + if (shouldThrowAccessException) { + throw new MyAuthorizationProviderAccessException(); + } } } @@ -238,6 +255,7 @@ public void cleanUp() throws IOException { miniDFS = null; } runPermissionCheck = false; + shouldThrowAccessException = false; Assert.assertTrue(CALLED.contains("stop")); } @@ -457,6 +475,44 @@ public Void run() throws Exception { }); } + @Test + // HDFS-16529 - Ensure enforcer AccessControlException subclass are caught + // and re-thrown as plain ACE exceptions. + public void testSubClassedAccessControlExceptions() throws Exception { + FileSystem fs = FileSystem.get(miniDFS.getConfiguration(0)); + shouldThrowAccessException = true; + final Path userPath = new Path("/user"); + final Path authz = new Path("/user/authz"); + final Path authzChild = new Path("/user/authz/child2"); + + fs.mkdirs(userPath); + fs.setPermission(userPath, new FsPermission(HDFS_PERMISSION)); + fs.mkdirs(authz); + fs.setPermission(authz, new FsPermission(HDFS_PERMISSION)); + fs.mkdirs(authzChild); + fs.setPermission(authzChild, new FsPermission(HDFS_PERMISSION)); + UserGroupInformation ugi = UserGroupInformation.createUserForTesting("u1", + new String[]{"g1"}); + ugi.doAs(new PrivilegedExceptionAction() { + @Override + public Void run() throws Exception { + FileSystem fs = FileSystem.get(miniDFS.getConfiguration(0)); + try { + fs.access(authzChild, FsAction.ALL); + fail("Exception should be thrown"); + // The DFS Client will get a RemoteException containing an + // AccessControlException (ACE). If the ACE is a subclass of ACE then + // the client does not unwrap it correctly. The change in HDFS-16529 + // is to ensure ACE is always thrown rather than a sub class to avoid + // this issue. + } catch (AccessControlException ace) { + Assert.assertEquals(AccessControlException.class, ace.getClass()); + } + return null; + } + }); + } + @Test // HDFS-15165 - ContentSummary calls should use the provider permissions(if diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNNThroughputBenchmark.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNNThroughputBenchmark.java index 44bf5b78784e0..f7a8d92864fa0 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNNThroughputBenchmark.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNNThroughputBenchmark.java @@ -166,4 +166,25 @@ public void testNNThroughputForAppendOp() throws Exception { } } } + + /** + * This test runs {@link NNThroughputBenchmark} against a mini DFS cluster + * for block report operation. + */ + @Test(timeout = 120000) + public void testNNThroughputForBlockReportOp() throws Exception { + final Configuration conf = new HdfsConfiguration(); + conf.setInt(DFSConfigKeys.DFS_NAMENODE_MIN_BLOCK_SIZE_KEY, 16); + conf.setInt(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, 16); + try (MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf). + numDataNodes(3).build()) { + cluster.waitActive(); + final Configuration benchConf = new HdfsConfiguration(); + benchConf.setInt(DFSConfigKeys.DFS_NAMENODE_MIN_BLOCK_SIZE_KEY, 16); + benchConf.setInt(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, 16); + NNThroughputBenchmark.runBenchmark(benchConf, + new String[]{"-fs", cluster.getURI().toString(), "-op", + "blockReport", "-datanodes", "3", "-reports", "2"}); + } + } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNamenodeStorageDirectives.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNamenodeStorageDirectives.java index 5bcb3a571a75c..0469b91988451 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNamenodeStorageDirectives.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNamenodeStorageDirectives.java @@ -192,7 +192,7 @@ private void testStorageTypes(StorageType[][] storageTypes, * Types. * @throws IOException */ - @Test(timeout=60000) + @Test(timeout=120000) public void testTargetStorageTypes() throws ReconfigurationException, InterruptedException, TimeoutException, IOException { // DISK and not anything else. diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/shortcircuit/TestShortCircuitCache.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/shortcircuit/TestShortCircuitCache.java index aac1bf5059bc3..9754da3348359 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/shortcircuit/TestShortCircuitCache.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/shortcircuit/TestShortCircuitCache.java @@ -89,7 +89,7 @@ import org.mockito.invocation.InvocationOnMock; import org.mockito.stubbing.Answer; -import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; +import org.apache.hadoop.util.Preconditions; import java.util.function.Supplier; import org.apache.hadoop.thirdparty.com.google.common.collect.HashMultimap; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestDebugAdmin.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestDebugAdmin.java index 5c890a5481415..8dd303d84db02 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestDebugAdmin.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestDebugAdmin.java @@ -17,15 +17,22 @@ */ package org.apache.hadoop.hdfs.tools; +import org.apache.commons.io.FileUtils; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.permission.FsPermission; import org.apache.hadoop.hdfs.DFSTestUtil; import org.apache.hadoop.hdfs.DistributedFileSystem; import org.apache.hadoop.hdfs.MiniDFSCluster; +import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy; import org.apache.hadoop.hdfs.protocol.ExtendedBlock; +import org.apache.hadoop.hdfs.protocol.LocatedBlock; +import org.apache.hadoop.hdfs.protocol.LocatedStripedBlock; +import org.apache.hadoop.hdfs.protocol.SystemErasureCodingPolicies; import org.apache.hadoop.hdfs.server.datanode.DataNode; import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsDatasetSpi; +import org.apache.hadoop.hdfs.util.StripedBlockUtil; import org.apache.hadoop.io.IOUtils; import org.junit.After; import org.junit.Before; @@ -34,6 +41,8 @@ import java.io.ByteArrayOutputStream; import java.io.File; import java.io.PrintStream; +import java.util.List; +import java.util.Random; import static org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetTestUtil.*; import static org.junit.Assert.assertEquals; @@ -44,23 +53,16 @@ public class TestDebugAdmin { static private final String TEST_ROOT_DIR = new File(System.getProperty("test.build.data", "/tmp"), TestDebugAdmin.class.getSimpleName()).getAbsolutePath(); - + private Configuration conf = new Configuration(); private MiniDFSCluster cluster; - private DistributedFileSystem fs; private DebugAdmin admin; - private DataNode datanode; @Before public void setUp() throws Exception { final File testRoot = new File(TEST_ROOT_DIR); testRoot.delete(); testRoot.mkdirs(); - Configuration conf = new Configuration(); - cluster = new MiniDFSCluster.Builder(conf).numDataNodes(1).build(); - cluster.waitActive(); - fs = cluster.getFileSystem(); admin = new DebugAdmin(conf); - datanode = cluster.getDataNodes().get(0); } @After @@ -92,8 +94,11 @@ private String runCmd(String[] cmd) throws Exception { @Test(timeout = 60000) public void testRecoverLease() throws Exception { + cluster = new MiniDFSCluster.Builder(conf).numDataNodes(1).build(); + cluster.waitActive(); assertEquals("ret: 1, You must supply a -path argument to recoverLease.", runCmd(new String[]{"recoverLease", "-retries", "1"})); + DistributedFileSystem fs = cluster.getFileSystem(); FSDataOutputStream out = fs.create(new Path("/foo")); out.write(123); out.close(); @@ -103,6 +108,10 @@ public void testRecoverLease() throws Exception { @Test(timeout = 60000) public void testVerifyMetaCommand() throws Exception { + cluster = new MiniDFSCluster.Builder(conf).numDataNodes(1).build(); + cluster.waitActive(); + DistributedFileSystem fs = cluster.getFileSystem(); + DataNode datanode = cluster.getDataNodes().get(0); DFSTestUtil.createFile(fs, new Path("/bar"), 1234, (short) 1, 0xdeadbeef); FsDatasetSpi fsd = datanode.getFSDataset(); ExtendedBlock block = DFSTestUtil.getFirstBlock(fs, new Path("/bar")); @@ -128,6 +137,10 @@ public void testVerifyMetaCommand() throws Exception { @Test(timeout = 60000) public void testComputeMetaCommand() throws Exception { + cluster = new MiniDFSCluster.Builder(conf).numDataNodes(1).build(); + cluster.waitActive(); + DistributedFileSystem fs = cluster.getFileSystem(); + DataNode datanode = cluster.getDataNodes().get(0); DFSTestUtil.createFile(fs, new Path("/bar"), 1234, (short) 1, 0xdeadbeef); FsDatasetSpi fsd = datanode.getFSDataset(); ExtendedBlock block = DFSTestUtil.getFirstBlock(fs, new Path("/bar")); @@ -166,8 +179,97 @@ public void testComputeMetaCommand() throws Exception { @Test(timeout = 60000) public void testRecoverLeaseforFileNotFound() throws Exception { + cluster = new MiniDFSCluster.Builder(conf).numDataNodes(1).build(); + cluster.waitActive(); assertTrue(runCmd(new String[] { "recoverLease", "-path", "/foo", "-retries", "2" }).contains( "Giving up on recoverLease for /foo after 1 try")); } + + @Test(timeout = 60000) + public void testVerifyECCommand() throws Exception { + final ErasureCodingPolicy ecPolicy = SystemErasureCodingPolicies.getByID( + SystemErasureCodingPolicies.RS_3_2_POLICY_ID); + cluster = DFSTestUtil.setupCluster(conf, 6, 5, 0); + cluster.waitActive(); + DistributedFileSystem fs = cluster.getFileSystem(); + + assertEquals("ret: 1, verifyEC -file Verify HDFS erasure coding on " + + "all block groups of the file.", runCmd(new String[]{"verifyEC"})); + + assertEquals("ret: 1, File /bar does not exist.", + runCmd(new String[]{"verifyEC", "-file", "/bar"})); + + fs.create(new Path("/bar")).close(); + assertEquals("ret: 1, File /bar is not erasure coded.", + runCmd(new String[]{"verifyEC", "-file", "/bar"})); + + + final Path ecDir = new Path("/ec"); + fs.mkdir(ecDir, FsPermission.getDirDefault()); + fs.enableErasureCodingPolicy(ecPolicy.getName()); + fs.setErasureCodingPolicy(ecDir, ecPolicy.getName()); + + assertEquals("ret: 1, File /ec is not a regular file.", + runCmd(new String[]{"verifyEC", "-file", "/ec"})); + + fs.create(new Path(ecDir, "foo")); + assertEquals("ret: 1, File /ec/foo is not closed.", + runCmd(new String[]{"verifyEC", "-file", "/ec/foo"})); + + final short repl = 1; + final long k = 1024; + final long m = k * k; + final long seed = 0x1234567L; + DFSTestUtil.createFile(fs, new Path(ecDir, "foo_65535"), 65535, repl, seed); + assertTrue(runCmd(new String[]{"verifyEC", "-file", "/ec/foo_65535"}) + .contains("All EC block group status: OK")); + DFSTestUtil.createFile(fs, new Path(ecDir, "foo_256k"), 256 * k, repl, seed); + assertTrue(runCmd(new String[]{"verifyEC", "-file", "/ec/foo_256k"}) + .contains("All EC block group status: OK")); + DFSTestUtil.createFile(fs, new Path(ecDir, "foo_1m"), m, repl, seed); + assertTrue(runCmd(new String[]{"verifyEC", "-file", "/ec/foo_1m"}) + .contains("All EC block group status: OK")); + DFSTestUtil.createFile(fs, new Path(ecDir, "foo_2m"), 2 * m, repl, seed); + assertTrue(runCmd(new String[]{"verifyEC", "-file", "/ec/foo_2m"}) + .contains("All EC block group status: OK")); + DFSTestUtil.createFile(fs, new Path(ecDir, "foo_3m"), 3 * m, repl, seed); + assertTrue(runCmd(new String[]{"verifyEC", "-file", "/ec/foo_3m"}) + .contains("All EC block group status: OK")); + DFSTestUtil.createFile(fs, new Path(ecDir, "foo_5m"), 5 * m, repl, seed); + assertTrue(runCmd(new String[]{"verifyEC", "-file", "/ec/foo_5m"}) + .contains("All EC block group status: OK")); + DFSTestUtil.createFile(fs, new Path(ecDir, "foo_6m"), (int) k, 6 * m, m, repl, seed); + assertEquals("ret: 0, Checking EC block group: blk_x;Status: OK" + + "Checking EC block group: blk_x;Status: OK" + + "All EC block group status: OK", + runCmd(new String[]{"verifyEC", "-file", "/ec/foo_6m"}) + .replaceAll("blk_-[0-9]+", "blk_x;")); + + Path corruptFile = new Path(ecDir, "foo_corrupt"); + DFSTestUtil.createFile(fs, corruptFile, 5841961, repl, seed); + List blocks = DFSTestUtil.getAllBlocks(fs, corruptFile); + assertEquals(1, blocks.size()); + LocatedStripedBlock blockGroup = (LocatedStripedBlock) blocks.get(0); + LocatedBlock[] indexedBlocks = StripedBlockUtil.parseStripedBlockGroup(blockGroup, + ecPolicy.getCellSize(), ecPolicy.getNumDataUnits(), ecPolicy.getNumParityUnits()); + // Try corrupt block 0 in block group. + LocatedBlock toCorruptLocatedBlock = indexedBlocks[0]; + ExtendedBlock toCorruptBlock = toCorruptLocatedBlock.getBlock(); + DataNode datanode = cluster.getDataNode(toCorruptLocatedBlock.getLocations()[0].getIpcPort()); + File blockFile = getBlockFile(datanode.getFSDataset(), + toCorruptBlock.getBlockPoolId(), toCorruptBlock.getLocalBlock()); + File metaFile = getMetaFile(datanode.getFSDataset(), + toCorruptBlock.getBlockPoolId(), toCorruptBlock.getLocalBlock()); + // Write error bytes to block file and re-generate meta checksum. + byte[] errorBytes = new byte[2097152]; + new Random(seed).nextBytes(errorBytes); + FileUtils.writeByteArrayToFile(blockFile, errorBytes); + metaFile.delete(); + runCmd(new String[]{"computeMeta", "-block", blockFile.getAbsolutePath(), + "-out", metaFile.getAbsolutePath()}); + assertTrue(runCmd(new String[]{"verifyEC", "-file", "/ec/foo_corrupt"}) + .contains("Status: ERROR, message: EC compute result not match.")); + } + } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineEditsViewer/TestOfflineEditsViewer.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineEditsViewer/TestOfflineEditsViewer.java index 226e4861f45f5..6f30416ed60e7 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineEditsViewer/TestOfflineEditsViewer.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineEditsViewer/TestOfflineEditsViewer.java @@ -164,7 +164,7 @@ public void testRecoveryMode() throws IOException { public void testStored() throws IOException { // reference edits stored with source code (see build.xml) final String cacheDir = System.getProperty("test.cache.data", - "build/test/cache"); + "target/test-classes"); // binary, XML, reparsed binary String editsStored = cacheDir + "/editsStored"; String editsStoredParsedXml = cacheDir + "/editsStoredParsed.xml"; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/util/TestStripedBlockUtil.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/util/TestStripedBlockUtil.java index 9d8c82c63708a..d1922dff8216e 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/util/TestStripedBlockUtil.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/util/TestStripedBlockUtil.java @@ -18,7 +18,7 @@ package org.apache.hadoop.hdfs.util; -import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; +import org.apache.hadoop.util.Preconditions; import org.apache.hadoop.fs.StorageType; import org.apache.hadoop.hdfs.DFSTestUtil; import org.apache.hadoop.hdfs.StripedFileTestUtil; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestJsonUtil.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestJsonUtil.java index 839f8946be06b..d27aecb54c083 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestJsonUtil.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestJsonUtil.java @@ -21,8 +21,6 @@ import static org.apache.hadoop.fs.permission.AclEntryType.*; import static org.apache.hadoop.fs.permission.FsAction.*; import static org.apache.hadoop.hdfs.server.namenode.AclTestHelpers.*; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; import java.io.IOException; import java.util.EnumSet; @@ -48,8 +46,11 @@ import org.apache.hadoop.hdfs.protocol.HdfsConstants; import org.apache.hadoop.hdfs.protocol.HdfsFileStatus; import org.apache.hadoop.hdfs.protocol.HdfsFileStatus.Flags; +import org.apache.hadoop.hdfs.protocol.SnapshotDiffReportListing; +import org.apache.hadoop.hdfs.protocol.SnapshotDiffReportListing.DiffReportListingEntry; import org.apache.hadoop.io.erasurecode.ECSchema; import org.apache.hadoop.test.LambdaTestUtils; +import org.apache.hadoop.util.ChunkedArrayList; import org.apache.hadoop.util.Lists; import org.apache.hadoop.util.Time; @@ -128,7 +129,7 @@ public void testHdfsFileStatus() throws Exception { .isdir(true) .build(); - assertFalse(hdfsFileStatus.isSymlink()); + Assert.assertFalse(hdfsFileStatus.isSymlink()); LambdaTestUtils.intercept(IOException.class, "Path " + hdfsFileStatus.getPath() + " is not a symbolic link", () -> hdfsFileStatus.getSymlink()); @@ -150,7 +151,7 @@ public void testHdfsFileStatus() throws Exception { .append("}") .toString(); - assertEquals(expectString, hdfsFileStatus.toString()); + Assert.assertEquals(expectString, hdfsFileStatus.toString()); } @Test @@ -391,6 +392,83 @@ public void testGetXAttrFromJson() throws IOException { Assert.assertArrayEquals(XAttrCodec.decodeValue("0x313131"), value); } + @Test + public void testSnapshotDiffReportListingEmptyReport() throws IOException { + SnapshotDiffReportListing report = new SnapshotDiffReportListing(); + String jsonString = JsonUtil.toJsonString(report); + Map json = READER.readValue(jsonString); + SnapshotDiffReportListing parsed = + JsonUtilClient.toSnapshotDiffReportListing(json); + + assertEquals(report, parsed); + } + + @Test + public void testSnapshotDiffReportListing() throws IOException { + List mlist = new ChunkedArrayList<>(); + List clist = new ChunkedArrayList<>(); + List dlist = new ChunkedArrayList<>(); + clist.add(new DiffReportListingEntry( + 1L, 2L, DFSUtilClient.string2Bytes("dir1/file2"), false, null)); + clist.add(new DiffReportListingEntry( + 1L, 3L, DFSUtilClient.string2Bytes("dir1/file3"), false, null)); + dlist.add(new DiffReportListingEntry( + 1L, 4L, DFSUtilClient.string2Bytes("dir1/file4"), false, null)); + dlist.add(new DiffReportListingEntry( + 1L, 5L, + DFSUtilClient.string2Bytes("dir1/file5"), + true, + DFSUtilClient.string2Bytes("dir1/file6"))); + + SnapshotDiffReportListing report = + new SnapshotDiffReportListing( + DFSUtilClient.string2Bytes("dir1/file2"), mlist, clist, dlist, 3, true); + String jsonString = JsonUtil.toJsonString(report); + Map json = READER.readValue(jsonString); + SnapshotDiffReportListing parsed = + JsonUtilClient.toSnapshotDiffReportListing(json); + + assertEquals(report, parsed); + } + + private void assertEquals( + SnapshotDiffReportListing expected, SnapshotDiffReportListing actual) { + Assert.assertEquals(expected.getLastIndex(), actual.getLastIndex()); + Assert.assertEquals(expected.getIsFromEarlier(), actual.getIsFromEarlier()); + assertEquals(expected.getModifyList(), actual.getModifyList()); + assertEquals(expected.getCreateList(), actual.getCreateList()); + assertEquals(expected.getDeleteList(), actual.getDeleteList()); + Assert.assertArrayEquals(expected.getLastPath(), actual.getLastPath()); + } + + private void assertEquals( + List expected, List actual) { + Assert.assertEquals(expected.size(), actual.size()); + + for (int i = 0; i < expected.size(); i++) { + DiffReportListingEntry a = expected.get(i); + DiffReportListingEntry b = actual.get(i); + + Assert.assertEquals(a.getFileId(), b.getFileId()); + Assert.assertEquals(a.getDirId(), b.getDirId()); + Assert.assertEquals(a.isReference(), b.isReference()); + if (a.getSourcePath() != null) { + Assert.assertArrayEquals( + DFSUtilClient.byteArray2bytes(a.getSourcePath()), + DFSUtilClient.byteArray2bytes(b.getSourcePath())); + } else { + Assert.assertArrayEquals(a.getSourcePath(), b.getSourcePath()); + } + if (a.getTargetPath() != null) { + Assert.assertArrayEquals( + DFSUtilClient.byteArray2bytes(a.getTargetPath()), + DFSUtilClient.byteArray2bytes(b.getTargetPath())); + } else { + Assert.assertArrayEquals(a.getTargetPath(), b.getTargetPath()); + } + } + } + private void checkDecodeFailure(Map map) { try { JsonUtilClient.toDatanodeInfo(map); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHDFS.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHDFS.java index f43e541aca44a..2461d22dde81c 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHDFS.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHDFS.java @@ -23,6 +23,7 @@ import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_BLOCK_SIZE_KEY; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CHECKSUM_TYPE_KEY; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_ENCRYPT_DATA_TRANSFER_KEY; +import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_SNAPSHOT_DIFF_LISTING_LIMIT; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_REPLICATION_KEY; import static org.apache.hadoop.hdfs.TestDistributedFileSystem.checkOpStatistics; import static org.apache.hadoop.hdfs.TestDistributedFileSystem.checkStatistics; @@ -723,6 +724,7 @@ public void testWebHdfsDeleteSnapshot() throws Exception { @Test public void testWebHdfsSnapshotDiff() throws Exception { final Configuration conf = WebHdfsTestUtil.createConf(); + conf.setInt(DFS_NAMENODE_SNAPSHOT_DIFF_LISTING_LIMIT, 2); cluster = new MiniDFSCluster.Builder(conf).numDataNodes(1).build(); cluster.waitActive(); final DistributedFileSystem dfs = cluster.getFileSystem(); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHdfsFileSystemContract.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHdfsFileSystemContract.java index 722ebad72d239..6821d9b431f87 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHdfsFileSystemContract.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHdfsFileSystemContract.java @@ -396,7 +396,7 @@ public void testOffsetPlusLengthParamsLongerThanFile() throws IOException { } } - @Test + @Test(timeout = 60000) public void testResponseCode() throws IOException { final WebHdfsFileSystem webhdfs = (WebHdfsFileSystem)fs; final Path root = new Path("/"); diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/pom.xml b/hadoop-mapreduce-project/hadoop-mapreduce-client/pom.xml index 86994dffc5b64..55940ae4ff744 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/pom.xml +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/pom.xml @@ -177,38 +177,6 @@ - - org.apache.maven.plugins - maven-enforcer-plugin - - - de.skuzzle.enforcer - restrict-imports-enforcer-rule - ${restrict-imports.enforcer.version} - - - - - banned-illegal-imports - process-sources - - enforce - - - - - true - Use hadoop-common provided VisibleForTesting rather than the one provided by Guava - - org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting - com.google.common.annotations.VisibleForTesting - - - - - - - diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-examples/pom.xml b/hadoop-mapreduce-project/hadoop-mapreduce-examples/pom.xml index c4a804fabe4b7..48cf27efe437f 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-examples/pom.xml +++ b/hadoop-mapreduce-project/hadoop-mapreduce-examples/pom.xml @@ -155,38 +155,6 @@ - - org.apache.maven.plugins - maven-enforcer-plugin - - - de.skuzzle.enforcer - restrict-imports-enforcer-rule - ${restrict-imports.enforcer.version} - - - - - banned-illegal-imports - process-sources - - enforce - - - - - true - Use hadoop-common provided VisibleForTesting rather than the one provided by Guava - - org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting - com.google.common.annotations.VisibleForTesting - - - - - - - diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml index 8de03f51c92b6..9392a9f67fb3e 100644 --- a/hadoop-project/pom.xml +++ b/hadoop-project/pom.xml @@ -119,7 +119,7 @@ 1.21 1.0 2.8.0 - 3.7 + 3.12.0 1.1.3 1.1 3.1.1 diff --git a/hadoop-tools/hadoop-aliyun/pom.xml b/hadoop-tools/hadoop-aliyun/pom.xml index 8a68d9b4f7db6..c5da3e513949f 100644 --- a/hadoop-tools/hadoop-aliyun/pom.xml +++ b/hadoop-tools/hadoop-aliyun/pom.xml @@ -107,6 +107,12 @@ compile + + org.assertj + assertj-core + test + + org.apache.hadoop hadoop-common diff --git a/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSFileSystem.java b/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSFileSystem.java index 759484e4239fa..5f40488bfd684 100644 --- a/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSFileSystem.java +++ b/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSFileSystem.java @@ -49,7 +49,6 @@ import org.apache.hadoop.util.Progressable; import com.aliyun.oss.model.OSSObjectSummary; -import com.aliyun.oss.model.ObjectListing; import com.aliyun.oss.model.ObjectMetadata; import org.apache.hadoop.util.SemaphoredDelegatingExecutor; @@ -271,14 +270,15 @@ public FileStatus getFileStatus(Path path) throws IOException { meta = store.getObjectMetadata(key); } if (meta == null) { - ObjectListing listing = store.listObjects(key, 1, null, false); + OSSListRequest listRequest = store.createListObjectsRequest(key, + maxKeys, null, null, false); + OSSListResult listing = store.listObjects(listRequest); do { if (CollectionUtils.isNotEmpty(listing.getObjectSummaries()) || CollectionUtils.isNotEmpty(listing.getCommonPrefixes())) { return new OSSFileStatus(0, true, 1, 0, 0, qualifiedPath, username); } else if (listing.isTruncated()) { - listing = store.listObjects(key, 1000, listing.getNextMarker(), - false); + listing = store.continueListObjects(listRequest, listing); } else { throw new FileNotFoundException( path + ": No such file or directory!"); @@ -416,7 +416,9 @@ public FileStatus[] listStatus(Path path) throws IOException { LOG.debug("listStatus: doing listObjects for directory " + key); } - ObjectListing objects = store.listObjects(key, maxKeys, null, false); + OSSListRequest listRequest = store.createListObjectsRequest(key, + maxKeys, null, null, false); + OSSListResult objects = store.listObjects(listRequest); while (true) { for (OSSObjectSummary objectSummary : objects.getObjectSummaries()) { String objKey = objectSummary.getKey(); @@ -456,8 +458,7 @@ public FileStatus[] listStatus(Path path) throws IOException { if (LOG.isDebugEnabled()) { LOG.debug("listStatus: list truncated - getting next batch"); } - String nextMarker = objects.getNextMarker(); - objects = store.listObjects(key, maxKeys, nextMarker, false); + objects = store.continueListObjects(listRequest, objects); } else { break; } @@ -520,7 +521,7 @@ private RemoteIterator innerList(final Path f, locations); } else { return store.createLocatedFileStatusIterator(key, maxKeys, this, filter, - acceptor, recursive ? null : "/"); + acceptor, recursive); } } @@ -707,7 +708,9 @@ private boolean copyDirectory(Path srcPath, Path dstPath) throws IOException { ExecutorService executorService = MoreExecutors.listeningDecorator( new SemaphoredDelegatingExecutor(boundedCopyThreadPool, maxConcurrentCopyTasksPerDir, true)); - ObjectListing objects = store.listObjects(srcKey, maxKeys, null, true); + OSSListRequest listRequest = store.createListObjectsRequest(srcKey, + maxKeys, null, null, true); + OSSListResult objects = store.listObjects(listRequest); // Copy files from src folder to dst int copiesToFinish = 0; while (true) { @@ -729,8 +732,7 @@ private boolean copyDirectory(Path srcPath, Path dstPath) throws IOException { } } if (objects.isTruncated()) { - String nextMarker = objects.getNextMarker(); - objects = store.listObjects(srcKey, maxKeys, nextMarker, true); + objects = store.continueListObjects(listRequest, objects); } else { break; } diff --git a/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSFileSystemStore.java b/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSFileSystemStore.java index 3df9a730161d3..e9ac1ddea9ee0 100644 --- a/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSFileSystemStore.java +++ b/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSFileSystemStore.java @@ -36,8 +36,8 @@ import com.aliyun.oss.model.InitiateMultipartUploadRequest; import com.aliyun.oss.model.InitiateMultipartUploadResult; import com.aliyun.oss.model.ListObjectsRequest; +import com.aliyun.oss.model.ListObjectsV2Request; import com.aliyun.oss.model.ObjectMetadata; -import com.aliyun.oss.model.ObjectListing; import com.aliyun.oss.model.OSSObjectSummary; import com.aliyun.oss.model.PartETag; import com.aliyun.oss.model.PutObjectResult; @@ -90,6 +90,7 @@ public class AliyunOSSFileSystemStore { private long uploadPartSize; private int maxKeys; private String serverSideEncryptionAlgorithm; + private boolean useListV1; public void initialize(URI uri, Configuration conf, String user, FileSystem.Statistics stat) throws IOException { @@ -170,6 +171,12 @@ public void initialize(URI uri, Configuration conf, String user, } maxKeys = conf.getInt(MAX_PAGING_KEYS_KEY, MAX_PAGING_KEYS_DEFAULT); + int listVersion = conf.getInt(LIST_VERSION, DEFAULT_LIST_VERSION); + if (listVersion < 1 || listVersion > 2) { + LOG.warn("Configured fs.oss.list.version {} is invalid, forcing " + + "version 2", listVersion); + } + useListV1 = (listVersion == 1); } /** @@ -231,14 +238,10 @@ public void deleteObjects(List keysToDelete) throws IOException { * @throws IOException if failed to delete directory. */ public void deleteDirs(String key) throws IOException { - key = AliyunOSSUtils.maybeAddTrailingSlash(key); - ListObjectsRequest listRequest = new ListObjectsRequest(bucketName); - listRequest.setPrefix(key); - listRequest.setDelimiter(null); - listRequest.setMaxKeys(maxKeys); - + OSSListRequest listRequest = createListObjectsRequest(key, + maxKeys, null, null, true); while (true) { - ObjectListing objects = ossClient.listObjects(listRequest); + OSSListResult objects = listObjects(listRequest); statistics.incrementReadOps(1); List keysToDelete = new ArrayList(); for (OSSObjectSummary objectSummary : objects.getObjectSummaries()) { @@ -246,7 +249,12 @@ public void deleteDirs(String key) throws IOException { } deleteObjects(keysToDelete); if (objects.isTruncated()) { - listRequest.setMarker(objects.getNextMarker()); + if (objects.isV1()) { + listRequest.getV1().setMarker(objects.getV1().getNextMarker()); + } else { + listRequest.getV2().setContinuationToken( + objects.getV2().getNextContinuationToken()); + } } else { break; } @@ -418,25 +426,76 @@ public void uploadObject(String key, File file) throws IOException { /** * list objects. * + * @param listRequest list request. + * @return a list of matches. + */ + public OSSListResult listObjects(OSSListRequest listRequest) { + OSSListResult listResult; + if (listRequest.isV1()) { + listResult = OSSListResult.v1( + ossClient.listObjects(listRequest.getV1())); + } else { + listResult = OSSListResult.v2( + ossClient.listObjectsV2(listRequest.getV2())); + } + statistics.incrementReadOps(1); + return listResult; + } + + /** + * continue to list objects depends on previous list result. + * + * @param listRequest list request. + * @param preListResult previous list result. + * @return a list of matches. + */ + public OSSListResult continueListObjects(OSSListRequest listRequest, + OSSListResult preListResult) { + OSSListResult listResult; + if (listRequest.isV1()) { + listRequest.getV1().setMarker(preListResult.getV1().getNextMarker()); + listResult = OSSListResult.v1( + ossClient.listObjects(listRequest.getV1())); + } else { + listRequest.getV2().setContinuationToken( + preListResult.getV2().getNextContinuationToken()); + listResult = OSSListResult.v2( + ossClient.listObjectsV2(listRequest.getV2())); + } + statistics.incrementReadOps(1); + return listResult; + } + + /** + * create list objects request. + * * @param prefix prefix. * @param maxListingLength max no. of entries * @param marker last key in any previous search. + * @param continuationToken list from a specific point. * @param recursive whether to list directory recursively. * @return a list of matches. */ - public ObjectListing listObjects(String prefix, int maxListingLength, - String marker, boolean recursive) { + protected OSSListRequest createListObjectsRequest(String prefix, + int maxListingLength, String marker, + String continuationToken, boolean recursive) { String delimiter = recursive ? null : "/"; prefix = AliyunOSSUtils.maybeAddTrailingSlash(prefix); - ListObjectsRequest listRequest = new ListObjectsRequest(bucketName); - listRequest.setPrefix(prefix); - listRequest.setDelimiter(delimiter); - listRequest.setMaxKeys(maxListingLength); - listRequest.setMarker(marker); - - ObjectListing listing = ossClient.listObjects(listRequest); - statistics.incrementReadOps(1); - return listing; + if (useListV1) { + ListObjectsRequest listRequest = new ListObjectsRequest(bucketName); + listRequest.setPrefix(prefix); + listRequest.setDelimiter(delimiter); + listRequest.setMaxKeys(maxListingLength); + listRequest.setMarker(marker); + return OSSListRequest.v1(listRequest); + } else { + ListObjectsV2Request listV2Request = new ListObjectsV2Request(bucketName); + listV2Request.setPrefix(prefix); + listV2Request.setDelimiter(delimiter); + listV2Request.setMaxKeys(maxListingLength); + listV2Request.setContinuationToken(continuationToken); + return OSSListRequest.v2(listV2Request); + } } /** @@ -478,21 +537,7 @@ public void close() { * @throws IOException if failed to clean up objects. */ public void purge(String prefix) throws IOException { - String key; - try { - ObjectListing objects = listObjects(prefix, maxKeys, null, true); - for (OSSObjectSummary object : objects.getObjectSummaries()) { - key = object.getKey(); - ossClient.deleteObject(bucketName, key); - statistics.incrementWriteOps(1); - } - - for (String dir: objects.getCommonPrefixes()) { - deleteDirs(dir); - } - } catch (OSSException | ClientException e) { - LOG.error("Failed to purge " + prefix); - } + deleteDirs(prefix); } public RemoteIterator singleStatusRemoteIterator( @@ -520,12 +565,12 @@ public LocatedFileStatus next() throws IOException { public RemoteIterator createLocatedFileStatusIterator( final String prefix, final int maxListingLength, FileSystem fs, - PathFilter filter, FileStatusAcceptor acceptor, String delimiter) { + PathFilter filter, FileStatusAcceptor acceptor, boolean recursive) { return new RemoteIterator() { - private String nextMarker = null; private boolean firstListing = true; private boolean meetEnd = false; private ListIterator batchIterator; + private OSSListRequest listRequest = null; @Override public boolean hasNext() throws IOException { @@ -550,15 +595,24 @@ public LocatedFileStatus next() throws IOException { } private boolean requestNextBatch() { + while (!meetEnd) { + if (continueListStatus()) { + return true; + } + } + + return false; + } + + private boolean continueListStatus() { if (meetEnd) { return false; } - ListObjectsRequest listRequest = new ListObjectsRequest(bucketName); - listRequest.setPrefix(AliyunOSSUtils.maybeAddTrailingSlash(prefix)); - listRequest.setMaxKeys(maxListingLength); - listRequest.setMarker(nextMarker); - listRequest.setDelimiter(delimiter); - ObjectListing listing = ossClient.listObjects(listRequest); + if (listRequest == null) { + listRequest = createListObjectsRequest(prefix, + maxListingLength, null, null, recursive); + } + OSSListResult listing = listObjects(listRequest); List stats = new ArrayList<>( listing.getObjectSummaries().size() + listing.getCommonPrefixes().size()); @@ -584,7 +638,12 @@ private boolean requestNextBatch() { batchIterator = stats.listIterator(); if (listing.isTruncated()) { - nextMarker = listing.getNextMarker(); + if (listing.isV1()) { + listRequest.getV1().setMarker(listing.getV1().getNextMarker()); + } else { + listRequest.getV2().setContinuationToken( + listing.getV2().getNextContinuationToken()); + } } else { meetEnd = true; } diff --git a/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/Constants.java b/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/Constants.java index 71693d057030f..3421b421813ba 100644 --- a/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/Constants.java +++ b/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/Constants.java @@ -154,4 +154,8 @@ private Constants() { public static final String UPLOAD_ACTIVE_BLOCKS_KEY = "fs.oss.upload.active.blocks"; public static final int UPLOAD_ACTIVE_BLOCKS_DEFAULT = 4; + + public static final String LIST_VERSION = "fs.oss.list.version"; + + public static final int DEFAULT_LIST_VERSION = 2; } diff --git a/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/OSSListRequest.java b/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/OSSListRequest.java new file mode 100644 index 0000000000000..0e42b7d74fdc3 --- /dev/null +++ b/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/OSSListRequest.java @@ -0,0 +1,88 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.aliyun.oss; + +import com.aliyun.oss.model.ListObjectsRequest; +import com.aliyun.oss.model.ListObjectsV2Request; + +/** + * API version-independent container for OSS List requests. + */ +public class OSSListRequest { + /** + * Format for the toString() method: {@value}. + */ + private static final String DESCRIPTION + = "List %s:/%s delimiter=%s keys=%d"; + + private final ListObjectsRequest v1Request; + private final ListObjectsV2Request v2Request; + + private OSSListRequest(ListObjectsRequest v1, ListObjectsV2Request v2) { + v1Request = v1; + v2Request = v2; + } + + /** + * Restricted constructors to ensure v1 or v2, not both. + * @param request v1 request + * @return new list request container + */ + public static OSSListRequest v1(ListObjectsRequest request) { + return new OSSListRequest(request, null); + } + + /** + * Restricted constructors to ensure v1 or v2, not both. + * @param request v2 request + * @return new list request container + */ + public static OSSListRequest v2(ListObjectsV2Request request) { + return new OSSListRequest(null, request); + } + + /** + * Is this a v1 API request or v2? + * @return true if v1, false if v2 + */ + public boolean isV1() { + return v1Request != null; + } + + public ListObjectsRequest getV1() { + return v1Request; + } + + public ListObjectsV2Request getV2() { + return v2Request; + } + + @Override + public String toString() { + if (isV1()) { + return String.format(DESCRIPTION, + v1Request.getBucketName(), v1Request.getPrefix(), + v1Request.getDelimiter(), v1Request.getMaxKeys()); + } else { + return String.format(DESCRIPTION, + v2Request.getBucketName(), v2Request.getPrefix(), + v2Request.getDelimiter(), v2Request.getMaxKeys()); + } + } +} \ No newline at end of file diff --git a/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/OSSListResult.java b/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/OSSListResult.java new file mode 100644 index 0000000000000..145bbb036da33 --- /dev/null +++ b/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/OSSListResult.java @@ -0,0 +1,115 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.aliyun.oss; + +import com.aliyun.oss.model.ListObjectsV2Result; +import com.aliyun.oss.model.OSSObjectSummary; +import com.aliyun.oss.model.ObjectListing; +import org.slf4j.Logger; + +import java.util.Collection; +import java.util.List; + +/** + * API version-independent container for OSS List responses. + */ +public final class OSSListResult { + private ObjectListing v1Result; + private ListObjectsV2Result v2Result; + + protected OSSListResult(ObjectListing v1, ListObjectsV2Result v2) { + v1Result = v1; + v2Result = v2; + } + + /** + * Restricted constructors to ensure v1 or v2, not both. + * @param result v1 result + * @return new list result container + */ + public static OSSListResult v1(ObjectListing result) { + return new OSSListResult(result, null); + } + + /** + * Restricted constructors to ensure v1 or v2, not both. + * @param result v2 result + * @return new list result container + */ + public static OSSListResult v2(ListObjectsV2Result result) { + return new OSSListResult(null, result); + } + + /** + * Is this a v1 API result or v2? + * @return true if v1, false if v2 + */ + public boolean isV1() { + return v1Result != null; + } + + public ObjectListing getV1() { + return v1Result; + } + + public ListObjectsV2Result getV2() { + return v2Result; + } + + public List getObjectSummaries() { + if (isV1()) { + return v1Result.getObjectSummaries(); + } else { + return v2Result.getObjectSummaries(); + } + } + + public boolean isTruncated() { + if (isV1()) { + return v1Result.isTruncated(); + } else { + return v2Result.isTruncated(); + } + } + + public List getCommonPrefixes() { + if (isV1()) { + return v1Result.getCommonPrefixes(); + } else { + return v2Result.getCommonPrefixes(); + } + } + + /** + * Dump the result at debug level. + * @param log log to use + */ + public void logAtDebug(Logger log) { + Collection prefixes = getCommonPrefixes(); + Collection summaries = getObjectSummaries(); + log.debug("Prefix count = {}; object count={}", + prefixes.size(), summaries.size()); + for (OSSObjectSummary summary : summaries) { + log.debug("Summary: {} {}", summary.getKey(), summary.getSize()); + } + for (String prefix : prefixes) { + log.debug("Prefix: {}", prefix); + } + } +} \ No newline at end of file diff --git a/hadoop-tools/hadoop-aliyun/src/site/markdown/tools/hadoop-aliyun/index.md b/hadoop-tools/hadoop-aliyun/src/site/markdown/tools/hadoop-aliyun/index.md index 9978bc1fb6931..d48bf1c6b02cb 100644 --- a/hadoop-tools/hadoop-aliyun/src/site/markdown/tools/hadoop-aliyun/index.md +++ b/hadoop-tools/hadoop-aliyun/src/site/markdown/tools/hadoop-aliyun/index.md @@ -243,6 +243,14 @@ please raise your issues with them. Size in bytes in each request from ALiyun OSS. + + fs.oss.list.version + 2 + Select which version of the OSS SDK's List Objects API to use. + Currently support 2(default) and 1(older API). + + + fs.oss.buffer.dir Comma separated list of directories to buffer OSS data before uploading to Aliyun OSS diff --git a/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSTestUtils.java b/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSTestUtils.java index cdf4971b603cf..5e53ef5307a81 100644 --- a/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSTestUtils.java +++ b/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSTestUtils.java @@ -83,4 +83,14 @@ public static String generateUniqueTestPath() { return testUniqueForkId == null ? "/test" : "/" + testUniqueForkId + "/test"; } + + /** + * Turn off FS Caching: use if a filesystem with different options from + * the default is required. + * @param conf configuration to patch + */ + public static void disableFilesystemCaching(Configuration conf) { + conf.setBoolean(TestAliyunOSSFileSystemContract.FS_OSS_IMPL_DISABLE_CACHE, + true); + } } diff --git a/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/TestAliyunOSSBlockOutputStream.java b/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/TestAliyunOSSBlockOutputStream.java index 835e5ffaef5d5..f6e0b7731dbeb 100644 --- a/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/TestAliyunOSSBlockOutputStream.java +++ b/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/TestAliyunOSSBlockOutputStream.java @@ -43,6 +43,7 @@ */ public class TestAliyunOSSBlockOutputStream { private FileSystem fs; + private static final int PART_SIZE = 1024 * 1024; private static String testRootPath = AliyunOSSTestUtils.generateUniqueTestPath(); @@ -52,7 +53,7 @@ public class TestAliyunOSSBlockOutputStream { @Before public void setUp() throws Exception { Configuration conf = new Configuration(); - conf.setInt(MULTIPART_UPLOAD_PART_SIZE_KEY, 1024 * 1024); + conf.setInt(MULTIPART_UPLOAD_PART_SIZE_KEY, PART_SIZE); conf.setInt(IO_CHUNK_BUFFER_SIZE, conf.getInt(MULTIPART_UPLOAD_PART_SIZE_KEY, 0)); conf.setInt(Constants.UPLOAD_ACTIVE_BLOCKS_KEY, 20); @@ -155,10 +156,8 @@ public void testMultiPartUploadConcurrent() throws IOException { @Test public void testHugeUpload() throws IOException { - ContractTestUtils.createAndVerifyFile(fs, getTestPath(), - MULTIPART_UPLOAD_PART_SIZE_DEFAULT - 1); - ContractTestUtils.createAndVerifyFile(fs, getTestPath(), - MULTIPART_UPLOAD_PART_SIZE_DEFAULT); + ContractTestUtils.createAndVerifyFile(fs, getTestPath(), PART_SIZE - 1); + ContractTestUtils.createAndVerifyFile(fs, getTestPath(), PART_SIZE); ContractTestUtils.createAndVerifyFile(fs, getTestPath(), MULTIPART_UPLOAD_PART_SIZE_DEFAULT + 1); bufferDirShouldEmpty(); diff --git a/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/TestAliyunOSSFileSystemContract.java b/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/TestAliyunOSSFileSystemContract.java index e6467f49f49f8..4ae09623b20a3 100644 --- a/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/TestAliyunOSSFileSystemContract.java +++ b/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/TestAliyunOSSFileSystemContract.java @@ -47,6 +47,8 @@ public class TestAliyunOSSFileSystemContract extends FileSystemContractBaseTest { public static final String TEST_FS_OSS_NAME = "test.fs.oss.name"; + public static final String FS_OSS_IMPL_DISABLE_CACHE + = "fs.oss.impl.disable.cache"; private static Path testRootPath = new Path(AliyunOSSTestUtils.generateUniqueTestPath()); @@ -413,7 +415,7 @@ private void testRenameDir(boolean changing, boolean result, boolean empty) Thread thread = new Thread(task); thread.start(); while (!task.isRunning()) { - Thread.sleep(1000); + Thread.sleep(1); } if (changing) { @@ -421,7 +423,11 @@ private void testRenameDir(boolean changing, boolean result, boolean empty) } thread.join(); - assertEquals(result, task.isSucceed()); + if (changing) { + assertTrue(task.isSucceed() || fs.exists(this.path("a"))); + } else { + assertEquals(result, task.isSucceed()); + } } class TestRenameTask implements Runnable { @@ -451,6 +457,8 @@ public void run() { running = true; result = fs.rename(srcPath, dstPath); } catch (Exception e) { + e.printStackTrace(); + this.result = false; } } } diff --git a/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/TestAliyunOSSFileSystemStore.java b/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/TestAliyunOSSFileSystemStore.java index ebd555d643f4f..f85871dd86e3d 100644 --- a/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/TestAliyunOSSFileSystemStore.java +++ b/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/TestAliyunOSSFileSystemStore.java @@ -18,6 +18,7 @@ package org.apache.hadoop.fs.aliyun.oss; +import com.aliyun.oss.model.ObjectMetadata; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; import org.junit.After; @@ -89,13 +90,17 @@ protected void writeRenameReadCompare(Path path, long len) assertTrue("Exists", fs.exists(path)); + ObjectMetadata srcMeta = fs.getStore().getObjectMetadata( + path.toUri().getPath().substring(1)); + Path copyPath = path.suffix(".copy"); - long start = System.currentTimeMillis(); fs.rename(path, copyPath); assertTrue("Copy exists", fs.exists(copyPath)); - // should less than 1 second - assertTrue(System.currentTimeMillis() - start < 1000); + // file type should not change + ObjectMetadata dstMeta = fs.getStore().getObjectMetadata( + copyPath.toUri().getPath().substring(1)); + assertEquals(srcMeta.getObjectType(), dstMeta.getObjectType()); // Download file from Aliyun OSS and compare the digest against the original MessageDigest digest2 = MessageDigest.getInstance("MD5"); InputStream in = new BufferedInputStream( @@ -120,6 +125,7 @@ public void testSmallUpload() throws IOException, NoSuchAlgorithmException { public void testLargeUpload() throws IOException, NoSuchAlgorithmException { // Multipart upload, shallow copy - writeRenameReadCompare(new Path("/test/xlarge"), 2147483648L); // 2GB + writeRenameReadCompare(new Path("/test/xlarge"), + Constants.MULTIPART_UPLOAD_PART_SIZE_DEFAULT + 1); } } diff --git a/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/contract/TestAliyunOSSContractGetFileStatusV1List.java b/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/contract/TestAliyunOSSContractGetFileStatusV1List.java new file mode 100644 index 0000000000000..2f1213c65ded2 --- /dev/null +++ b/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/contract/TestAliyunOSSContractGetFileStatusV1List.java @@ -0,0 +1,53 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.fs.aliyun.oss.contract; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.aliyun.oss.AliyunOSSTestUtils; +import org.apache.hadoop.fs.aliyun.oss.Constants; +import org.apache.hadoop.fs.contract.AbstractContractGetFileStatusTest; +import org.apache.hadoop.fs.contract.AbstractFSContract; + +/** + * Test getFileStatus and related listing operations, + * using the v1 List Objects API. + */ +public class TestAliyunOSSContractGetFileStatusV1List + extends AbstractContractGetFileStatusTest { + + @Override + protected AbstractFSContract createContract(Configuration conf) { + return new AliyunOSSContract(conf); + } + + @Override + public void teardown() throws Exception { + getLogger().info("FS details {}", getFileSystem()); + super.teardown(); + } + + @Override + protected Configuration createConfiguration() { + Configuration conf = super.createConfiguration(); + AliyunOSSTestUtils.disableFilesystemCaching(conf); + conf.setInt(Constants.MAX_PAGING_KEYS_KEY, 2); + // Use v1 List Objects API + conf.setInt(Constants.LIST_VERSION, 1); + return conf; + } +} diff --git a/hadoop-tools/hadoop-aliyun/src/test/resources/contract/aliyun-oss.xml b/hadoop-tools/hadoop-aliyun/src/test/resources/contract/aliyun-oss.xml index 9ec4be6c8afaa..ac7c38c48c23f 100644 --- a/hadoop-tools/hadoop-aliyun/src/test/resources/contract/aliyun-oss.xml +++ b/hadoop-tools/hadoop-aliyun/src/test/resources/contract/aliyun-oss.xml @@ -95,7 +95,7 @@ fs.contract.rename-overwrites-dest - true + false diff --git a/hadoop-tools/hadoop-archive-logs/pom.xml b/hadoop-tools/hadoop-archive-logs/pom.xml index aa98c1bf3547e..3caa26c52f884 100644 --- a/hadoop-tools/hadoop-archive-logs/pom.xml +++ b/hadoop-tools/hadoop-archive-logs/pom.xml @@ -204,38 +204,6 @@ Max - - org.apache.maven.plugins - maven-enforcer-plugin - - - de.skuzzle.enforcer - restrict-imports-enforcer-rule - ${restrict-imports.enforcer.version} - - - - - banned-illegal-imports - process-sources - - enforce - - - - - true - Use hadoop-common provided VisibleForTesting rather than the one provided by Guava - - org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting - com.google.common.annotations.VisibleForTesting - - - - - - - diff --git a/hadoop-tools/hadoop-aws/pom.xml b/hadoop-tools/hadoop-aws/pom.xml index dddbe6fb34641..0cab5ada2169d 100644 --- a/hadoop-tools/hadoop-aws/pom.xml +++ b/hadoop-tools/hadoop-aws/pom.xml @@ -467,38 +467,6 @@ - - org.apache.maven.plugins - maven-enforcer-plugin - - - de.skuzzle.enforcer - restrict-imports-enforcer-rule - ${restrict-imports.enforcer.version} - - - - - banned-illegal-imports - process-sources - - enforce - - - - - true - Use hadoop-common provided VisibleForTesting rather than the one provided by Guava - - org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting - com.google.common.annotations.VisibleForTesting - - - - - - - diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java index 0eadc35c3c27f..ac98af5ada60d 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java @@ -387,7 +387,7 @@ private Constants() { * Value: {@value}. */ public static final boolean DOWNGRADE_SYNCABLE_EXCEPTIONS_DEFAULT = - false; + true; /** * The capacity of executor queues for operations other than block diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/DeleteOperation.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/DeleteOperation.java index 5db996763885b..41415c94bc5a9 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/DeleteOperation.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/DeleteOperation.java @@ -28,7 +28,7 @@ import com.amazonaws.services.s3.model.DeleteObjectsRequest; import com.amazonaws.services.s3.model.DeleteObjectsResult; import org.apache.hadoop.thirdparty.com.google.common.util.concurrent.ListeningExecutorService; -import org.apache.hadoop.thirdparty.com.google.common.util.concurrent.MoreExecutors;; +import org.apache.hadoop.thirdparty.com.google.common.util.concurrent.MoreExecutors; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/DynamoDBMetadataStore.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/DynamoDBMetadataStore.java index a031d33ba3d5c..a4dace15aceba 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/DynamoDBMetadataStore.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/DynamoDBMetadataStore.java @@ -69,7 +69,7 @@ import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; import org.apache.hadoop.util.Lists; import org.apache.hadoop.thirdparty.com.google.common.util.concurrent.ListeningExecutorService; -import org.apache.hadoop.thirdparty.com.google.common.util.concurrent.MoreExecutors;; +import org.apache.hadoop.thirdparty.com.google.common.util.concurrent.MoreExecutors; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/troubleshooting_s3a.md b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/troubleshooting_s3a.md index 6f55d9effedd4..d55e522910850 100644 --- a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/troubleshooting_s3a.md +++ b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/troubleshooting_s3a.md @@ -924,30 +924,48 @@ connector isn't saving any data at all. The `Syncable` API, especially the `hsync()` call, are critical for applications such as HBase to safely persist data. -The S3A connector throws an `UnsupportedOperationException` when these API calls -are made, because the guarantees absolutely cannot be met: nothing is being flushed -or saved. +When configured to do so, the S3A connector throws an `UnsupportedOperationException` +when these API calls are made, because the API guarantees absolutely cannot be met: +_nothing is being flushed or saved_. -* Applications which intend to invoke the Syncable APIs call `hasCapability("hsync")` on +* Applications which intend to invoke the Syncable APIs should call `hasCapability("hsync")` on the stream to see if they are supported. * Or catch and downgrade `UnsupportedOperationException`. -These recommendations _apply to all filesystems_. +These recommendations _apply to all filesystems_. -To downgrade the S3A connector to simply warning of the use of +For consistency with other filesystems, S3A output streams +do not by default reject the `Syncable` calls -instead +they print a warning of its use. + + +The count of invocations of the two APIs are collected in the S3A filesystem +Statistics/IOStatistics and so their use can be monitored. + +To switch the S3A connector to rejecting all use of `hsync()` or `hflush()` calls, set the option -`fs.s3a.downgrade.syncable.exceptions` to true. +`fs.s3a.downgrade.syncable.exceptions` to `false`. ```xml fs.s3a.downgrade.syncable.exceptions - true + false ``` -The count of invocations of the two APIs are collected -in the S3A filesystem Statistics/IOStatistics and so -their use can be monitored. +Regardless of the setting, the `Syncable` API calls do not work. +Telling the store to *not* downgrade the calls is a way to +1. Prevent applications which require Syncable to work from being deployed + against S3. +2. Identify applications which are making the calls even though they don't + need to. These applications can then be fixed -something which may take + time. + +Put differently: it is safest to disable downgrading syncable exceptions. +However, enabling the downgrade stops applications unintentionally using the API +from breaking. + +*Tip*: try turning it on in staging environments to see what breaks. ### `RemoteFileChangedException` and read-during-overwrite diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3ABlockOutputStream.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3ABlockOutputStream.java index 9ae24c19f3da5..21f268dfb2ed5 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3ABlockOutputStream.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3ABlockOutputStream.java @@ -141,6 +141,10 @@ public void testCallingCloseAfterCallingAbort() throws Exception { */ @Test public void testSyncableUnsupported() throws Exception { + final S3ABlockOutputStream.BlockOutputStreamBuilder + builder = mockS3ABuilder(); + builder.withDowngradeSyncableExceptions(false); + stream = spy(new S3ABlockOutputStream(builder)); intercept(UnsupportedOperationException.class, () -> stream.hflush()); intercept(UnsupportedOperationException.class, () -> stream.hsync()); } diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/delegation/ITestSessionDelegationInFileystem.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/delegation/ITestSessionDelegationInFileystem.java index 6aed9e7e314e9..47fad292156b0 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/delegation/ITestSessionDelegationInFileystem.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/delegation/ITestSessionDelegationInFileystem.java @@ -22,6 +22,7 @@ import java.io.File; import java.io.IOException; import java.io.PrintStream; +import java.io.UncheckedIOException; import java.net.URI; import java.nio.file.AccessDeniedException; @@ -41,7 +42,6 @@ import org.apache.hadoop.fs.s3a.Constants; import org.apache.hadoop.fs.s3a.DefaultS3ClientFactory; import org.apache.hadoop.fs.s3a.Invoker; -import org.apache.hadoop.fs.s3a.S3AEncryptionMethods; import org.apache.hadoop.fs.s3a.S3AFileSystem; import org.apache.hadoop.fs.s3a.S3ATestUtils; import org.apache.hadoop.fs.s3a.S3ClientFactory; @@ -69,6 +69,7 @@ import static org.apache.hadoop.fs.s3a.S3ATestUtils.getTestBucketName; import static org.apache.hadoop.fs.s3a.S3ATestUtils.removeBaseAndBucketOverrides; import static org.apache.hadoop.fs.s3a.S3ATestUtils.unsetHadoopCredentialProviders; +import static org.apache.hadoop.fs.s3a.S3AUtils.getEncryptionAlgorithm; import static org.apache.hadoop.fs.s3a.S3AUtils.getS3EncryptionKey; import static org.apache.hadoop.fs.s3a.auth.delegation.DelegationConstants.*; import static org.apache.hadoop.fs.s3a.auth.delegation.DelegationTokenIOException.TOKEN_MISMATCH; @@ -145,9 +146,14 @@ protected Configuration createConfiguration() { // disable if assume role opts are off assumeSessionTestsEnabled(conf); disableFilesystemCaching(conf); - String s3EncryptionMethod = - conf.getTrimmed(Constants.S3_ENCRYPTION_ALGORITHM, - S3AEncryptionMethods.SSE_KMS.getMethod()); + String s3EncryptionMethod; + try { + s3EncryptionMethod = + getEncryptionAlgorithm(getTestBucketName(conf), conf).getMethod(); + } catch (IOException e) { + throw new UncheckedIOException("Failed to lookup encryption algorithm.", + e); + } String s3EncryptionKey = getS3EncryptionKey(getTestBucketName(conf), conf); removeBaseAndBucketOverrides(conf, DELEGATION_TOKEN_BINDING, diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/fileContext/ITestS3AFileContextStatistics.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/fileContext/ITestS3AFileContextStatistics.java index ca8ce15996388..d28f4279f1ba0 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/fileContext/ITestS3AFileContextStatistics.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/fileContext/ITestS3AFileContextStatistics.java @@ -13,6 +13,7 @@ */ package org.apache.hadoop.fs.s3a.fileContext; +import java.io.IOException; import java.net.URI; import com.amazonaws.services.s3.model.CryptoStorageMode; @@ -32,9 +33,10 @@ import org.junit.Assert; import org.junit.Before; -import static org.apache.hadoop.fs.s3a.Constants.S3_ENCRYPTION_ALGORITHM; -import static org.apache.hadoop.fs.s3a.Constants.S3_ENCRYPTION_KEY; import static org.apache.hadoop.fs.s3a.S3ATestConstants.KMS_KEY_GENERATION_REQUEST_PARAMS_BYTES_WRITTEN; +import static org.apache.hadoop.fs.s3a.S3ATestUtils.getTestBucketName; +import static org.apache.hadoop.fs.s3a.S3AUtils.getEncryptionAlgorithm; +import static org.apache.hadoop.fs.s3a.S3AUtils.getS3EncryptionKey; import static org.apache.hadoop.fs.s3a.impl.InternalConstants.CSE_PADDING_LENGTH; /** @@ -83,12 +85,14 @@ protected void verifyReadBytes(FileSystem.Statistics stats) { * @param stats Filesystem statistics. */ @Override - protected void verifyWrittenBytes(FileSystem.Statistics stats) { + protected void verifyWrittenBytes(FileSystem.Statistics stats) + throws IOException { //No extra bytes are written long expectedBlockSize = blockSize; - if (conf.get(S3_ENCRYPTION_ALGORITHM, "") - .equals(S3AEncryptionMethods.CSE_KMS.getMethod())) { - String keyId = conf.get(S3_ENCRYPTION_KEY, ""); + if (S3AEncryptionMethods.CSE_KMS.getMethod() + .equals(getEncryptionAlgorithm(getTestBucketName(conf), conf) + .getMethod())) { + String keyId = getS3EncryptionKey(getTestBucketName(conf), conf); // Adding padding length and KMS key generation bytes written. expectedBlockSize += CSE_PADDING_LENGTH + keyId.getBytes().length + KMS_KEY_GENERATION_REQUEST_PARAMS_BYTES_WRITTEN; diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/impl/ITestPartialRenamesDeletes.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/impl/ITestPartialRenamesDeletes.java index df45d0def5a4f..b03d52fd5671d 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/impl/ITestPartialRenamesDeletes.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/impl/ITestPartialRenamesDeletes.java @@ -34,7 +34,7 @@ import com.amazonaws.services.s3.model.MultiObjectDeleteException; import org.apache.hadoop.thirdparty.com.google.common.base.Charsets; import org.apache.hadoop.thirdparty.com.google.common.util.concurrent.ListeningExecutorService; -import org.apache.hadoop.thirdparty.com.google.common.util.concurrent.MoreExecutors;; +import org.apache.hadoop.thirdparty.com.google.common.util.concurrent.MoreExecutors; import org.assertj.core.api.Assertions; import org.junit.Test; import org.junit.runner.RunWith; diff --git a/hadoop-tools/hadoop-azure-datalake/pom.xml b/hadoop-tools/hadoop-azure-datalake/pom.xml index 6d73cfc09a719..446e2957b60ed 100644 --- a/hadoop-tools/hadoop-azure-datalake/pom.xml +++ b/hadoop-tools/hadoop-azure-datalake/pom.xml @@ -65,38 +65,6 @@ - - org.apache.maven.plugins - maven-enforcer-plugin - - - de.skuzzle.enforcer - restrict-imports-enforcer-rule - ${restrict-imports.enforcer.version} - - - - - banned-illegal-imports - process-sources - - enforce - - - - - true - Use hadoop-common provided VisibleForTesting rather than the one provided by Guava - - org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting - com.google.common.annotations.VisibleForTesting - - - - - - - diff --git a/hadoop-tools/hadoop-azure/pom.xml b/hadoop-tools/hadoop-azure/pom.xml index df852d5b426ae..1896e15d27018 100644 --- a/hadoop-tools/hadoop-azure/pom.xml +++ b/hadoop-tools/hadoop-azure/pom.xml @@ -100,38 +100,6 @@ - - org.apache.maven.plugins - maven-enforcer-plugin - - - de.skuzzle.enforcer - restrict-imports-enforcer-rule - ${restrict-imports.enforcer.version} - - - - - banned-illegal-imports - process-sources - - enforce - - - - - true - Use hadoop-common provided VisibleForTesting rather than the one provided by Guava - - org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting - com.google.common.annotations.VisibleForTesting - - - - - - - diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/AzureNativeFileSystemStore.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/AzureNativeFileSystemStore.java index 3fc1a9a9fc27c..6e50cfc627e05 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/AzureNativeFileSystemStore.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/AzureNativeFileSystemStore.java @@ -282,9 +282,9 @@ public class AzureNativeFileSystemStore implements NativeFileSystemStore { // Retry parameter defaults. // - private static final int DEFAULT_MIN_BACKOFF_INTERVAL = 3 * 1000; // 1s + private static final int DEFAULT_MIN_BACKOFF_INTERVAL = 3 * 1000; // 3s private static final int DEFAULT_MAX_BACKOFF_INTERVAL = 30 * 1000; // 30s - private static final int DEFAULT_BACKOFF_INTERVAL = 3 * 1000; // 1s + private static final int DEFAULT_BACKOFF_INTERVAL = 3 * 1000; // 3s private static final int DEFAULT_MAX_RETRY_ATTEMPTS = 30; private static final int DEFAULT_COPYBLOB_MIN_BACKOFF_INTERVAL = 3 * 1000; diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsListResult.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsListResult.java new file mode 100644 index 0000000000000..0b63a342936a8 --- /dev/null +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsListResult.java @@ -0,0 +1,57 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.azurebfs.services; + +import java.io.IOException; +import java.util.Collections; +import java.util.Iterator; + +import org.apache.hadoop.fs.FileStatus; + +/** + * Class to store listStatus results for AbfsListStatusRemoteIterator. The + * results can either be of type Iterator or an exception thrown during the + * operation + */ +public class AbfsListResult { + private IOException listException = null; + + private Iterator fileStatusIterator + = Collections.emptyIterator(); + + AbfsListResult(IOException ex) { + this.listException = ex; + } + + AbfsListResult(Iterator fileStatusIterator) { + this.fileStatusIterator = fileStatusIterator; + } + + IOException getListingException() { + return listException; + } + + Iterator getFileStatusIterator() { + return fileStatusIterator; + } + + boolean isFailedListing() { + return (listException != null); + } +} diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsListStatusRemoteIterator.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsListStatusRemoteIterator.java index 835217f945af5..ce6207bf5f20c 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsListStatusRemoteIterator.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsListStatusRemoteIterator.java @@ -27,7 +27,6 @@ import java.util.concurrent.ArrayBlockingQueue; import java.util.concurrent.CompletableFuture; import java.util.concurrent.TimeUnit; -import javax.activation.UnsupportedDataTypeException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -48,7 +47,7 @@ public class AbfsListStatusRemoteIterator private final FileStatus fileStatus; private final ListingSupport listingSupport; - private final ArrayBlockingQueue iteratorsQueue; + private final ArrayBlockingQueue listResultQueue; private final TracingContext tracingContext; private volatile boolean isAsyncInProgress = false; @@ -61,7 +60,7 @@ public AbfsListStatusRemoteIterator(final FileStatus fileStatus, this.fileStatus = fileStatus; this.listingSupport = listingSupport; this.tracingContext = tracingContext; - iteratorsQueue = new ArrayBlockingQueue<>(MAX_QUEUE_SIZE); + listResultQueue = new ArrayBlockingQueue<>(MAX_QUEUE_SIZE); currIterator = Collections.emptyIterator(); fetchBatchesAsync(); } @@ -86,19 +85,17 @@ public FileStatus next() throws IOException { private Iterator getNextIterator() throws IOException { fetchBatchesAsync(); try { - Object obj = null; - while (obj == null - && (!isIterationComplete || !iteratorsQueue.isEmpty())) { - obj = iteratorsQueue.poll(POLL_WAIT_TIME_IN_MS, TimeUnit.MILLISECONDS); + AbfsListResult listResult = null; + while (listResult == null + && (!isIterationComplete || !listResultQueue.isEmpty())) { + listResult = listResultQueue.poll(POLL_WAIT_TIME_IN_MS, TimeUnit.MILLISECONDS); } - if (obj == null) { + if (listResult == null) { return Collections.emptyIterator(); - } else if (obj instanceof Iterator) { - return (Iterator) obj; - } else if (obj instanceof IOException) { - throw (IOException) obj; + } else if (listResult.isFailedListing()) { + throw listResult.getListingException(); } else { - throw new UnsupportedDataTypeException(); + return listResult.getFileStatusIterator(); } } catch (InterruptedException e) { Thread.currentThread().interrupt(); @@ -122,13 +119,13 @@ private void fetchBatchesAsync() { private void asyncOp() { try { - while (!isIterationComplete && iteratorsQueue.size() <= MAX_QUEUE_SIZE) { + while (!isIterationComplete && listResultQueue.size() <= MAX_QUEUE_SIZE) { addNextBatchIteratorToQueue(); } } catch (IOException ioe) { LOG.error("Fetching filestatuses failed", ioe); try { - iteratorsQueue.put(ioe); + listResultQueue.put(new AbfsListResult(ioe)); } catch (InterruptedException interruptedException) { Thread.currentThread().interrupt(); LOG.error("Thread got interrupted: {}", interruptedException); @@ -143,19 +140,17 @@ private void asyncOp() { } } - private void addNextBatchIteratorToQueue() + private synchronized void addNextBatchIteratorToQueue() throws IOException, InterruptedException { List fileStatuses = new ArrayList<>(); continuation = listingSupport .listStatus(fileStatus.getPath(), null, fileStatuses, FETCH_ALL_FALSE, continuation, tracingContext); if (!fileStatuses.isEmpty()) { - iteratorsQueue.put(fileStatuses.iterator()); + listResultQueue.put(new AbfsListResult(fileStatuses.iterator())); } - synchronized (this) { - if (continuation == null || continuation.isEmpty()) { - isIterationComplete = true; - } + if (continuation == null || continuation.isEmpty()) { + isIterationComplete = true; } } diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/utils/TextFileBasedIdentityHandler.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/utils/TextFileBasedIdentityHandler.java index 523fd3179405a..6e57d0d294944 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/utils/TextFileBasedIdentityHandler.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/utils/TextFileBasedIdentityHandler.java @@ -29,6 +29,7 @@ import org.apache.commons.io.FileUtils; import org.apache.commons.io.LineIterator; +import org.apache.hadoop.io.IOUtils; import static org.apache.hadoop.fs.azurebfs.constants.AbfsHttpConstants.COLON; import static org.apache.hadoop.fs.azurebfs.constants.AbfsHttpConstants.EMPTY_STRING; @@ -189,7 +190,7 @@ private static void loadMap(HashMap cache, String fileLocation, } catch (ArrayIndexOutOfBoundsException e) { LOG.error("Error while parsing mapping file", e); } finally { - LineIterator.closeQuietly(it); + IOUtils.cleanupWithLogger(LOG, it); } } } diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAbfsListStatusRemoteIterator.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAbfsListStatusRemoteIterator.java index 9e81a0127b6cb..3f50aec659142 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAbfsListStatusRemoteIterator.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAbfsListStatusRemoteIterator.java @@ -21,17 +21,20 @@ import java.io.FileNotFoundException; import java.io.IOException; import java.util.ArrayList; +import java.util.Collections; import java.util.List; import java.util.NoSuchElementException; -import java.util.concurrent.Callable; import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.Future; import org.assertj.core.api.Assertions; +import org.junit.Assert; import org.junit.Test; import org.mockito.Mockito; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.Path; @@ -39,6 +42,7 @@ import org.apache.hadoop.fs.azurebfs.services.AbfsListStatusRemoteIterator; import org.apache.hadoop.fs.azurebfs.services.ListingSupport; import org.apache.hadoop.fs.azurebfs.utils.TracingContext; +import org.apache.hadoop.test.LambdaTestUtils; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyBoolean; @@ -52,6 +56,8 @@ public class ITestAbfsListStatusRemoteIterator extends AbstractAbfsIntegrationTest { private static final int TEST_FILES_NUMBER = 1000; + private static final Logger LOG = LoggerFactory.getLogger( + ITestAbfsListStatusRemoteIterator.class); public ITestAbfsListStatusRemoteIterator() throws Exception { } @@ -60,8 +66,7 @@ public ITestAbfsListStatusRemoteIterator() throws Exception { public void testAbfsIteratorWithHasNext() throws Exception { Path testDir = createTestDirectory(); setPageSize(10); - final List fileNames = createFilesUnderDirectory(TEST_FILES_NUMBER, - testDir, "testListPath"); + final List fileNames = createFilesUnderDirectory(testDir); ListingSupport listngSupport = Mockito.spy(getFileSystem().getAbfsStore()); RemoteIterator fsItr = new AbfsListStatusRemoteIterator( @@ -74,20 +79,12 @@ public void testAbfsIteratorWithHasNext() throws Exception { int itrCount = 0; while (fsItr.hasNext()) { FileStatus fileStatus = fsItr.next(); - String pathStr = fileStatus.getPath().toString(); - fileNames.remove(pathStr); + verifyIteratorResultContent(fileStatus, fileNames); itrCount++; } - Assertions.assertThat(itrCount) - .describedAs("Number of iterations should be equal to the files " - + "created") - .isEqualTo(TEST_FILES_NUMBER); - Assertions.assertThat(fileNames.size()) - .describedAs("After removing every iterm found from the iterator, " - + "there should be no more elements in the fileNames") - .isEqualTo(0); - int minNumberOfInvokations = TEST_FILES_NUMBER / 10; - verify(listngSupport, Mockito.atLeast(minNumberOfInvokations)) + verifyIteratorResultCount(itrCount, fileNames); + int minNumberOfInvocations = TEST_FILES_NUMBER / 10; + verify(listngSupport, Mockito.atLeast(minNumberOfInvocations)) .listStatus(any(Path.class), nullable(String.class), anyList(), anyBoolean(), nullable(String.class), @@ -98,8 +95,7 @@ public void testAbfsIteratorWithHasNext() throws Exception { public void testAbfsIteratorWithoutHasNext() throws Exception { Path testDir = createTestDirectory(); setPageSize(10); - final List fileNames = createFilesUnderDirectory(TEST_FILES_NUMBER, - testDir, "testListPath"); + final List fileNames = createFilesUnderDirectory(testDir); ListingSupport listngSupport = Mockito.spy(getFileSystem().getAbfsStore()); RemoteIterator fsItr = new AbfsListStatusRemoteIterator( @@ -112,25 +108,13 @@ public void testAbfsIteratorWithoutHasNext() throws Exception { int itrCount = 0; for (int i = 0; i < TEST_FILES_NUMBER; i++) { FileStatus fileStatus = fsItr.next(); - String pathStr = fileStatus.getPath().toString(); - fileNames.remove(pathStr); + verifyIteratorResultContent(fileStatus, fileNames); itrCount++; } - Assertions.assertThatThrownBy(() -> fsItr.next()) - .describedAs( - "next() should throw NoSuchElementException since next has been " - + "called " + TEST_FILES_NUMBER + " times") - .isInstanceOf(NoSuchElementException.class); - Assertions.assertThat(itrCount) - .describedAs("Number of iterations should be equal to the files " - + "created") - .isEqualTo(TEST_FILES_NUMBER); - Assertions.assertThat(fileNames.size()) - .describedAs("After removing every iterm found from the iterator, " - + "there should be no more elements in the fileNames") - .isEqualTo(0); - int minNumberOfInvokations = TEST_FILES_NUMBER / 10; - verify(listngSupport, Mockito.atLeast(minNumberOfInvokations)) + LambdaTestUtils.intercept(NoSuchElementException.class, fsItr::next); + verifyIteratorResultCount(itrCount, fileNames); + int minNumberOfInvocations = TEST_FILES_NUMBER / 10; + verify(listngSupport, Mockito.atLeast(minNumberOfInvocations)) .listStatus(any(Path.class), nullable(String.class), anyList(), anyBoolean(), nullable(String.class), @@ -141,9 +125,8 @@ public void testAbfsIteratorWithoutHasNext() throws Exception { public void testWithAbfsIteratorDisabled() throws Exception { Path testDir = createTestDirectory(); setPageSize(10); - setEnableAbfsIterator(false); - final List fileNames = createFilesUnderDirectory(TEST_FILES_NUMBER, - testDir, "testListPath"); + disableAbfsIterator(); + final List fileNames = createFilesUnderDirectory(testDir); RemoteIterator fsItr = getFileSystem().listStatusIterator(testDir); @@ -154,73 +137,46 @@ public void testWithAbfsIteratorDisabled() throws Exception { int itrCount = 0; while (fsItr.hasNext()) { FileStatus fileStatus = fsItr.next(); - String pathStr = fileStatus.getPath().toString(); - fileNames.remove(pathStr); + verifyIteratorResultContent(fileStatus, fileNames); itrCount++; } - Assertions.assertThat(itrCount) - .describedAs("Number of iterations should be equal to the files " - + "created") - .isEqualTo(TEST_FILES_NUMBER); - Assertions.assertThat(fileNames.size()) - .describedAs("After removing every iterm found from the iterator, " - + "there should be no more elements in the fileNames") - .isEqualTo(0); + verifyIteratorResultCount(itrCount, fileNames); } @Test public void testWithAbfsIteratorDisabledWithoutHasNext() throws Exception { Path testDir = createTestDirectory(); setPageSize(10); - setEnableAbfsIterator(false); - final List fileNames = createFilesUnderDirectory(TEST_FILES_NUMBER, - testDir, "testListPath"); - - RemoteIterator fsItr = - getFileSystem().listStatusIterator(testDir); - Assertions.assertThat(fsItr) - .describedAs("RemoteIterator should not be instance of " - + "AbfsListStatusRemoteIterator when it is disabled") + disableAbfsIterator(); + final List fileNames = createFilesUnderDirectory(testDir); + + RemoteIterator fsItr = getFileSystem().listStatusIterator( + testDir); + Assertions.assertThat(fsItr).describedAs( + "RemoteIterator should not be instance of " + + "AbfsListStatusRemoteIterator when it is disabled") .isNotInstanceOf(AbfsListStatusRemoteIterator.class); - int itrCount = 0; - for (int i = 0; i < TEST_FILES_NUMBER; i++) { + int itrCount; + for (itrCount = 0; itrCount < TEST_FILES_NUMBER; itrCount++) { FileStatus fileStatus = fsItr.next(); - String pathStr = fileStatus.getPath().toString(); - fileNames.remove(pathStr); - itrCount++; + verifyIteratorResultContent(fileStatus, fileNames); } - Assertions.assertThatThrownBy(() -> fsItr.next()) - .describedAs( - "next() should throw NoSuchElementException since next has been " - + "called " + TEST_FILES_NUMBER + " times") - .isInstanceOf(NoSuchElementException.class); - Assertions.assertThat(itrCount) - .describedAs("Number of iterations should be equal to the files " - + "created") - .isEqualTo(TEST_FILES_NUMBER); - Assertions.assertThat(fileNames.size()) - .describedAs("After removing every iterm found from the iterator, " - + "there should be no more elements in the fileNames") - .isEqualTo(0); + LambdaTestUtils.intercept(NoSuchElementException.class, fsItr::next); + verifyIteratorResultCount(itrCount, fileNames); } @Test public void testNextWhenNoMoreElementsPresent() throws Exception { Path testDir = createTestDirectory(); setPageSize(10); - RemoteIterator fsItr = + RemoteIterator fsItr = new AbfsListStatusRemoteIterator(getFileSystem().getFileStatus(testDir), getFileSystem().getAbfsStore(), getTestTracingContext(getFileSystem(), true)); fsItr = Mockito.spy(fsItr); Mockito.doReturn(false).when(fsItr).hasNext(); - RemoteIterator finalFsItr = fsItr; - Assertions.assertThatThrownBy(() -> finalFsItr.next()) - .describedAs( - "next() should throw NoSuchElementException if hasNext() return " - + "false") - .isInstanceOf(NoSuchElementException.class); + LambdaTestUtils.intercept(NoSuchElementException.class, fsItr::next); } @Test @@ -257,38 +213,47 @@ public void testIOException() throws Exception { String exceptionMessage = "test exception"; ListingSupport lsSupport =getMockListingSupport(exceptionMessage); - RemoteIterator fsItr = + RemoteIterator fsItr = new AbfsListStatusRemoteIterator(getFileSystem().getFileStatus(testDir), lsSupport, getTestTracingContext(getFileSystem(), true)); - Assertions.assertThatThrownBy(() -> fsItr.next()) - .describedAs( - "When ioException is not null and queue is empty exception should be " - + "thrown") - .isInstanceOf(IOException.class) - .hasMessage(exceptionMessage); + LambdaTestUtils.intercept(IOException.class, fsItr::next); } @Test - public void testNonExistingPath() throws Throwable { + public void testNonExistingPath() throws Exception { Path nonExistingDir = new Path("nonExistingPath"); - Assertions.assertThatThrownBy( - () -> getFileSystem().listStatusIterator(nonExistingDir)).describedAs( - "test the listStatusIterator call on a path which is not " - + "present should result in FileNotFoundException") - .isInstanceOf(FileNotFoundException.class); + LambdaTestUtils.intercept(FileNotFoundException.class, + () -> getFileSystem().listStatusIterator(nonExistingDir)); + } + + private void verifyIteratorResultContent(FileStatus fileStatus, + List fileNames) { + String pathStr = fileStatus.getPath().toString(); + Assert.assertTrue( + String.format("Could not remove path %s from filenames %s", pathStr, + fileNames), fileNames.remove(pathStr)); + } + + private void verifyIteratorResultCount(int itrCount, List fileNames) { + Assertions.assertThat(itrCount).describedAs( + "Number of iterations should be equal to the files created") + .isEqualTo(TEST_FILES_NUMBER); + Assertions.assertThat(fileNames) + .describedAs("After removing every item found from the iterator, " + + "there should be no more elements in the fileNames") + .hasSize(0); } private ListingSupport getMockListingSupport(String exceptionMessage) { return new ListingSupport() { @Override - public FileStatus[] listStatus(Path path, TracingContext tracingContext) throws IOException { + public FileStatus[] listStatus(Path path, TracingContext tracingContext) { return null; } @Override - public FileStatus[] listStatus(Path path, String startFrom, TracingContext tracingContext) - throws IOException { + public FileStatus[] listStatus(Path path, String startFrom, TracingContext tracingContext) { return null; } @@ -303,15 +268,14 @@ public String listStatus(Path path, String startFrom, } private Path createTestDirectory() throws IOException { - String testDirectoryName = "testDirectory" + System.currentTimeMillis(); - Path testDirectory = path(testDirectoryName); + Path testDirectory = path("testDirectory"); getFileSystem().mkdirs(testDirectory); return testDirectory; } - private void setEnableAbfsIterator(boolean shouldEnable) throws IOException { + private void disableAbfsIterator() throws IOException { AzureBlobFileSystemStore abfsStore = getAbfsStore(getFileSystem()); - abfsStore.getAbfsConfiguration().setEnableAbfsListIterator(shouldEnable); + abfsStore.getAbfsConfiguration().setEnableAbfsListIterator(false); } private void setPageSize(int pageSize) throws IOException { @@ -319,21 +283,21 @@ private void setPageSize(int pageSize) throws IOException { abfsStore.getAbfsConfiguration().setListMaxResults(pageSize); } - private List createFilesUnderDirectory(int numFiles, Path rootPath, - String filenamePrefix) + private List createFilesUnderDirectory(Path rootPath) throws ExecutionException, InterruptedException, IOException { final List> tasks = new ArrayList<>(); - final List fileNames = new ArrayList<>(); + final List fileNames = Collections.synchronizedList(new ArrayList<>()); ExecutorService es = Executors.newFixedThreadPool(10); try { - for (int i = 0; i < numFiles; i++) { - final Path filePath = new Path(rootPath, filenamePrefix + i); - Callable callable = () -> { - getFileSystem().create(filePath); - fileNames.add(makeQualified(filePath).toString()); + for (int i = 0; i < ITestAbfsListStatusRemoteIterator.TEST_FILES_NUMBER; i++) { + Path filePath = makeQualified(new Path(rootPath, "testListPath" + i)); + tasks.add(es.submit(() -> { + touch(filePath); + synchronized (fileNames) { + Assert.assertTrue(fileNames.add(filePath.toString())); + } return null; - }; - tasks.add(es.submit(callable)); + })); } for (Future task : tasks) { task.get(); @@ -341,6 +305,10 @@ private List createFilesUnderDirectory(int numFiles, Path rootPath, } finally { es.shutdownNow(); } + LOG.debug(fileNames.toString()); + Assertions.assertThat(fileNames) + .describedAs("File creation incorrect or fileNames not added to list") + .hasSize(ITestAbfsListStatusRemoteIterator.TEST_FILES_NUMBER); return fileNames; } diff --git a/hadoop-tools/hadoop-distcp/pom.xml b/hadoop-tools/hadoop-distcp/pom.xml index 20109b0cf61f7..7e5aaebc08513 100644 --- a/hadoop-tools/hadoop-distcp/pom.xml +++ b/hadoop-tools/hadoop-distcp/pom.xml @@ -223,38 +223,6 @@ - - org.apache.maven.plugins - maven-enforcer-plugin - - - de.skuzzle.enforcer - restrict-imports-enforcer-rule - ${restrict-imports.enforcer.version} - - - - - banned-illegal-imports - process-sources - - enforce - - - - - true - Use hadoop-common provided VisibleForTesting rather than the one provided by Guava - - org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting - com.google.common.annotations.VisibleForTesting - - - - - - - diff --git a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/mapred/RetriableFileCopyCommand.java b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/mapred/RetriableFileCopyCommand.java index e836ccce41223..544dbfbc4c0f0 100644 --- a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/mapred/RetriableFileCopyCommand.java +++ b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/mapred/RetriableFileCopyCommand.java @@ -154,7 +154,7 @@ private long doCopy(CopyListingFileStatus source, Path target, if (!source.isSplit()) { DistCpUtils.compareFileLengthsAndChecksums(source.getLen(), sourceFS, sourcePath, sourceChecksum, targetFS, - targetPath, skipCrc, source.getLen()); + targetPath, skipCrc, offset + bytesRead); } // it's not append or direct write (preferred for s3a) case, thus we first // write to a temporary file, then rename it to the target path. diff --git a/hadoop-tools/hadoop-dynamometer/hadoop-dynamometer-infra/pom.xml b/hadoop-tools/hadoop-dynamometer/hadoop-dynamometer-infra/pom.xml index 3a1c3e9506b33..7bd4318cf70ce 100644 --- a/hadoop-tools/hadoop-dynamometer/hadoop-dynamometer-infra/pom.xml +++ b/hadoop-tools/hadoop-dynamometer/hadoop-dynamometer-infra/pom.xml @@ -137,38 +137,6 @@ - - org.apache.maven.plugins - maven-enforcer-plugin - - - de.skuzzle.enforcer - restrict-imports-enforcer-rule - ${restrict-imports.enforcer.version} - - - - - banned-illegal-imports - process-sources - - enforce - - - - - true - Use hadoop-common provided VisibleForTesting rather than the one provided by Guava - - org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting - com.google.common.annotations.VisibleForTesting - - - - - - - diff --git a/hadoop-tools/hadoop-federation-balance/pom.xml b/hadoop-tools/hadoop-federation-balance/pom.xml index f048e2a11f639..588bb98f3e75a 100644 --- a/hadoop-tools/hadoop-federation-balance/pom.xml +++ b/hadoop-tools/hadoop-federation-balance/pom.xml @@ -233,38 +233,6 @@ - - org.apache.maven.plugins - maven-enforcer-plugin - - - de.skuzzle.enforcer - restrict-imports-enforcer-rule - ${restrict-imports.enforcer.version} - - - - - banned-illegal-imports - process-sources - - enforce - - - - - true - Use hadoop-common provided VisibleForTesting rather than the one provided by Guava - - org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting - com.google.common.annotations.VisibleForTesting - - - - - - - diff --git a/hadoop-tools/hadoop-fs2img/pom.xml b/hadoop-tools/hadoop-fs2img/pom.xml index 66f971c9e1a74..9a273621cfa9d 100644 --- a/hadoop-tools/hadoop-fs2img/pom.xml +++ b/hadoop-tools/hadoop-fs2img/pom.xml @@ -107,38 +107,6 @@ - - org.apache.maven.plugins - maven-enforcer-plugin - - - de.skuzzle.enforcer - restrict-imports-enforcer-rule - ${restrict-imports.enforcer.version} - - - - - banned-illegal-imports - process-sources - - enforce - - - - - true - Use hadoop-common provided VisibleForTesting rather than the one provided by Guava - - org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting - com.google.common.annotations.VisibleForTesting - - - - - - - diff --git a/hadoop-tools/hadoop-gridmix/pom.xml b/hadoop-tools/hadoop-gridmix/pom.xml index b903bb6f097f5..f383ecad64150 100644 --- a/hadoop-tools/hadoop-gridmix/pom.xml +++ b/hadoop-tools/hadoop-gridmix/pom.xml @@ -188,38 +188,6 @@ - - org.apache.maven.plugins - maven-enforcer-plugin - - - de.skuzzle.enforcer - restrict-imports-enforcer-rule - ${restrict-imports.enforcer.version} - - - - - banned-illegal-imports - process-sources - - enforce - - - - - true - Use hadoop-common provided VisibleForTesting rather than the one provided by Guava - - org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting - com.google.common.annotations.VisibleForTesting - - - - - - - diff --git a/hadoop-tools/hadoop-sls/pom.xml b/hadoop-tools/hadoop-sls/pom.xml index 329cc790fb9e5..208cbdf6c142a 100644 --- a/hadoop-tools/hadoop-sls/pom.xml +++ b/hadoop-tools/hadoop-sls/pom.xml @@ -147,38 +147,6 @@ - - org.apache.maven.plugins - maven-enforcer-plugin - - - de.skuzzle.enforcer - restrict-imports-enforcer-rule - ${restrict-imports.enforcer.version} - - - - - banned-illegal-imports - process-sources - - enforce - - - - - true - Use hadoop-common provided VisibleForTesting rather than the one provided by Guava - - org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting - com.google.common.annotations.VisibleForTesting - - - - - - - diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/com/nec/VEDeviceDiscoverer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/com/nec/VEDeviceDiscoverer.java index 4bc92f2711727..8d801198644e8 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/com/nec/VEDeviceDiscoverer.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/com/nec/VEDeviceDiscoverer.java @@ -27,6 +27,7 @@ import java.util.Set; import java.util.function.Function; import java.util.stream.Collectors; +import java.util.stream.Stream; import org.apache.commons.lang3.mutable.MutableInt; import org.apache.hadoop.util.Shell; @@ -58,11 +59,11 @@ class VEDeviceDiscoverer { public Set getDevicesFromPath(String path) throws IOException { MutableInt counter = new MutableInt(0); - - return Files.walk(Paths.get(path), 1) - .filter(p -> p.toFile().getName().startsWith("veslot")) - .map(p -> toDevice(p, counter)) - .collect(Collectors.toSet()); + try (Stream stream = Files.walk(Paths.get(path), 1)) { + return stream.filter(p -> p.toFile().getName().startsWith("veslot")) + .map(p -> toDevice(p, counter)) + .collect(Collectors.toSet()); + } } private Device toDevice(Path p, MutableInt counter) { @@ -140,4 +141,4 @@ void setCommandExecutorProvider( Function provider) { this.commandExecutorProvider = provider; } -} \ No newline at end of file +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java index dbadc3ce87d7f..04efc8864139b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java @@ -600,10 +600,12 @@ public SubmitApplicationResponse submitApplication( // checked here, those that are dependent on RM configuration are validated // in RMAppManager. + UserGroupInformation userUgi = null; String user = null; try { // Safety - user = UserGroupInformation.getCurrentUser().getShortUserName(); + userUgi = UserGroupInformation.getCurrentUser(); + user = userUgi.getShortUserName(); } catch (IOException ie) { LOG.warn("Unable to get the current user.", ie); RMAuditLogger.logFailure(user, AuditConstants.SUBMIT_APP_REQUEST, @@ -694,7 +696,7 @@ public SubmitApplicationResponse submitApplication( try { // call RMAppManager to submit application directly rmAppManager.submitApplication(submissionContext, - System.currentTimeMillis(), user); + System.currentTimeMillis(), userUgi); LOG.info("Application with id " + applicationId.getId() + " submitted by user " + user); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java index 762d043b1f6f1..7ebcd656f9b70 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java @@ -352,16 +352,26 @@ protected synchronized void checkAppNumCompletedLimit() { } } - @SuppressWarnings("unchecked") + @VisibleForTesting + @Deprecated protected void submitApplication( ApplicationSubmissionContext submissionContext, long submitTime, String user) throws YarnException { + submitApplication(submissionContext, submitTime, + UserGroupInformation.createRemoteUser(user)); + } + + @VisibleForTesting + @SuppressWarnings("unchecked") + protected void submitApplication( + ApplicationSubmissionContext submissionContext, long submitTime, + UserGroupInformation userUgi) throws YarnException { ApplicationId applicationId = submissionContext.getApplicationId(); // Passing start time as -1. It will be eventually set in RMAppImpl // constructor. RMAppImpl application = createAndPopulateNewRMApp( - submissionContext, submitTime, user, false, -1, null); + submissionContext, submitTime, userUgi, false, -1, null); try { if (UserGroupInformation.isSecurityEnabled()) { this.rmContext.getDelegationTokenRenewer() @@ -394,11 +404,21 @@ protected void recoverApplication(ApplicationStateData appState, ApplicationSubmissionContext appContext = appState.getApplicationSubmissionContext(); ApplicationId appId = appContext.getApplicationId(); + UserGroupInformation userUgi = null; + if (appState.getRealUser() != null) { + UserGroupInformation realUserUgi = null; + realUserUgi = + UserGroupInformation.createRemoteUser(appState.getRealUser()); + userUgi = UserGroupInformation.createProxyUser(appState.getUser(), + realUserUgi); + } else { + userUgi = UserGroupInformation.createRemoteUser(appState.getUser()); + } // create and recover app. RMAppImpl application = createAndPopulateNewRMApp(appContext, appState.getSubmitTime(), - appState.getUser(), true, appState.getStartTime(), + userUgi, true, appState.getStartTime(), appState.getState()); application.handle(new RMAppRecoverEvent(appId, rmState)); @@ -406,8 +426,9 @@ protected void recoverApplication(ApplicationStateData appState, private RMAppImpl createAndPopulateNewRMApp( ApplicationSubmissionContext submissionContext, long submitTime, - String user, boolean isRecovery, long startTime, + UserGroupInformation userUgi, boolean isRecovery, long startTime, RMAppState recoveredFinalState) throws YarnException { + String user = userUgi.getShortUserName(); ApplicationPlacementContext placementContext = null; if (recoveredFinalState == null) { @@ -431,7 +452,6 @@ private RMAppImpl createAndPopulateNewRMApp( // Verify and get the update application priority and set back to // submissionContext - UserGroupInformation userUgi = UserGroupInformation.createRemoteUser(user); // Application priority needed to be validated only while submitting. During // recovery, validated priority could be recovered from submission context. @@ -519,7 +539,7 @@ private RMAppImpl createAndPopulateNewRMApp( // Create RMApp RMAppImpl application = new RMAppImpl(applicationId, rmContext, this.conf, - submissionContext.getApplicationName(), user, + submissionContext.getApplicationName(), userUgi, placementQueueName, submissionContext, this.scheduler, this.masterService, submitTime, submissionContext.getApplicationType(), diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/CSMappingPlacementRule.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/CSMappingPlacementRule.java index ef98c1a79f178..91e0138c4a094 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/CSMappingPlacementRule.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/CSMappingPlacementRule.java @@ -133,7 +133,7 @@ public boolean initialize(ResourceScheduler scheduler) throws IOException { overrideWithQueueMappings = conf.getOverrideWithQueueMappings(); if (groups == null) { - groups = Groups.getUserToGroupsMappingService(conf); + groups = Groups.getUserToGroupsMappingService(csContext.getConf()); } MappingRuleValidationContext validationContext = buildValidationContext(); @@ -535,4 +535,9 @@ private String cleanName(String name) { return name; } } + + @VisibleForTesting + public Groups getGroups() { + return groups; + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStore.java index 8a444911c81d3..a93ed1bc6b495 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStore.java @@ -937,7 +937,8 @@ public void storeNewApplication(RMApp app) { assert context instanceof ApplicationSubmissionContextPBImpl; ApplicationStateData appState = ApplicationStateData.newInstance(app.getSubmitTime(), - app.getStartTime(), context, app.getUser(), app.getCallerContext()); + app.getStartTime(), context, app.getUser(), app.getRealUser(), + app.getCallerContext()); appState.setApplicationTimeouts(app.getApplicationTimeouts()); getRMStateStoreEventHandler().handle(new RMStateStoreAppEvent(appState)); } @@ -1170,7 +1171,7 @@ public void removeApplication(RMApp app) { ApplicationStateData appState = ApplicationStateData.newInstance(app.getSubmitTime(), app.getStartTime(), app.getApplicationSubmissionContext(), - app.getUser(), app.getCallerContext()); + app.getUser(), app.getRealUser(), app.getCallerContext()); for(RMAppAttempt appAttempt : app.getAppAttempts().values()) { appState.attempts.put(appAttempt.getAppAttemptId(), null); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/records/ApplicationStateData.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/records/ApplicationStateData.java index 2b0bd2b5bad39..8b908e56d0a4e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/records/ApplicationStateData.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/records/ApplicationStateData.java @@ -92,9 +92,46 @@ public static ApplicationStateData newInstance(long submitTime, public static ApplicationStateData newInstance(long submitTime, long startTime, ApplicationSubmissionContext context, String user) { - return newInstance(submitTime, startTime, context, user, null); + return newInstance(submitTime, startTime, context, user, + (CallerContext) null); } - + + public static ApplicationStateData newInstance(long submitTime, + long startTime, String user, String realUser, + ApplicationSubmissionContext submissionContext, RMAppState state, + String diagnostics, long launchTime, long finishTime, + CallerContext callerContext) { + ApplicationStateData appState = + newInstance(submitTime, startTime, user, submissionContext, state, + diagnostics, launchTime, finishTime, callerContext); + if (realUser != null) { + appState.setRealUser(realUser); + } + return appState; + } + + public static ApplicationStateData newInstance(long submitTime, + long startTime, String user, String realUser, + ApplicationSubmissionContext submissionContext, RMAppState state, + String diagnostics, long launchTime, long finishTime, + CallerContext callerContext, + Map applicationTimeouts) { + ApplicationStateData appState = + newInstance(submitTime, startTime, user, submissionContext, state, + diagnostics, launchTime, finishTime, callerContext, applicationTimeouts); + if (realUser != null) { + appState.setRealUser(realUser); + } + return appState; + } + + public static ApplicationStateData newInstance(long submitTime, + long startTime, ApplicationSubmissionContext context, String user, + String realUser, CallerContext callerContext) { + return newInstance(submitTime, startTime, user, realUser, context, null, "", + 0, 0, callerContext); + } + public int getAttemptCount() { return attempts.size(); } @@ -213,4 +250,8 @@ public abstract void setApplicationSubmissionContext( @Public public abstract void setApplicationTimeouts( Map applicationTimeouts); + + public abstract String getRealUser(); + + public abstract void setRealUser(String realUser); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/records/impl/pb/ApplicationStateDataPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/records/impl/pb/ApplicationStateDataPBImpl.java index a6a99ee7cb062..c0f7bb02627ed 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/records/impl/pb/ApplicationStateDataPBImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/records/impl/pb/ApplicationStateDataPBImpl.java @@ -148,7 +148,22 @@ public void setUser(String user) { maybeInitBuilder(); builder.setUser(user); } - + + @Override + public String getRealUser() { + ApplicationStateDataProtoOrBuilder p = viaProto ? proto : builder; + if (!p.hasRealUser()) { + return null; + } + return (p.getRealUser()); + } + + @Override + public void setRealUser(String realUser) { + maybeInitBuilder(); + builder.setRealUser(realUser); + } + @Override public ApplicationSubmissionContext getApplicationSubmissionContext() { ApplicationStateDataProtoOrBuilder p = viaProto ? proto : builder; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMApp.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMApp.java index d8323f5497b9a..949b34e8bb504 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMApp.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMApp.java @@ -325,4 +325,6 @@ ApplicationReport createAndGetApplicationReport(String clientUserName, * @return Map of envs related to application scheduling preferences. */ Map getApplicationSchedulingEnvs(); + + String getRealUser(); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java index 6e2db905a5959..ca88b8be3281c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java @@ -134,6 +134,7 @@ public class RMAppImpl implements RMApp, Recoverable { private final RMContext rmContext; private final Configuration conf; private final String user; + private final UserGroupInformation userUgi; private final String name; private final ApplicationSubmissionContext submissionContext; private final Dispatcher dispatcher; @@ -421,7 +422,19 @@ public RMAppImpl(ApplicationId applicationId, RMContext rmContext, String applicationType, Set applicationTags, List amReqs, ApplicationPlacementContext placementContext, long startTime) { + this(applicationId, rmContext, config, name, + (user != null ? UserGroupInformation.createRemoteUser(user) : null), + queue, submissionContext, scheduler, masterService, submitTime, + applicationType, applicationTags, amReqs, placementContext, startTime); + } + public RMAppImpl(ApplicationId applicationId, RMContext rmContext, + Configuration config, String name, UserGroupInformation userUgi, + String queue, ApplicationSubmissionContext submissionContext, + YarnScheduler scheduler, ApplicationMasterService masterService, + long submitTime, String applicationType, Set applicationTags, + List amReqs, ApplicationPlacementContext + placementContext, long startTime) { this.systemClock = SystemClock.getInstance(); this.applicationId = applicationId; @@ -430,7 +443,9 @@ public RMAppImpl(ApplicationId applicationId, RMContext rmContext, this.dispatcher = rmContext.getDispatcher(); this.handler = dispatcher.getEventHandler(); this.conf = config; - this.user = StringInterner.weakIntern(user); + this.user = StringInterner.weakIntern( + (userUgi != null) ? userUgi.getShortUserName() : null); + this.userUgi = userUgi; this.queue = queue; this.submissionContext = submissionContext; this.scheduler = scheduler; @@ -1313,7 +1328,7 @@ private void rememberTargetTransitionsAndStoreState(RMAppEvent event, ApplicationStateData appState = ApplicationStateData.newInstance(this.submitTime, this.startTime, - this.user, this.submissionContext, + this.getUser(), this.getRealUser(), this.submissionContext, stateToBeStored, diags, this.launchTime, this.storedFinishTime, this.callerContext); appState.setApplicationTimeouts(this.applicationTimeouts); @@ -1902,4 +1917,10 @@ long getLogAggregationStartTime() { Clock getSystemClock() { return systemClock; } + + @Override + public String getRealUser() { + UserGroupInformation realUserUgi = this.userUgi.getRealUser(); + return (realUserUgi != null) ? realUserUgi.getShortUserName() : null; + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/QueueStateManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/QueueStateManager.java index 0127e1ecaae7a..ab0f445275ee1 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/QueueStateManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/QueueStateManager.java @@ -77,7 +77,7 @@ public synchronized void activateQueue(String queueName) throw new YarnException("The specified queue:" + queueName + " does not exist!"); } - queue.activeQueue(); + queue.activateQueue(); } /** diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerQueue.java index 9a67e0188aeae..33037fc43fba2 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerQueue.java @@ -62,8 +62,8 @@ public interface SchedulerQueue extends Queue { void stopQueue(); /** - * Active the queue. + * Activate the queue. * @throws YarnException if the queue can not be activated. */ - void activeQueue() throws YarnException; + void activateQueue() throws YarnException; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractAutoCreatedLeafQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractAutoCreatedLeafQueue.java index 8d7733453f8eb..b9c2ec6236490 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractAutoCreatedLeafQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractAutoCreatedLeafQueue.java @@ -111,7 +111,7 @@ public void setEntitlement(String nodeLabel, QueueEntitlement entitlement) } setCapacity(nodeLabel, capacity); setAbsoluteCapacity(nodeLabel, - getParent().getQueueCapacities(). + this.getParent().getQueueCapacities(). getAbsoluteCapacity(nodeLabel) * getQueueCapacities().getCapacity(nodeLabel)); // note: we currently set maxCapacity to capacity diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java index 5c520b783b943..efdfa8e5af880 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java @@ -18,7 +18,6 @@ package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity; -import org.apache.commons.lang3.StringUtils; import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.ipc.Server; import org.apache.hadoop.security.AccessControlException; @@ -37,7 +36,6 @@ import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.api.records.ResourceInformation; import org.apache.hadoop.yarn.exceptions.YarnException; -import org.apache.hadoop.yarn.exceptions.YarnRuntimeException; import org.apache.hadoop.yarn.factories.RecordFactory; import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider; import org.apache.hadoop.yarn.security.AccessRequest; @@ -77,55 +75,35 @@ import java.util.concurrent.locks.ReentrantReadWriteLock; import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration.DOT; -import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration.ROOT; -import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration.UNDEFINED; public abstract class AbstractCSQueue implements CSQueue { - private static final Logger LOG = LoggerFactory.getLogger(AbstractCSQueue.class); + protected final QueueAllocationSettings queueAllocationSettings; volatile CSQueue parent; + protected final QueuePath queuePath; final String queueName; - private final String queuePath; - volatile int numContainers; + protected QueueNodeLabelsSettings queueNodeLabelsSettings; + private volatile QueueAppLifetimeAndLimitSettings queueAppLifetimeSettings; + private CSQueuePreemptionSettings preemptionSettings; - final Resource minimumAllocation; - volatile Resource maximumAllocation; private volatile QueueState state = null; - final CSQueueMetrics metrics; protected final PrivilegedEntity queueEntity; final ResourceCalculator resourceCalculator; - Set accessibleLabels; - protected Set configuredNodeLabels; Set resourceTypes; final RMNodeLabelsManager labelManager; - String defaultLabelExpression; private String multiNodeSortingPolicyName = null; Map acls = new HashMap(); volatile boolean reservationsContinueLooking; - // Track resource usage-by-label like used-resource/pending-resource, etc. - volatile ResourceUsage queueUsage; - // Track capacities like // used-capacity/abs-used-capacity/capacity/abs-capacity, // etc. QueueCapacities queueCapacities; - - QueueResourceQuotas queueResourceQuotas; - - // -1 indicates lifetime is disabled - private volatile long maxApplicationLifetime = -1; - - private volatile long defaultApplicationLifetime = -1; - - // Indicates if this queue's default lifetime was set by a config property, - // either at this level or anywhere in the queue's hierarchy. - private volatile boolean defaultAppLifetimeWasSpecifiedInConfig = false; - private CSQueuePreemption preemptionSettings; + CSQueueUsageTracker usageTracker; public enum CapacityConfigType { // FIXME, from what I can see, Percentage mode can almost apply to weighted @@ -136,6 +114,8 @@ public enum CapacityConfigType { protected CapacityConfigType capacityConfigType = CapacityConfigType.NONE; + protected Map configuredCapacityVectors; + private final RecordFactory recordFactory = RecordFactoryProvider.getRecordFactory(null); protected CapacitySchedulerContext csContext; @@ -148,15 +128,10 @@ public enum CapacityConfigType { volatile Priority priority = Priority.newInstance(0); private UserWeights userWeights = UserWeights.createEmpty(); - private int maxParallelApps; // is it a dynamic queue? private boolean dynamicQueue = false; - // The timestamp of the last submitted application to this queue. - // Only applies to dynamic queues. - private long lastSubmittedTimestamp; - public AbstractCSQueue(CapacitySchedulerContext cs, String queueName, CSQueue parent, CSQueue old) throws IOException { this(cs, cs.getConfiguration(), queueName, parent, old); @@ -165,39 +140,35 @@ public AbstractCSQueue(CapacitySchedulerContext cs, public AbstractCSQueue(CapacitySchedulerContext cs, CapacitySchedulerConfiguration configuration, String queueName, CSQueue parent, CSQueue old) { - this.labelManager = cs.getRMContext().getNodeLabelManager(); this.parent = parent; - this.queueName = queueName; - this.queuePath = ((parent == null) ? "" : (parent.getQueuePath() + ".")) - + this.queueName; + this.queuePath = createQueuePath(parent, queueName); + this.queueName = queuePath.getLeafName(); this.resourceCalculator = cs.getResourceCalculator(); this.activitiesManager = cs.getActivitiesManager(); // must be called after parent and queueName is set - this.metrics = old != null ? + CSQueueMetrics metrics = old != null ? (CSQueueMetrics) old.getMetrics() : CSQueueMetrics.forQueue(getQueuePath(), parent, cs.getConfiguration().getEnableUserMetrics(), configuration); - + usageTracker = new CSQueueUsageTracker(metrics); this.csContext = cs; - this.minimumAllocation = csContext.getMinimumResourceCapability(); - - // initialize ResourceUsage - queueUsage = new ResourceUsage(); + this.queueAllocationSettings = new QueueAllocationSettings(csContext); queueEntity = new PrivilegedEntity(EntityType.QUEUE, getQueuePath()); - - // initialize QueueCapacities queueCapacities = new QueueCapacities(parent == null); - - // initialize queueResourceQuotas - queueResourceQuotas = new QueueResourceQuotas(); - ReentrantReadWriteLock lock = new ReentrantReadWriteLock(); readLock = lock.readLock(); writeLock = lock.writeLock(); } + private static QueuePath createQueuePath(CSQueue parent, String queueName) { + if (parent == null) { + return new QueuePath(null, queueName); + } + return new QueuePath(parent.getQueuePath(), queueName); + } + @VisibleForTesting protected void setupConfigurableCapacities() { setupConfigurableCapacities(csContext.getConfiguration()); @@ -206,12 +177,12 @@ protected void setupConfigurableCapacities() { protected void setupConfigurableCapacities( CapacitySchedulerConfiguration configuration) { CSQueueUtils.loadCapacitiesByLabelsFromConf(getQueuePath(), queueCapacities, - configuration, configuredNodeLabels); + configuration, this.queueNodeLabelsSettings.getConfiguredNodeLabels()); } @Override public String getQueuePath() { - return queuePath; + return queuePath.getFullPath(); } @Override @@ -246,11 +217,11 @@ public float getUsedCapacity() { @Override public Resource getUsedResources() { - return queueUsage.getUsed(); + return usageTracker.getQueueUsage().getUsed(); } public int getNumContainers() { - return numContainers; + return usageTracker.getNumContainers(); } @Override @@ -260,12 +231,12 @@ public QueueState getState() { @Override public CSQueueMetrics getMetrics() { - return metrics; + return usageTracker.getMetrics(); } @Override public String getQueueShortName() { - return queueName; + return queuePath.getLeafName(); } @Override @@ -273,11 +244,6 @@ public String getQueueName() { return queueName; } - @Override - public PrivilegedEntity getPrivilegedEntity() { - return queueEntity; - } - @Override public CSQueue getParent() { return parent; @@ -288,8 +254,13 @@ public void setParent(CSQueue newParentQueue) { this.parent = newParentQueue; } + @Override + public PrivilegedEntity getPrivilegedEntity() { + return queueEntity; + } + public Set getAccessibleNodeLabels() { - return accessibleLabels; + return queueNodeLabelsSettings.getAccessibleNodeLabels(); } @Override @@ -303,6 +274,7 @@ public boolean hasAccess(QueueACL acl, UserGroupInformation user) { * Set maximum capacity - used only for testing. * @param maximumCapacity new max capacity */ + @VisibleForTesting void setMaxCapacity(float maximumCapacity) { writeLock.lock(); try { @@ -346,7 +318,7 @@ void setMaxCapacity(String nodeLabel, float maximumCapacity) { @Override public String getDefaultNodeLabelExpression() { - return defaultLabelExpression; + return this.queueNodeLabelsSettings.getDefaultLabelExpression(); } protected void setupQueueConfigs(Resource clusterResource, @@ -360,7 +332,8 @@ protected void setupQueueConfigs(Resource clusterResource, } // Collect and set the Node label configuration - initializeNodeLabels(configuration); + this.queueNodeLabelsSettings = new QueueNodeLabelsSettings(configuration, parent, + getQueuePath(), csContext); // Initialize the queue capacities setupConfigurableCapacities(configuration); @@ -378,7 +351,8 @@ protected void setupQueueConfigs(Resource clusterResource, // Setup queue's maximumAllocation respecting the global // and the queue settings - setupMaximumAllocation(configuration); + this.queueAllocationSettings.setupMaximumAllocation(configuration, getQueuePath(), + parent, csContext); // Initialize the queue state based on previous state, configured state // and its parent state @@ -392,13 +366,16 @@ protected void setupQueueConfigs(Resource clusterResource, this.reservationsContinueLooking = configuration.getReservationContinueLook(); + this.configuredCapacityVectors = csContext.getConfiguration() + .parseConfiguredResourceVector(queuePath.getFullPath(), + this.queueNodeLabelsSettings.getConfiguredNodeLabels()); // Update metrics CSQueueUtils.updateQueueStatistics(resourceCalculator, clusterResource, this, labelManager, null); // Store preemption settings - this.preemptionSettings = new CSQueuePreemption(this, csContext, configuration); + this.preemptionSettings = new CSQueuePreemptionSettings(this, csContext, configuration); this.priority = configuration.getQueuePriority( getQueuePath()); @@ -407,7 +384,8 @@ protected void setupQueueConfigs(Resource clusterResource, configuration.getMultiNodesSortingAlgorithmPolicy(getQueuePath())); // Setup application related limits - setupApplicationLimits(configuration); + this.queueAppLifetimeSettings = new QueueAppLifetimeAndLimitSettings(configuration, + this, getQueuePath()); } finally { writeLock.unlock(); } @@ -420,11 +398,11 @@ protected void setupQueueConfigs(Resource clusterResource, protected void setDynamicQueueProperties( CapacitySchedulerConfiguration configuration) { // Set properties from parent template - if (getParent() instanceof ParentQueue) { - ((ParentQueue) getParent()).getAutoCreatedQueueTemplate() + if (parent instanceof ParentQueue) { + ((ParentQueue) parent).getAutoCreatedQueueTemplate() .setTemplateEntriesForChild(configuration, getQueuePath()); - String parentTemplate = String.format("%s.%s", getParent().getQueuePath(), + String parentTemplate = String.format("%s.%s", parent.getQueuePath(), AutoCreatedQueueTemplate.AUTO_QUEUE_TEMPLATE_PREFIX); parentTemplate = parentTemplate.substring(0, parentTemplate.lastIndexOf( DOT)); @@ -434,135 +412,7 @@ protected void setDynamicQueueProperties( if (parentNodeLabels != null && parentNodeLabels.size() > 1) { csContext.getCapacitySchedulerQueueManager().getConfiguredNodeLabels() - .setLabelsByQueue(queuePath, new HashSet<>(parentNodeLabels)); - } - } - } - - private void initializeNodeLabels( - CapacitySchedulerConfiguration configuration) throws IOException { - // Collect and store labels - this.accessibleLabels = - configuration.getAccessibleNodeLabels(getQueuePath()); - this.defaultLabelExpression = - configuration.getDefaultNodeLabelExpression( - getQueuePath()); - - // Inherit labels from parent if not set - if (this.accessibleLabels == null && parent != null) { - this.accessibleLabels = parent.getAccessibleNodeLabels(); - } - - // If the accessible labels is not null and the queue has a parent with a - // similar set of labels copy the defaultNodeLabelExpression from the parent - if (this.accessibleLabels != null && parent != null - && this.defaultLabelExpression == null && - this.accessibleLabels.containsAll(parent.getAccessibleNodeLabels())) { - this.defaultLabelExpression = parent.getDefaultNodeLabelExpression(); - } - - if (csContext.getCapacitySchedulerQueueManager() != null - && csContext.getCapacitySchedulerQueueManager() - .getConfiguredNodeLabels() != null) { - if (getQueuePath().equals(ROOT)) { - this.configuredNodeLabels = csContext.getCapacitySchedulerQueueManager() - .getConfiguredNodeLabels().getAllConfiguredLabels(); - } else { - this.configuredNodeLabels = csContext.getCapacitySchedulerQueueManager() - .getConfiguredNodeLabels().getLabelsByQueue(getQueuePath()); - } - } else { - // Fallback to suboptimal but correct logic - this.configuredNodeLabels = csContext.getConfiguration() - .getConfiguredNodeLabels(queuePath); - } - - // Validate the initialized settings - validateNodeLabels(); - } - - private void validateNodeLabels() throws IOException { - // Check if labels of this queue is a subset of parent queue, only do this - // when the queue in question is not root - if (parent != null && parent.getParent() != null) { - if (parent.getAccessibleNodeLabels() != null && !parent - .getAccessibleNodeLabels().contains(RMNodeLabelsManager.ANY)) { - // if parent isn't "*", child shouldn't be "*" too - if (this.getAccessibleNodeLabels().contains( - RMNodeLabelsManager.ANY)) { - throw new IOException("Parent's accessible queue is not ANY(*), " - + "but child's accessible queue is *"); - } else{ - Set diff = Sets.difference(this.getAccessibleNodeLabels(), - parent.getAccessibleNodeLabels()); - if (!diff.isEmpty()) { - throw new IOException( - "Some labels of child queue is not a subset " - + "of parent queue, these labels=[" + StringUtils - .join(diff, ",") + "]"); - } - } - } - } - } - - private void setupApplicationLimits(CapacitySchedulerConfiguration configuration) { - // Store max parallel apps property - this.maxParallelApps = configuration.getMaxParallelAppsForQueue(getQueuePath()); - - maxApplicationLifetime = getInheritedMaxAppLifetime(this, configuration); - defaultApplicationLifetime = - getInheritedDefaultAppLifetime(this, configuration, - maxApplicationLifetime); - } - - private void setupMaximumAllocation(CapacitySchedulerConfiguration csConf) { - String myQueuePath = getQueuePath(); - /* YARN-10869: When using AutoCreatedLeafQueues, the passed configuration - * object is a cloned one containing only the template configs - * (see ManagedParentQueue#getLeafQueueConfigs). To ensure that the actual - * cluster maximum allocation is fetched the original config object should - * be used. - */ - Resource clusterMax = ResourceUtils - .fetchMaximumAllocationFromConfig(this.csContext.getConfiguration()); - Resource queueMax = csConf.getQueueMaximumAllocation(myQueuePath); - - maximumAllocation = Resources.clone( - parent == null ? clusterMax : parent.getMaximumAllocation()); - - String errMsg = - "Queue maximum allocation cannot be larger than the cluster setting" - + " for queue " + myQueuePath - + " max allocation per queue: %s" - + " cluster setting: " + clusterMax; - - if (queueMax == Resources.none()) { - // Handle backward compatibility - long queueMemory = csConf.getQueueMaximumAllocationMb(myQueuePath); - int queueVcores = csConf.getQueueMaximumAllocationVcores(myQueuePath); - if (queueMemory != UNDEFINED) { - maximumAllocation.setMemorySize(queueMemory); - } - - if (queueVcores != UNDEFINED) { - maximumAllocation.setVirtualCores(queueVcores); - } - - if ((queueMemory != UNDEFINED && queueMemory > clusterMax.getMemorySize() - || (queueVcores != UNDEFINED - && queueVcores > clusterMax.getVirtualCores()))) { - throw new IllegalArgumentException( - String.format(errMsg, maximumAllocation)); - } - } else { - // Queue level maximum-allocation can't be larger than cluster setting - for (ResourceInformation ri : queueMax.getResources()) { - if (ri.compareTo(clusterMax.getResourceInformation(ri.getName())) > 0) { - throw new IllegalArgumentException(String.format(errMsg, queueMax)); - } - - maximumAllocation.setResourceInformation(ri.getName(), ri); + .setLabelsByQueue(getQueuePath(), new HashSet<>(parentNodeLabels)); } } } @@ -570,7 +420,7 @@ private void setupMaximumAllocation(CapacitySchedulerConfiguration csConf) { private UserWeights getUserWeightsFromHierarchy( CapacitySchedulerConfiguration configuration) { UserWeights unionInheritedWeights = UserWeights.createEmpty(); - CSQueue parentQ = getParent(); + CSQueue parentQ = parent; if (parentQ != null) { // Inherit all of parent's userWeights unionInheritedWeights.addFrom(parentQ.getUserWeights()); @@ -602,12 +452,12 @@ protected boolean checkConfigTypeIsAbsoluteResource(String queuePath, protected void updateCapacityConfigType() { this.capacityConfigType = CapacityConfigType.NONE; - for (String label : configuredNodeLabels) { + for (String label : queueNodeLabelsSettings.getConfiguredNodeLabels()) { LOG.debug("capacityConfigType is '{}' for queue {}", capacityConfigType, getQueuePath()); CapacityConfigType localType = checkConfigTypeIsAbsoluteResource( - queuePath, label) ? CapacityConfigType.ABSOLUTE_RESOURCE + getQueuePath(), label) ? CapacityConfigType.ABSOLUTE_RESOURCE : CapacityConfigType.PERCENTAGE; if (this.capacityConfigType.equals(CapacityConfigType.NONE)) { @@ -621,12 +471,13 @@ protected void updateCapacityConfigType() { } protected void updateConfigurableResourceLimits(Resource clusterResource) { - for (String label : configuredNodeLabels) { - final Resource minResource = getMinimumAbsoluteResource(queuePath, label); - Resource maxResource = getMaximumAbsoluteResource(queuePath, label); + for (String label : queueNodeLabelsSettings.getConfiguredNodeLabels()) { + final Resource minResource = getMinimumAbsoluteResource(getQueuePath(), label); + Resource maxResource = getMaximumAbsoluteResource(getQueuePath(), label); if (parent != null) { - final Resource parentMax = parent.getQueueResourceQuotas().getConfiguredMaxResource(label); + final Resource parentMax = parent.getQueueResourceQuotas() + .getConfiguredMaxResource(label); validateMinResourceIsNotGreaterThanMaxResource(maxResource, parentMax, clusterResource, "Max resource configuration " + maxResource + " is greater than parents max value:" @@ -650,8 +501,8 @@ protected void updateConfigurableResourceLimits(Resource clusterResource) { + " minResource={} and maxResource={}", getQueuePath(), minResource, maxResource); - queueResourceQuotas.setConfiguredMinResource(label, minResource); - queueResourceQuotas.setConfiguredMaxResource(label, maxResource); + usageTracker.getQueueResourceQuotas().setConfiguredMinResource(label, minResource); + usageTracker.getQueueResourceQuotas().setConfiguredMaxResource(label, maxResource); } } @@ -667,7 +518,7 @@ private void validateMinResourceIsNotGreaterThanMaxResource(Resource minResource private void validateAbsoluteVsPercentageCapacityConfig( CapacityConfigType localType) { - if (!queuePath.equals("root") + if (!getQueuePath().equals("root") && !this.capacityConfigType.equals(localType)) { throw new IllegalArgumentException("Queue '" + getQueuePath() + "' should use either percentage based capacity" @@ -690,7 +541,7 @@ public Resource getEffectiveCapacity(String label) { public Resource getEffectiveCapacityDown(String label, Resource factor) { return Resources.normalizeDown(resourceCalculator, getQueueResourceQuotas().getEffectiveMinResource(label), - minimumAllocation); + queueAllocationSettings.getMinimumAllocation()); } @Override @@ -703,7 +554,13 @@ public Resource getEffectiveMaxCapacity(String label) { public Resource getEffectiveMaxCapacityDown(String label, Resource factor) { return Resources.normalizeDown(resourceCalculator, getQueueResourceQuotas().getEffectiveMaxResource(label), - minimumAllocation); + queueAllocationSettings.getMinimumAllocation()); + } + + @Override + public QueueCapacityVector getConfiguredCapacityVector( + String label) { + return configuredCapacityVectors.get(label); } private void initializeQueueState(CapacitySchedulerConfiguration configuration) { @@ -736,7 +593,7 @@ private void initializeQueueState(CapacitySchedulerConfiguration configuration) && parentState != QueueState.RUNNING) { throw new IllegalArgumentException( "The parent queue:" + parent.getQueuePath() - + " cannot be STOPPED as the child queue:" + queuePath + + " cannot be STOPPED as the child queue:" + getQueuePath() + " is in RUNNING state."); } else { updateQueueState(configuredState); @@ -751,7 +608,7 @@ private void initializeQueueState(CapacitySchedulerConfiguration configuration) } else { if (configuredState == QueueState.RUNNING) { try { - activeQueue(); + activateQueue(); } catch (YarnException ex) { throw new IllegalArgumentException(ex.getMessage()); } @@ -767,20 +624,20 @@ protected QueueInfo getQueueInfo() { // TODO, improve this QueueInfo queueInfo = recordFactory.newRecordInstance(QueueInfo.class); queueInfo.setQueueName(queueName); - queueInfo.setQueuePath(queuePath); - queueInfo.setAccessibleNodeLabels(accessibleLabels); + queueInfo.setQueuePath(getQueuePath()); + queueInfo.setAccessibleNodeLabels(queueNodeLabelsSettings.getAccessibleNodeLabels()); queueInfo.setCapacity(queueCapacities.getCapacity()); queueInfo.setMaximumCapacity(queueCapacities.getMaximumCapacity()); queueInfo.setQueueState(getState()); - queueInfo.setDefaultNodeLabelExpression(defaultLabelExpression); + queueInfo.setDefaultNodeLabelExpression(queueNodeLabelsSettings.getDefaultLabelExpression()); queueInfo.setCurrentCapacity(getUsedCapacity()); queueInfo.setQueueStatistics(getQueueStatistics()); - queueInfo.setPreemptionDisabled(getPreemptionDisabled()); + queueInfo.setPreemptionDisabled(preemptionSettings.isPreemptionDisabled()); queueInfo.setIntraQueuePreemptionDisabled( getIntraQueuePreemptionDisabled()); queueInfo.setQueueConfigurations(getQueueConfigurations()); queueInfo.setWeight(queueCapacities.getWeight()); - queueInfo.setMaxParallelApps(maxParallelApps); + queueInfo.setMaxParallelApps(queueAppLifetimeSettings.getMaxParallelApps()); return queueInfo; } @@ -815,6 +672,7 @@ public QueueStatistics getQueueStatistics() { public Map getQueueConfigurations() { Map queueConfigurations = new HashMap<>(); Set nodeLabels = getNodeLabelsForQueue(); + QueueResourceQuotas queueResourceQuotas = usageTracker.getQueueResourceQuotas(); for (String nodeLabel : nodeLabels) { QueueConfigurations queueConfiguration = recordFactory.newRecordInstance(QueueConfigurations.class); @@ -845,22 +703,20 @@ public Map getQueueConfigurations() { @Private public Resource getMaximumAllocation() { - return maximumAllocation; + return queueAllocationSettings.getMaximumAllocation(); } @Private public Resource getMinimumAllocation() { - return minimumAllocation; + return queueAllocationSettings.getMinimumAllocation(); } void allocateResource(Resource clusterResource, Resource resource, String nodePartition) { writeLock.lock(); try { - queueUsage.incUsed(nodePartition, resource); - - ++numContainers; - + usageTracker.getQueueUsage().incUsed(nodePartition, resource); + usageTracker.increaseNumContainers(); CSQueueUtils.updateQueueStatistics(resourceCalculator, clusterResource, this, labelManager, nodePartition); } finally { @@ -872,12 +728,12 @@ protected void releaseResource(Resource clusterResource, Resource resource, String nodePartition) { writeLock.lock(); try { - queueUsage.decUsed(nodePartition, resource); + usageTracker.getQueueUsage().decUsed(nodePartition, resource); CSQueueUtils.updateQueueStatistics(resourceCalculator, clusterResource, this, labelManager, nodePartition); - --numContainers; + usageTracker.decreaseNumContainers(); } finally { writeLock.unlock(); } @@ -905,8 +761,7 @@ public boolean getPreemptionDisabled() { @Private public boolean getIntraQueuePreemptionDisabled() { - return preemptionSettings.isIntraQueuePreemptionDisabledInHierarchy() || - preemptionSettings.isPreemptionDisabled(); + return preemptionSettings.getIntraQueuePreemptionDisabled(); } @Private @@ -921,12 +776,12 @@ public QueueCapacities getQueueCapacities() { @Private public ResourceUsage getQueueResourceUsage() { - return queueUsage; + return usageTracker.getQueueUsage(); } @Override public QueueResourceQuotas getQueueResourceQuotas() { - return queueResourceQuotas; + return usageTracker.getQueueResourceQuotas(); } @Override @@ -934,76 +789,6 @@ public ReentrantReadWriteLock.ReadLock getReadLock() { return readLock; } - private long getInheritedMaxAppLifetime(CSQueue q, - CapacitySchedulerConfiguration conf) { - CSQueue parentQ = q.getParent(); - long maxAppLifetime = conf.getMaximumLifetimePerQueue(q.getQueuePath()); - - // If q is the root queue, then get max app lifetime from conf. - if (parentQ == null) { - return maxAppLifetime; - } - - // If this is not the root queue, get this queue's max app lifetime - // from the conf. The parent's max app lifetime will be used if it's - // not set for this queue. - // A value of 0 will override the parent's value and means no max lifetime. - // A negative value means that the parent's max should be used. - long parentsMaxAppLifetime = getParent().getMaximumApplicationLifetime(); - return (maxAppLifetime >= 0) ? maxAppLifetime : parentsMaxAppLifetime; - } - - private long getInheritedDefaultAppLifetime(CSQueue q, - CapacitySchedulerConfiguration conf, long myMaxAppLifetime) { - CSQueue parentQ = q.getParent(); - long defaultAppLifetime = conf.getDefaultLifetimePerQueue(getQueuePath()); - defaultAppLifetimeWasSpecifiedInConfig = - (defaultAppLifetime >= 0 - || (parentQ != null && - parentQ.getDefaultAppLifetimeWasSpecifiedInConfig())); - - // If q is the root queue, then get default app lifetime from conf. - if (parentQ == null) { - return defaultAppLifetime; - } - - // If this is not the root queue, get the parent's default app lifetime. The - // parent's default app lifetime will be used if not set for this queue. - long parentsDefaultAppLifetime = - getParent().getDefaultApplicationLifetime(); - - // Negative value indicates default lifetime was not set at this level. - // If default lifetime was not set at this level, calculate it based on - // parent's default lifetime or current queue's max lifetime. - if (defaultAppLifetime < 0) { - // If default lifetime was not set at this level but was set somewhere in - // the parent's hierarchy, set default lifetime to parent queue's default - // only if parent queue's lifetime is less than current queue's max - // lifetime. Otherwise, use current queue's max lifetime value for its - // default lifetime. - if (defaultAppLifetimeWasSpecifiedInConfig) { - defaultAppLifetime = - Math.min(parentsDefaultAppLifetime, myMaxAppLifetime); - } else { - // Default app lifetime value was not set anywhere in this queue's - // hierarchy. Use current queue's max lifetime as its default. - defaultAppLifetime = myMaxAppLifetime; - } - } // else if >= 0, default lifetime was set at this level. Just use it. - - if (myMaxAppLifetime > 0 && - defaultAppLifetime > myMaxAppLifetime) { - throw new YarnRuntimeException( - "Default lifetime " + defaultAppLifetime - + " can't exceed maximum lifetime " + myMaxAppLifetime); - } - - if (defaultAppLifetime <= 0) { - defaultAppLifetime = myMaxAppLifetime; - } - return defaultAppLifetime; - } - private Resource getCurrentLimitResource(String nodePartition, Resource clusterResource, ResourceLimits currentResourceLimits, SchedulingMode schedulingMode) { @@ -1032,7 +817,8 @@ Resource getQueueMaxResource(String nodePartition) { return getEffectiveMaxCapacity(nodePartition); } - public boolean hasChildQueues() { + @VisibleForTesting + boolean hasChildQueues() { List childQueues = getChildQueues(); return childQueues != null && !childQueues.isEmpty(); } @@ -1056,7 +842,7 @@ boolean canAssignToThisQueue(Resource clusterResource, Resource currentLimitResource = getCurrentLimitResource(nodePartition, clusterResource, currentResourceLimits, schedulingMode); - Resource nowTotalUsed = queueUsage.getUsed(nodePartition); + Resource nowTotalUsed = usageTracker.getQueueUsage().getUsed(nodePartition); // Set headroom for currentResourceLimits: // When queue is a parent queue: Headroom = limit - used + killable @@ -1088,7 +874,7 @@ boolean canAssignToThisQueue(Resource clusterResource, newTotalWithoutReservedResource, currentLimitResource)) { if (LOG.isDebugEnabled()) { LOG.debug("try to use reserved: " + getQueuePath() - + " usedResources: " + queueUsage.getUsed() + + " usedResources: " + usageTracker.getQueueUsage().getUsed() + ", clusterResources: " + clusterResource + ", reservedResources: " + resourceCouldBeUnreserved + ", capacity-without-reserved: " @@ -1103,7 +889,7 @@ boolean canAssignToThisQueue(Resource clusterResource, if (LOG.isDebugEnabled()) { LOG.debug("Failed to assign to queue: " + getQueuePath() + " nodePartition: " + nodePartition - + ", usedResources: " + queueUsage.getUsed(nodePartition) + + ", usedResources: " + usageTracker.getQueueUsage().getUsed(nodePartition) + ", clusterResources: " + clusterResource + ", reservedResources: " + resourceCouldBeUnreserved + ", maxLimitCapacity: " + currentLimitResource @@ -1114,11 +900,11 @@ boolean canAssignToThisQueue(Resource clusterResource, if (LOG.isDebugEnabled()) { LOG.debug("Check assign to queue: " + getQueuePath() + " nodePartition: " + nodePartition - + ", usedResources: " + queueUsage.getUsed(nodePartition) + + ", usedResources: " + usageTracker.getQueueUsage().getUsed(nodePartition) + ", clusterResources: " + clusterResource + ", currentUsedCapacity: " + Resources .divide(resourceCalculator, clusterResource, - queueUsage.getUsed(nodePartition), labelManager + usageTracker.getQueueUsage().getUsed(nodePartition), labelManager .getResourceByLabel(nodePartition, clusterResource)) + ", max-capacity: " + queueCapacities .getAbsoluteMaximumCapacity(nodePartition)); @@ -1162,39 +948,39 @@ private void countAndUpdate(String partition, Resource resource, @Override public void incReservedResource(String partition, Resource reservedRes) { - count(partition, reservedRes, queueUsage::incReserved, + count(partition, reservedRes, usageTracker.getQueueUsage()::incReserved, parent == null ? null : parent::incReservedResource); } @Override public void decReservedResource(String partition, Resource reservedRes) { - count(partition, reservedRes, queueUsage::decReserved, + count(partition, reservedRes, usageTracker.getQueueUsage()::decReserved, parent == null ? null : parent::decReservedResource); } @Override public void incPendingResource(String nodeLabel, Resource resourceToInc) { - count(nodeLabel, resourceToInc, queueUsage::incPending, + count(nodeLabel, resourceToInc, usageTracker.getQueueUsage()::incPending, parent == null ? null : parent::incPendingResource); } @Override public void decPendingResource(String nodeLabel, Resource resourceToDec) { - count(nodeLabel, resourceToDec, queueUsage::decPending, + count(nodeLabel, resourceToDec, usageTracker.getQueueUsage()::decPending, parent == null ? null : parent::decPendingResource); } @Override public void incUsedResource(String nodeLabel, Resource resourceToInc, SchedulerApplicationAttempt application) { - countAndUpdate(nodeLabel, resourceToInc, queueUsage::incUsed, + countAndUpdate(nodeLabel, resourceToInc, usageTracker.getQueueUsage()::incUsed, parent == null ? null : parent::incUsedResource); } @Override public void decUsedResource(String nodeLabel, Resource resourceToDec, SchedulerApplicationAttempt application) { - countAndUpdate(nodeLabel, resourceToDec, queueUsage::decUsed, + countAndUpdate(nodeLabel, resourceToDec, usageTracker.getQueueUsage()::decUsed, parent == null ? null : parent::decUsedResource); } @@ -1205,26 +991,7 @@ public void decUsedResource(String nodeLabel, Resource resourceToDec, boolean hasPendingResourceRequest(String nodePartition, Resource cluster, SchedulingMode schedulingMode) { return SchedulerUtils.hasPendingResourceRequest(resourceCalculator, - queueUsage, nodePartition, cluster, schedulingMode); - } - - public boolean accessibleToPartition(String nodePartition) { - // if queue's label is *, it can access any node - if (accessibleLabels != null - && accessibleLabels.contains(RMNodeLabelsManager.ANY)) { - return true; - } - // any queue can access to a node without label - if (nodePartition == null - || nodePartition.equals(RMNodeLabelsManager.NO_LABEL)) { - return true; - } - // a queue can access to a node only if it contains any label of the node - if (accessibleLabels != null && accessibleLabels.contains(nodePartition)) { - return true; - } - // sorry, you cannot access - return false; + usageTracker.getQueueUsage(), nodePartition, cluster, schedulingMode); } @Override @@ -1304,10 +1071,10 @@ public boolean accept(Resource cluster, schedulerContainer.getNodePartition(), cluster); } if (!Resources.fitsIn(resourceCalculator, - Resources.add(queueUsage.getUsed(partition), netAllocated), + Resources.add(usageTracker.getQueueUsage().getUsed(partition), netAllocated), maxResourceLimit)) { if (LOG.isDebugEnabled()) { - LOG.debug("Used resource=" + queueUsage.getUsed(partition) + LOG.debug("Used resource=" + usageTracker.getQueueUsage().getUsed(partition) + " exceeded maxResourceLimit of the queue =" + maxResourceLimit); } @@ -1341,14 +1108,14 @@ public void updateQueueState(QueueState queueState) { } @Override - public void activeQueue() throws YarnException { + public void activateQueue() throws YarnException { this.writeLock.lock(); try { if (getState() == QueueState.RUNNING) { LOG.info("The specified queue:" + getQueuePath() + " is already in the RUNNING state."); } else { - CSQueue parentQueue = getParent(); + CSQueue parentQueue = parent; if (parentQueue == null || parentQueue.getState() == QueueState.RUNNING) { updateQueueState(QueueState.RUNNING); } else { @@ -1391,8 +1158,8 @@ public void recoverDrainingState() { updateQueueState(QueueState.DRAINING); } LOG.info("Recover draining state for queue " + this.getQueuePath()); - if (getParent() != null && getParent().getState() == QueueState.STOPPED) { - ((AbstractCSQueue) getParent()).recoverDrainingState(); + if (parent != null && parent.getState() == QueueState.STOPPED) { + ((AbstractCSQueue) parent).recoverDrainingState(); } } finally { this.writeLock.unlock(); @@ -1409,24 +1176,24 @@ public void setMultiNodeSortingPolicyName(String policyName) { } public long getMaximumApplicationLifetime() { - return maxApplicationLifetime; + return queueAppLifetimeSettings.getMaxApplicationLifetime(); } public long getDefaultApplicationLifetime() { - return defaultApplicationLifetime; + return queueAppLifetimeSettings.getDefaultApplicationLifetime(); } public boolean getDefaultAppLifetimeWasSpecifiedInConfig() { - return defaultAppLifetimeWasSpecifiedInConfig; + return queueAppLifetimeSettings.isDefaultAppLifetimeWasSpecifiedInConfig(); } public void setMaxParallelApps(int maxParallelApps) { - this.maxParallelApps = maxParallelApps; + this.queueAppLifetimeSettings.setMaxParallelApps(maxParallelApps); } @Override public int getMaxParallelApps() { - return maxParallelApps; + return this.queueAppLifetimeSettings.getMaxParallelApps(); } abstract int getNumRunnableApps(); @@ -1454,7 +1221,7 @@ private Resource createNormalizedMinResource(Resource minResource, ret.setResourceValue(i, (long) (nResourceInformation.getValue() * ratio)); if (LOG.isDebugEnabled()) { - LOG.debug("Updating min resource for Queue: " + queuePath + " as " + ret + LOG.debug("Updating min resource for Queue: " + getQueuePath() + " as " + ret .getResourceInformation(i) + ", Actual resource: " + nResourceInformation.getValue() + ", ratio: " + ratio); } @@ -1481,60 +1248,13 @@ private Resource getOrInheritMaxResource(Resource resourceByLabel, String label) configuredMaxResource, parentMaxResource)); } - void updateMaxAppRelatedField(CapacitySchedulerConfiguration conf, - LeafQueue leafQueue) { - int maxApplications = conf.getMaximumApplicationsPerQueue(queuePath); - int maxGlobalPerQueueApps = conf.getGlobalMaximumApplicationsPerQueue(); - String maxLabel = RMNodeLabelsManager.NO_LABEL; - - if (maxApplications < 0) { - for (String label : configuredNodeLabels) { - int maxApplicationsByLabel = 0; - if (maxGlobalPerQueueApps > 0) { - // In absolute mode, should - // shrink when change to corresponding label capacity. - maxApplicationsByLabel = this.capacityConfigType - != CapacityConfigType.ABSOLUTE_RESOURCE ? - maxGlobalPerQueueApps : - (int) (maxGlobalPerQueueApps * queueCapacities - .getAbsoluteCapacity(label)); - } else { - maxApplicationsByLabel = (int) (conf.getMaximumSystemApplications() - * queueCapacities.getAbsoluteCapacity(label)); - } - if (maxApplicationsByLabel > maxApplications) { - maxApplications = maxApplicationsByLabel; - maxLabel = label; - } - } - } - leafQueue.setMaxApplications(maxApplications); - - int maxApplicationsPerUser = Math.min(maxApplications, - (int) (maxApplications - * (leafQueue.getUsersManager().getUserLimit() / 100.0f) - * leafQueue.getUsersManager().getUserLimitFactor())); - if (leafQueue.getUsersManager().getUserLimitFactor() == -1) { - maxApplicationsPerUser = maxApplications; - } - - leafQueue.setMaxApplicationsPerUser(maxApplicationsPerUser); - LOG.info("LeafQueue:" + leafQueue.getQueuePath() + - "update max app related, maxApplications=" - + maxApplications + ", maxApplicationsPerUser=" - + maxApplicationsPerUser + ", Abs Cap:" + queueCapacities - .getAbsoluteCapacity(maxLabel) + ", Cap: " + queueCapacities - .getCapacity(maxLabel) + ", MaxCap : " + queueCapacities - .getMaximumCapacity(maxLabel)); - } - void deriveCapacityFromAbsoluteConfigurations(String label, Resource clusterResource) { // Update capacity with a float calculated from the parent's minResources // and the recently changed queue minResources. // capacity = effectiveMinResource / {parent's effectiveMinResource} float result = resourceCalculator.divide(clusterResource, - queueResourceQuotas.getEffectiveMinResource(label), + usageTracker.getQueueResourceQuotas().getEffectiveMinResource(label), parent.getQueueResourceQuotas().getEffectiveMinResource(label)); queueCapacities.setCapacity(label, Float.isInfinite(result) ? 0 : result); @@ -1543,7 +1263,7 @@ void deriveCapacityFromAbsoluteConfigurations(String label, // and the recently changed queue maxResources. // maxCapacity = effectiveMaxResource / parent's effectiveMaxResource result = resourceCalculator.divide(clusterResource, - queueResourceQuotas.getEffectiveMaxResource(label), + usageTracker.getQueueResourceQuotas().getEffectiveMaxResource(label), parent.getQueueResourceQuotas().getEffectiveMaxResource(label)); queueCapacities.setMaximumCapacity(label, Float.isInfinite(result) ? 0 : result); @@ -1567,7 +1287,7 @@ void deriveCapacityFromAbsoluteConfigurations(String label, } void updateEffectiveResources(Resource clusterResource) { - for (String label : configuredNodeLabels) { + for (String label : queueNodeLabelsSettings.getConfiguredNodeLabels()) { Resource resourceByLabel = labelManager.getResourceByLabel(label, clusterResource); Resource newEffectiveMinResource; @@ -1577,7 +1297,7 @@ void updateEffectiveResources(Resource clusterResource) { if (getCapacityConfigType().equals( CapacityConfigType.ABSOLUTE_RESOURCE)) { newEffectiveMinResource = createNormalizedMinResource( - queueResourceQuotas.getConfiguredMinResource(label), + usageTracker.getQueueResourceQuotas().getConfiguredMinResource(label), ((ParentQueue) parent).getEffectiveMinRatioPerResource()); // Max resource of a queue should be the minimum of {parent's maxResources, @@ -1597,13 +1317,13 @@ void updateEffectiveResources(Resource clusterResource) { } // Update the effective min - queueResourceQuotas.setEffectiveMinResource(label, + usageTracker.getQueueResourceQuotas().setEffectiveMinResource(label, newEffectiveMinResource); - queueResourceQuotas.setEffectiveMaxResource(label, + usageTracker.getQueueResourceQuotas().setEffectiveMaxResource(label, newEffectiveMaxResource); if (LOG.isDebugEnabled()) { - LOG.debug("Updating queue:" + queuePath + LOG.debug("Updating queue:" + getQueuePath() + " with effective minimum resource=" + newEffectiveMinResource + "and effective maximum resource=" + newEffectiveMaxResource); @@ -1664,30 +1384,31 @@ public boolean isInactiveDynamicQueue() { getAutoExpiredDeletionTime()); } - public void updateLastSubmittedTimeStamp() { + void updateLastSubmittedTimeStamp() { writeLock.lock(); try { - this.lastSubmittedTimestamp = Time.monotonicNow(); + usageTracker.setLastSubmittedTimestamp(Time.monotonicNow()); } finally { writeLock.unlock(); } } - public long getLastSubmittedTimestamp() { + @VisibleForTesting + long getLastSubmittedTimestamp() { readLock.lock(); try { - return lastSubmittedTimestamp; + return usageTracker.getLastSubmittedTimestamp(); } finally { readLock.unlock(); } } @VisibleForTesting - public void setLastSubmittedTimestamp(long lastSubmittedTimestamp) { + void setLastSubmittedTimestamp(long lastSubmittedTimestamp) { writeLock.lock(); try { - this.lastSubmittedTimestamp = lastSubmittedTimestamp; + usageTracker.setLastSubmittedTimestamp(lastSubmittedTimestamp); } finally { writeLock.unlock(); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueue.java index 03a5afbee51c2..2acc1d4b9baf2 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueue.java @@ -419,6 +419,14 @@ public void validateSubmitApplication(ApplicationId applicationId, */ Resource getEffectiveCapacity(String label); + /** + * Get configured capacity resource vector parsed from the capacity config + * of the queue. + * @param label node label (partition) + * @return capacity resource vector + */ + QueueCapacityVector getConfiguredCapacityVector(String label); + /** * Get effective capacity of queue. If min/max resource is configured, * preference will be given to absolute configuration over normal capacity. diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueuePreemption.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueuePreemptionSettings.java similarity index 96% rename from hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueuePreemption.java rename to hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueuePreemptionSettings.java index 0a11bcd62932c..2cfd5a4310f82 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueuePreemption.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueuePreemptionSettings.java @@ -18,13 +18,13 @@ import org.apache.hadoop.yarn.conf.YarnConfiguration; -public class CSQueuePreemption { +public class CSQueuePreemptionSettings { private final boolean preemptionDisabled; // Indicates if the in-queue preemption setting is ever disabled within the // hierarchy of this queue. private final boolean intraQueuePreemptionDisabledInHierarchy; - public CSQueuePreemption( + public CSQueuePreemptionSettings( CSQueue queue, CapacitySchedulerContext csContext, CapacitySchedulerConfiguration configuration) { @@ -109,6 +109,10 @@ private boolean isIntraQueueHierarchyPreemptionDisabled(CSQueue q, parentQ.getIntraQueuePreemptionDisabledInHierarchy()); } + public boolean getIntraQueuePreemptionDisabled() { + return intraQueuePreemptionDisabledInHierarchy || preemptionDisabled; + } + public boolean isIntraQueuePreemptionDisabledInHierarchy() { return intraQueuePreemptionDisabledInHierarchy; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueUsageTracker.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueUsageTracker.java new file mode 100644 index 0000000000000..0f18e944e9a38 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueUsageTracker.java @@ -0,0 +1,78 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity; + +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueResourceQuotas; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceUsage; + +public class CSQueueUsageTracker { + private final CSQueueMetrics metrics; + private int numContainers; + + /** + * The timestamp of the last submitted application to this queue. + * Only applies to dynamic queues. + */ + private long lastSubmittedTimestamp; + + /** + * Tracks resource usage by label like used-resource / pending-resource. + */ + private final ResourceUsage queueUsage; + + private final QueueResourceQuotas queueResourceQuotas; + + public CSQueueUsageTracker(CSQueueMetrics metrics) { + this.metrics = metrics; + this.queueUsage = new ResourceUsage(); + this.queueResourceQuotas = new QueueResourceQuotas(); + } + + public int getNumContainers() { + return numContainers; + } + + public synchronized void increaseNumContainers() { + numContainers++; + } + + public synchronized void decreaseNumContainers() { + numContainers--; + } + + public CSQueueMetrics getMetrics() { + return metrics; + } + + public long getLastSubmittedTimestamp() { + return lastSubmittedTimestamp; + } + + public void setLastSubmittedTimestamp(long lastSubmittedTimestamp) { + this.lastSubmittedTimestamp = lastSubmittedTimestamp; + } + + public ResourceUsage getQueueUsage() { + return queueUsage; + } + + public QueueResourceQuotas getQueueResourceQuotas() { + return queueResourceQuotas; + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueUtils.java index f43ea30001788..410117a912702 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueUtils.java @@ -92,15 +92,6 @@ public static void loadCapacitiesByLabelsFromConf( queueCapacities.setWeight(label, csConf.getLabeledQueueWeight(queuePath, label)); } - - /*float absCapacity = queueCapacities.getCapacity(label); - float absMaxCapacity = queueCapacities.getMaximumCapacity(label); - if (absCapacity > absMaxCapacity) { - throw new IllegalArgumentException("Illegal queue capacity setting " - + "(abs-capacity=" + absCapacity + ") > (abs-maximum-capacity=" - + absMaxCapacity + ") for queue=[" - + queuePath + "],label=[" + label + "]"); - }*/ } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java index 62489a82147e7..b8091c7e05cf7 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java @@ -302,38 +302,13 @@ void initScheduler(Configuration configuration) throws IOException, YarnException { writeLock.lock(); try { - String confProviderStr = configuration.get( - YarnConfiguration.SCHEDULER_CONFIGURATION_STORE_CLASS, - YarnConfiguration.DEFAULT_CONFIGURATION_STORE); - switch (confProviderStr) { - case YarnConfiguration.FILE_CONFIGURATION_STORE: - this.csConfProvider = - new FileBasedCSConfigurationProvider(rmContext); - break; - case YarnConfiguration.MEMORY_CONFIGURATION_STORE: - case YarnConfiguration.LEVELDB_CONFIGURATION_STORE: - case YarnConfiguration.ZK_CONFIGURATION_STORE: - case YarnConfiguration.FS_CONFIGURATION_STORE: - this.csConfProvider = new MutableCSConfigurationProvider(rmContext); - break; - default: - throw new IOException("Invalid configuration store class: " + - confProviderStr); - } + this.csConfProvider = getCsConfProvider(configuration); this.csConfProvider.init(configuration); this.conf = this.csConfProvider.loadConfiguration(configuration); validateConf(this.conf); this.minimumAllocation = super.getMinimumAllocation(); initMaximumResourceCapability(super.getMaximumAllocation()); - this.calculator = this.conf.getResourceCalculator(); - if (this.calculator instanceof DefaultResourceCalculator - && ResourceUtils.getNumberOfKnownResourceTypes() > 2) { - throw new YarnRuntimeException("RM uses DefaultResourceCalculator which" - + " used only memory as resource-type but invalid resource-types" - + " specified " + ResourceUtils.getResourceTypes() + ". Use" - + " DominantResourceCalculator instead to make effective use of" - + " these resource-types"); - } + this.calculator = initResourceCalculator(); this.usePortForNodeName = this.conf.getUsePortForNodeName(); this.applications = new ConcurrentHashMap<>(); this.labelManager = rmContext.getNodeLabelManager(); @@ -341,71 +316,109 @@ void initScheduler(Configuration configuration) throws this.queueManager = new CapacitySchedulerQueueManager(yarnConf, this.labelManager, this.appPriorityACLManager); this.queueManager.setCapacitySchedulerContext(this); - this.workflowPriorityMappingsMgr = new WorkflowPriorityMappingsManager(); - this.activitiesManager = new ActivitiesManager(rmContext); activitiesManager.init(conf); initializeQueues(this.conf); this.isLazyPreemptionEnabled = conf.getLazyPreemptionEnabled(); - - scheduleAsynchronously = this.conf.getScheduleAynschronously(); - asyncScheduleInterval = this.conf.getLong(ASYNC_SCHEDULER_INTERVAL, - DEFAULT_ASYNC_SCHEDULER_INTERVAL); - this.assignMultipleEnabled = this.conf.getAssignMultipleEnabled(); this.maxAssignPerHeartbeat = this.conf.getMaxAssignPerHeartbeat(); - - this.appShouldFailFast = CapacitySchedulerConfiguration.shouldAppFailFast( - getConfig()); - - // number of threads for async scheduling - int maxAsyncSchedulingThreads = this.conf.getInt( - CapacitySchedulerConfiguration.SCHEDULE_ASYNCHRONOUSLY_MAXIMUM_THREAD, - 1); - maxAsyncSchedulingThreads = Math.max(maxAsyncSchedulingThreads, 1); - - if (scheduleAsynchronously) { - asyncSchedulerThreads = new ArrayList<>(); - for (int i = 0; i < maxAsyncSchedulingThreads; i++) { - asyncSchedulerThreads.add(new AsyncScheduleThread(this)); - } - resourceCommitterService = new ResourceCommitterService(this); - asyncMaxPendingBacklogs = this.conf.getInt( - CapacitySchedulerConfiguration. - SCHEDULE_ASYNCHRONOUSLY_MAXIMUM_PENDING_BACKLOGS, - CapacitySchedulerConfiguration. - DEFAULT_SCHEDULE_ASYNCHRONOUSLY_MAXIMUM_PENDING_BACKLOGS); - } + this.appShouldFailFast = CapacitySchedulerConfiguration.shouldAppFailFast(getConfig()); + initAsyncSchedulingProperties(); // Setup how many containers we can allocate for each round offswitchPerHeartbeatLimit = this.conf.getOffSwitchPerHeartbeatLimit(); - // Register CS specific multi-node policies to common MultiNodeManager - // which will add to a MultiNodeSorter which gives a pre-sorted list of - // nodes to scheduler's allocation. - multiNodePlacementEnabled = this.conf.getMultiNodePlacementEnabled(); - if(rmContext.getMultiNodeSortingManager() != null) { - rmContext.getMultiNodeSortingManager().registerMultiNodePolicyNames( - multiNodePlacementEnabled, - this.conf.getMultiNodePlacementPolicies()); - } - - LOG.info("Initialized CapacityScheduler with " + "calculator=" - + getResourceCalculator().getClass() + ", " + "minimumAllocation=" - + getMinimumResourceCapability() + ", " + "maximumAllocation=" - + getMaximumResourceCapability() + ", " + "asynchronousScheduling=" - + scheduleAsynchronously + ", " + "asyncScheduleInterval=" - + asyncScheduleInterval + "ms" + ",multiNodePlacementEnabled=" - + multiNodePlacementEnabled + ", " + "assignMultipleEnabled=" - + assignMultipleEnabled + ", " + "maxAssignPerHeartbeat=" - + maxAssignPerHeartbeat + ", " + "offswitchPerHeartbeatLimit=" - + offswitchPerHeartbeatLimit); + initMultiNodePlacement(); + printSchedulerInitialized(); } finally { writeLock.unlock(); } } + private CSConfigurationProvider getCsConfProvider(Configuration configuration) + throws IOException { + String confProviderStr = configuration.get( + YarnConfiguration.SCHEDULER_CONFIGURATION_STORE_CLASS, + YarnConfiguration.DEFAULT_CONFIGURATION_STORE); + switch (confProviderStr) { + case YarnConfiguration.FILE_CONFIGURATION_STORE: + return new FileBasedCSConfigurationProvider(rmContext); + case YarnConfiguration.MEMORY_CONFIGURATION_STORE: + case YarnConfiguration.LEVELDB_CONFIGURATION_STORE: + case YarnConfiguration.ZK_CONFIGURATION_STORE: + case YarnConfiguration.FS_CONFIGURATION_STORE: + return new MutableCSConfigurationProvider(rmContext); + default: + throw new IOException("Invalid configuration store class: " + confProviderStr); + } + } + + private ResourceCalculator initResourceCalculator() { + ResourceCalculator resourceCalculator = this.conf.getResourceCalculator(); + if (resourceCalculator instanceof DefaultResourceCalculator + && ResourceUtils.getNumberOfKnownResourceTypes() > 2) { + throw new YarnRuntimeException("RM uses DefaultResourceCalculator which" + + " used only memory as resource-type but invalid resource-types" + + " specified " + ResourceUtils.getResourceTypes() + ". Use" + + " DominantResourceCalculator instead to make effective use of" + + " these resource-types"); + } + return resourceCalculator; + } + + private void initAsyncSchedulingProperties() { + scheduleAsynchronously = this.conf.getScheduleAynschronously(); + asyncScheduleInterval = this.conf.getLong(ASYNC_SCHEDULER_INTERVAL, + DEFAULT_ASYNC_SCHEDULER_INTERVAL); + + // number of threads for async scheduling + int maxAsyncSchedulingThreads = this.conf.getInt( + CapacitySchedulerConfiguration.SCHEDULE_ASYNCHRONOUSLY_MAXIMUM_THREAD, 1); + maxAsyncSchedulingThreads = Math.max(maxAsyncSchedulingThreads, 1); + + if (scheduleAsynchronously) { + asyncSchedulerThreads = new ArrayList<>(); + for (int i = 0; i < maxAsyncSchedulingThreads; i++) { + asyncSchedulerThreads.add(new AsyncScheduleThread(this)); + } + resourceCommitterService = new ResourceCommitterService(this); + asyncMaxPendingBacklogs = this.conf.getInt( + CapacitySchedulerConfiguration. + SCHEDULE_ASYNCHRONOUSLY_MAXIMUM_PENDING_BACKLOGS, + CapacitySchedulerConfiguration. + DEFAULT_SCHEDULE_ASYNCHRONOUSLY_MAXIMUM_PENDING_BACKLOGS); + } + } + + private void initMultiNodePlacement() { + // Register CS specific multi-node policies to common MultiNodeManager + // which will add to a MultiNodeSorter which gives a pre-sorted list of + // nodes to scheduler's allocation. + multiNodePlacementEnabled = this.conf.getMultiNodePlacementEnabled(); + if (rmContext.getMultiNodeSortingManager() != null) { + rmContext.getMultiNodeSortingManager().registerMultiNodePolicyNames( + multiNodePlacementEnabled, + this.conf.getMultiNodePlacementPolicies()); + } + } + + private void printSchedulerInitialized() { + LOG.info("Initialized CapacityScheduler with calculator={}, minimumAllocation={}, " + + "maximumAllocation={}, asynchronousScheduling={}, asyncScheduleInterval={} ms, " + + "multiNodePlacementEnabled={}, assignMultipleEnabled={}, maxAssignPerHeartbeat={}, " + + "offswitchPerHeartbeatLimit={}", + getResourceCalculator().getClass(), + getMinimumResourceCapability(), + getMaximumResourceCapability(), + scheduleAsynchronously, + asyncScheduleInterval, + multiNodePlacementEnabled, + assignMultipleEnabled, + maxAssignPerHeartbeat, + offswitchPerHeartbeatLimit); + } + private void startSchedulerThreads() { writeLock.lock(); try { @@ -937,6 +950,10 @@ private void addApplicationOnRecovery(ApplicationId applicationId, } catch (AccessControlException ace) { // Ignore the exception for recovered app as the app was previously // accepted. + LOG.warn("AccessControlException received when trying to recover " + + applicationId + " in queue " + queueName + " for user " + user + + ". Since the app was in the queue prior to recovery, the Capacity" + + " Scheduler will recover the app anyway.", ace); } queue.getMetrics().submitApp(user, unmanagedAM); @@ -2948,7 +2965,7 @@ public Priority updateApplicationPriority(Priority newPriority, ApplicationStateData appState = ApplicationStateData.newInstance( rmApp.getSubmitTime(), rmApp.getStartTime(), rmApp.getApplicationSubmissionContext(), rmApp.getUser(), - rmApp.getCallerContext()); + rmApp.getRealUser(), rmApp.getCallerContext()); appState.setApplicationTimeouts(rmApp.getApplicationTimeouts()); appState.setLaunchTime(rmApp.getLaunchTime()); rmContext.getStateStore().updateApplicationStateSynchronously(appState, diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java index 5e614f149eda3..615a4d08087d4 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java @@ -22,6 +22,7 @@ import org.apache.hadoop.thirdparty.com.google.common.base.Strings; import org.apache.hadoop.thirdparty.com.google.common.collect.ImmutableSet; import org.apache.hadoop.yarn.server.resourcemanager.placement.csmappingrule.MappingRule; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.conf.QueueCapacityConfigParser; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.placement.MappingRuleCreator; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -73,9 +74,9 @@ import java.util.List; import java.util.Map; import java.util.Map.Entry; +import java.util.Set; import java.util.regex.Matcher; import java.util.regex.Pattern; -import java.util.Set; public class CapacitySchedulerConfiguration extends ReservationSchedulerConfiguration { @@ -413,6 +414,10 @@ public class CapacitySchedulerConfiguration extends ReservationSchedulerConfigur public static final String MAPPING_RULE_FORMAT_DEFAULT = MAPPING_RULE_FORMAT_LEGACY; + + private static final QueueCapacityConfigParser queueCapacityConfigParser + = new QueueCapacityConfigParser(); + private ConfigurationProperties configurationProperties; /** @@ -454,13 +459,17 @@ static String getUserPrefix(String user) { return PREFIX + "user." + user + DOT; } - private String getNodeLabelPrefix(String queue, String label) { + public static String getNodeLabelPrefix(String queue, String label) { if (label.equals(CommonNodeLabelsManager.NO_LABEL)) { return getQueuePrefix(queue); } return getQueuePrefix(queue) + ACCESSIBLE_NODE_LABELS + DOT + label + DOT; } + public void setMaximumSystemApplications(int numMaxApps) { + setInt(MAXIMUM_SYSTEM_APPLICATIONS, numMaxApps); + } + public int getMaximumSystemApplications() { int maxApplications = getInt(MAXIMUM_SYSTEM_APPLICATIONS, DEFAULT_MAXIMUM_SYSTEM_APPLICATIIONS); @@ -2567,6 +2576,16 @@ public void setMaximumResourceRequirement(String label, String queue, updateMinMaxResourceToConf(label, queue, resource, MAXIMUM_CAPACITY); } + public Map parseConfiguredResourceVector( + String queuePath, Set labels) { + Map queueResourceVectors = new HashMap<>(); + for (String label : labels) { + queueResourceVectors.put(label, queueCapacityConfigParser.parse(this, queuePath, label)); + } + + return queueResourceVectors; + } + private void updateMinMaxResourceToConf(String label, String queue, Resource resource, String type) { if (queue.equals("root")) { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java index cc282ef59a23e..a4e2a822ead50 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java @@ -20,7 +20,6 @@ import java.io.IOException; import java.util.*; -import java.util.Map.Entry; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; @@ -161,7 +160,7 @@ public LeafQueue(CapacitySchedulerContext cs, super(cs, configuration, queueName, parent, old); setDynamicQueue(isDynamic); - this.usersManager = new UsersManager(metrics, this, labelManager, csContext, + this.usersManager = new UsersManager(usageTracker.getMetrics(), this, labelManager, csContext, resourceCalculator); // One time initialization is enough since it is static ordering policy @@ -216,15 +215,14 @@ protected void setupQueueConfigs(Resource clusterResource, priorityAcls = conf.getPriorityAcls(getQueuePath(), csContext.getMaxClusterLevelAppPriority()); - if (!SchedulerUtils.checkQueueLabelExpression(this.accessibleLabels, - this.defaultLabelExpression, null)) { + Set accessibleNodeLabels = this.queueNodeLabelsSettings.getAccessibleNodeLabels(); + if (!SchedulerUtils.checkQueueLabelExpression(accessibleNodeLabels, + this.queueNodeLabelsSettings.getDefaultLabelExpression(), null)) { throw new IOException( "Invalid default label expression of " + " queue=" + getQueuePath() + " doesn't have permission to access all labels " + "in default label expression. labelExpression of resource request=" - + (this.defaultLabelExpression == null ? - "" : - this.defaultLabelExpression) + ". Queue labels=" + ( + + getDefaultNodeLabelExpressionStr() + ". Queue labels=" + ( getAccessibleNodeLabels() == null ? "" : StringUtils @@ -239,8 +237,10 @@ protected void setupQueueConfigs(Resource clusterResource, // re-init this since max allocation could have changed this.minimumAllocationFactor = Resources.ratio(resourceCalculator, - Resources.subtract(maximumAllocation, minimumAllocation), - maximumAllocation); + Resources.subtract( + queueAllocationSettings.getMaximumAllocation(), + queueAllocationSettings.getMinimumAllocation()), + queueAllocationSettings.getMaximumAllocation()); StringBuilder aclsString = new StringBuilder(); for (Map.Entry e : acls.entrySet()) { @@ -248,10 +248,9 @@ protected void setupQueueConfigs(Resource clusterResource, } StringBuilder labelStrBuilder = new StringBuilder(); - if (accessibleLabels != null) { - for (String s : accessibleLabels) { - labelStrBuilder.append(s) - .append(","); + if (accessibleNodeLabels != null) { + for (String nodeLabel : accessibleNodeLabels) { + labelStrBuilder.append(nodeLabel).append(","); } } @@ -298,8 +297,9 @@ protected void setupQueueConfigs(Resource clusterResource, + "minimumAllocationFactor = " + minimumAllocationFactor + " [= (float)(maximumAllocationMemory - minimumAllocationMemory) / " + "maximumAllocationMemory ]" + "\n" + "maximumAllocation = " - + maximumAllocation + " [= configuredMaxAllocation ]" + "\n" - + "numContainers = " + numContainers + + queueAllocationSettings.getMaximumAllocation() + + " [= configuredMaxAllocation ]" + "\n" + + "numContainers = " + usageTracker.getNumContainers() + " [= currentNumContainers ]" + "\n" + "state = " + getState() + " [= configuredState ]" + "\n" + "acls = " + aclsString + " [= configuredAcls ]" + "\n" @@ -319,6 +319,11 @@ protected void setupQueueConfigs(Resource clusterResource, } } + private String getDefaultNodeLabelExpressionStr() { + String defaultLabelExpression = queueNodeLabelsSettings.getDefaultLabelExpression(); + return defaultLabelExpression == null ? "" : defaultLabelExpression; + } + /** * Used only by tests. */ @@ -483,7 +488,7 @@ public String toString() { try { return getQueuePath() + ": " + getCapacityOrWeightString() + ", " + "absoluteCapacity=" + queueCapacities.getAbsoluteCapacity() - + ", " + "usedResources=" + queueUsage.getUsed() + ", " + + ", " + "usedResources=" + usageTracker.getQueueUsage().getUsed() + ", " + "usedCapacity=" + getUsedCapacity() + ", " + "absoluteUsedCapacity=" + getAbsoluteUsedCapacity() + ", " + "numApps=" + getNumApplications() + ", " + "numContainers=" + getNumContainers() + ", " @@ -600,10 +605,10 @@ public void submitApplicationAttempt(FiCaSchedulerApp application, if (!isMoveApp) { boolean unmanagedAM = application.getAppSchedulingInfo() != null && application.getAppSchedulingInfo().isUnmanagedAM(); - metrics.submitAppAttempt(userName, unmanagedAM); + usageTracker.getMetrics().submitAppAttempt(userName, unmanagedAM); } - getParent().submitApplicationAttempt(application, userName); + parent.submitApplicationAttempt(application, userName); } @Override @@ -617,10 +622,10 @@ public void submitApplication(ApplicationId applicationId, String userName, // Inform the parent queue try { - getParent().submitApplication(applicationId, userName, queue); + parent.submitApplication(applicationId, userName, queue); } catch (AccessControlException ace) { LOG.info("Failed to submit application to parent-queue: " + - getParent().getQueuePath(), ace); + parent.getQueuePath(), ace); throw ace; } @@ -665,20 +670,20 @@ public void validateSubmitApplication(ApplicationId applicationId, } try { - getParent().validateSubmitApplication(applicationId, userName, queue); + parent.validateSubmitApplication(applicationId, userName, queue); } catch (AccessControlException ace) { LOG.info("Failed to submit application to parent-queue: " + - getParent().getQueuePath(), ace); + parent.getQueuePath(), ace); throw ace; } } public Resource getAMResourceLimit() { - return queueUsage.getAMLimit(); + return usageTracker.getQueueUsage().getAMLimit(); } public Resource getAMResourceLimitPerPartition(String nodePartition) { - return queueUsage.getAMLimit(nodePartition); + return usageTracker.getQueueUsage().getAMLimit(nodePartition); } @VisibleForTesting @@ -715,6 +720,8 @@ public Resource getUserAMResourceLimitPerPartition( Resource queuePartitionResource = getEffectiveCapacity(nodePartition); + Resource minimumAllocation = queueAllocationSettings.getMinimumAllocation(); + Resource userAMLimit = Resources.multiplyAndNormalizeUp( resourceCalculator, queuePartitionResource, queueCapacities.getMaxAMResourcePercentage(nodePartition) @@ -751,7 +758,7 @@ public Resource getUserAMResourceLimitPerPartition( Resources.min(resourceCalculator, lastClusterResource, preWeighteduserAMLimit, Resources.clone(getAMResourceLimitPerPartition(nodePartition))); - queueUsage.setUserAMLimit(nodePartition, preWeighteduserAMLimit); + usageTracker.getQueueUsage().setUserAMLimit(nodePartition, preWeighteduserAMLimit); LOG.debug("Effective user AM limit for \"{}\":{}. Effective weighted" + " user AM limit: {}. User weight: {}", userName, @@ -801,10 +808,10 @@ public Resource calculateAndGetAMResourceLimitPerPartition( Resource amResouceLimit = Resources.multiplyAndNormalizeUp( resourceCalculator, queuePartitionUsableResource, amResourcePercent, - minimumAllocation); + queueAllocationSettings.getMinimumAllocation()); - metrics.setAMResouceLimit(nodePartition, amResouceLimit); - queueUsage.setAMLimit(nodePartition, amResouceLimit); + usageTracker.getMetrics().setAMResouceLimit(nodePartition, amResouceLimit); + usageTracker.getQueueUsage().setAMLimit(nodePartition, amResouceLimit); LOG.debug("Queue: {}, node label : {}, queue partition resource : {}," + " queue current limit : {}, queue partition usable resource : {}," + " amResourceLimit : {}", getQueuePath(), nodePartition, @@ -849,7 +856,7 @@ protected void activateApplications() { // Check am resource limit. Resource amIfStarted = Resources.add( application.getAMResource(partitionName), - queueUsage.getAMUsed(partitionName)); + usageTracker.getQueueUsage().getAMUsed(partitionName)); if (LOG.isDebugEnabled()) { LOG.debug("application " + application.getId() + " AMResource " @@ -863,7 +870,7 @@ protected void activateApplications() { if (!resourceCalculator.fitsIn(amIfStarted, amLimit)) { if (getNumActiveApplications() < 1 || (Resources.lessThanOrEqual( resourceCalculator, lastClusterResource, - queueUsage.getAMUsed(partitionName), Resources.none()))) { + usageTracker.getQueueUsage().getAMUsed(partitionName), Resources.none()))) { LOG.warn("maximum-am-resource-percent is insufficient to start a" + " single application in queue, it is likely set too low." + " skipping enforcement to allow at least one application" @@ -895,7 +902,7 @@ protected void activateApplications() { if (!resourceCalculator.fitsIn(userAmIfStarted, userAMLimit)) { if (getNumActiveApplications() < 1 || (Resources.lessThanOrEqual( resourceCalculator, lastClusterResource, - queueUsage.getAMUsed(partitionName), Resources.none()))) { + usageTracker.getQueueUsage().getAMUsed(partitionName), Resources.none()))) { LOG.warn("maximum-am-resource-percent is insufficient to start a" + " single application in queue for user, it is likely set too" + " low. skipping enforcement to allow at least one application" @@ -913,14 +920,14 @@ protected void activateApplications() { orderingPolicy.addSchedulableEntity(application); application.updateAMContainerDiagnostics(AMState.ACTIVATED, null); - queueUsage.incAMUsed(partitionName, + usageTracker.getQueueUsage().incAMUsed(partitionName, application.getAMResource(partitionName)); user.getResourceUsage().incAMUsed(partitionName, application.getAMResource(partitionName)); user.getResourceUsage().setAMLimit(partitionName, userAMLimit); - metrics.incAMUsed(partitionName, application.getUser(), + usageTracker.getMetrics().incAMUsed(partitionName, application.getUser(), application.getAMResource(partitionName)); - metrics.setAMResouceLimitForUser(partitionName, + usageTracker.getMetrics().setAMResouceLimitForUser(partitionName, application.getUser(), userAMLimit); fsApp.remove(); LOG.info("Application " + applicationId + " from user: " + application @@ -988,14 +995,14 @@ public void finishApplication(ApplicationId application, String user) { appFinished(); // Inform the parent queue - getParent().finishApplication(application, user); + parent.finishApplication(application, user); } @Override public void finishApplicationAttempt(FiCaSchedulerApp application, String queue) { // Careful! Locking order is important! removeApplicationAttempt(application, application.getUser()); - getParent().finishApplicationAttempt(application, queue); + parent.finishApplicationAttempt(application, queue); } private void removeApplicationAttempt( @@ -1021,11 +1028,11 @@ private void removeApplicationAttempt( if (!wasActive) { pendingOrderingPolicy.removeSchedulableEntity(application); } else{ - queueUsage.decAMUsed(partitionName, + usageTracker.getQueueUsage().decAMUsed(partitionName, application.getAMResource(partitionName)); user.getResourceUsage().decAMUsed(partitionName, application.getAMResource(partitionName)); - metrics.decAMUsed(partitionName, application.getUser(), + usageTracker.getMetrics().decAMUsed(partitionName, application.getUser(), application.getAMResource(partitionName)); } applicationAttemptMap.remove(application.getApplicationAttemptId()); @@ -1058,11 +1065,11 @@ private FiCaSchedulerApp getApplication( private void setPreemptionAllowed(ResourceLimits limits, String nodePartition) { // Set preemption-allowed: // For leaf queue, only under-utilized queue is allowed to preempt resources from other queues - if (!queueResourceQuotas.getEffectiveMinResource(nodePartition) + if (!usageTracker.getQueueResourceQuotas().getEffectiveMinResource(nodePartition) .equals(Resources.none())) { limits.setIsAllowPreemption(Resources.lessThan(resourceCalculator, - csContext.getClusterResource(), queueUsage.getUsed(nodePartition), - queueResourceQuotas.getEffectiveMinResource(nodePartition))); + csContext.getClusterResource(), usageTracker.getQueueUsage().getUsed(nodePartition), + usageTracker.getQueueResourceQuotas().getEffectiveMinResource(nodePartition))); return; } @@ -1166,9 +1173,9 @@ public CSAssignment assignContainers(Resource clusterResource, // if our queue cannot access this node, just return if (schedulingMode == SchedulingMode.RESPECT_PARTITION_EXCLUSIVITY - && !accessibleToPartition(candidates.getPartition())) { + && !queueNodeLabelsSettings.isAccessibleToPartition(candidates.getPartition())) { ActivitiesLogger.QUEUE.recordQueueActivity(activitiesManager, node, - getParent().getQueuePath(), getQueuePath(), ActivityState.REJECTED, + parent.getQueuePath(), getQueuePath(), ActivityState.REJECTED, ActivityDiagnosticConstant.QUEUE_NOT_ABLE_TO_ACCESS_PARTITION); return CSAssignment.NULL_ASSIGNMENT; } @@ -1184,7 +1191,7 @@ public CSAssignment assignContainers(Resource clusterResource, .getPartition()); } ActivitiesLogger.QUEUE.recordQueueActivity(activitiesManager, node, - getParent().getQueuePath(), getQueuePath(), ActivityState.SKIPPED, + parent.getQueuePath(), getQueuePath(), ActivityState.SKIPPED, ActivityDiagnosticConstant.QUEUE_DO_NOT_NEED_MORE_RESOURCE); return CSAssignment.NULL_ASSIGNMENT; } @@ -1212,7 +1219,7 @@ public CSAssignment assignContainers(Resource clusterResource, activitiesManager, node, application, application.getPriority(), ActivityDiagnosticConstant.QUEUE_HIT_MAX_CAPACITY_LIMIT); ActivitiesLogger.QUEUE.recordQueueActivity(activitiesManager, node, - getParent().getQueuePath(), getQueuePath(), + parent.getQueuePath(), getQueuePath(), ActivityState.REJECTED, ActivityDiagnosticConstant.QUEUE_HIT_MAX_CAPACITY_LIMIT); return CSAssignment.NULL_ASSIGNMENT; @@ -1281,7 +1288,7 @@ public CSAssignment assignContainers(Resource clusterResource, if (Resources.greaterThan(resourceCalculator, clusterResource, assigned, Resources.none())) { ActivitiesLogger.QUEUE.recordQueueActivity(activitiesManager, node, - getParent().getQueuePath(), getQueuePath(), + parent.getQueuePath(), getQueuePath(), ActivityState.ACCEPTED, ActivityDiagnosticConstant.EMPTY); return assignment; } else if (assignment.getSkippedType() @@ -1293,7 +1300,7 @@ public CSAssignment assignContainers(Resource clusterResource, } else if (assignment.getSkippedType() == CSAssignment.SkippedType.QUEUE_LIMIT) { ActivitiesLogger.QUEUE.recordQueueActivity(activitiesManager, node, - getParent().getQueuePath(), getQueuePath(), ActivityState.REJECTED, + parent.getQueuePath(), getQueuePath(), ActivityState.REJECTED, () -> ActivityDiagnosticConstant.QUEUE_DO_NOT_HAVE_ENOUGH_HEADROOM + " from " + application.getApplicationId()); return assignment; @@ -1301,7 +1308,7 @@ public CSAssignment assignContainers(Resource clusterResource, // If we don't allocate anything, and it is not skipped by application, // we will return to respect FIFO of applications ActivitiesLogger.QUEUE.recordQueueActivity(activitiesManager, node, - getParent().getQueuePath(), getQueuePath(), ActivityState.SKIPPED, + parent.getQueuePath(), getQueuePath(), ActivityState.SKIPPED, ActivityDiagnosticConstant.QUEUE_SKIPPED_TO_RESPECT_FIFO); ActivitiesLogger.APP.finishSkippedAppAllocationRecording( activitiesManager, application.getApplicationId(), @@ -1310,7 +1317,7 @@ public CSAssignment assignContainers(Resource clusterResource, } } ActivitiesLogger.QUEUE.recordQueueActivity(activitiesManager, node, - getParent().getQueuePath(), getQueuePath(), ActivityState.SKIPPED, + parent.getQueuePath(), getQueuePath(), ActivityState.SKIPPED, ActivityDiagnosticConstant.EMPTY); return CSAssignment.NULL_ASSIGNMENT; @@ -1514,10 +1521,11 @@ private Resource getHeadroom(User user, Resources.subtractNonNegative(userLimitResource, user.getUsed(partition)), Resources.subtractNonNegative(currentPartitionResourceLimit, - queueUsage.getUsed(partition))); + usageTracker.getQueueUsage().getUsed(partition))); // Normalize it before return headroom = - Resources.roundDown(resourceCalculator, headroom, minimumAllocation); + Resources.roundDown(resourceCalculator, headroom, + queueAllocationSettings.getMinimumAllocation()); //headroom = min (unused resourcelimit of a label, calculated headroom ) Resource clusterPartitionResource = @@ -1560,7 +1568,7 @@ Resource computeUserLimitAndSetHeadroom(FiCaSchedulerApp application, setQueueResourceLimitsInfo(clusterResource); Resource headroom = - metrics.getUserMetrics(user) == null ? Resources.none() : + usageTracker.getMetrics().getUserMetrics(user) == null ? Resources.none() : getHeadroom(queueUser, cachedResourceLimitsForHeadroom.getLimit(), clusterResource, userLimit, nodePartition); @@ -1577,7 +1585,7 @@ Resource computeUserLimitAndSetHeadroom(FiCaSchedulerApp application, application.setHeadroomProvider(headroomProvider); - metrics.setAvailableResourcesToUser(nodePartition, user, headroom); + usageTracker.getMetrics().setAvailableResourcesToUser(nodePartition, user, headroom); return userLimit; } @@ -1796,7 +1804,7 @@ public void completedContainer(Resource clusterResource, if (removed) { // Inform the parent queue _outside_ of the leaf-queue lock - getParent().completedContainer(clusterResource, application, node, + parent.completedContainer(clusterResource, application, node, rmContainer, null, event, this, sortQueues); } } @@ -1844,20 +1852,20 @@ void allocateResource(Resource clusterResource, nodePartition, true); Resource partitionHeadroom = Resources.createResource(0, 0); - if (metrics.getUserMetrics(userName) != null) { + if (usageTracker.getMetrics().getUserMetrics(userName) != null) { partitionHeadroom = getHeadroom(user, cachedResourceLimitsForHeadroom.getLimit(), clusterResource, getResourceLimitForActiveUsers(userName, clusterResource, nodePartition, SchedulingMode.RESPECT_PARTITION_EXCLUSIVITY), nodePartition); } - metrics.setAvailableResourcesToUser(nodePartition, userName, + usageTracker.getMetrics().setAvailableResourcesToUser(nodePartition, userName, partitionHeadroom); if (LOG.isDebugEnabled()) { LOG.debug(getQueuePath() + " user=" + userName + " used=" - + queueUsage.getUsed(nodePartition) + " numContainers=" - + numContainers + " headroom = " + application.getHeadroom() + + usageTracker.getQueueUsage().getUsed(nodePartition) + " numContainers=" + + usageTracker.getNumContainers() + " headroom = " + application.getHeadroom() + " user-resources=" + user.getUsed()); } } finally { @@ -1892,20 +1900,20 @@ void releaseResource(Resource clusterResource, nodePartition, false); Resource partitionHeadroom = Resources.createResource(0, 0); - if (metrics.getUserMetrics(userName) != null) { + if (usageTracker.getMetrics().getUserMetrics(userName) != null) { partitionHeadroom = getHeadroom(user, cachedResourceLimitsForHeadroom.getLimit(), clusterResource, getResourceLimitForActiveUsers(userName, clusterResource, nodePartition, SchedulingMode.RESPECT_PARTITION_EXCLUSIVITY), nodePartition); } - metrics.setAvailableResourcesToUser(nodePartition, userName, + usageTracker.getMetrics().setAvailableResourcesToUser(nodePartition, userName, partitionHeadroom); if (LOG.isDebugEnabled()) { LOG.debug( - getQueuePath() + " used=" + queueUsage.getUsed() + " numContainers=" - + numContainers + " user=" + userName + " user-resources=" + getQueuePath() + " used=" + usageTracker.getQueueUsage().getUsed() + " numContainers=" + + usageTracker.getNumContainers() + " user=" + userName + " user-resources=" + user.getUsed()); } } finally { @@ -1923,7 +1931,7 @@ private void updateCurrentResourceLimits( this.cachedResourceLimitsForHeadroom = new ResourceLimits(currentResourceLimits.getLimit()); Resource queueMaxResource = getEffectiveMaxCapacityDown( - RMNodeLabelsManager.NO_LABEL, minimumAllocation); + RMNodeLabelsManager.NO_LABEL, queueAllocationSettings.getMinimumAllocation()); this.cachedResourceLimitsForHeadroom.setLimit(Resources.min( resourceCalculator, clusterResource, queueMaxResource, currentResourceLimits.getLimit())); @@ -1939,8 +1947,9 @@ public void updateClusterResource(Resource clusterResource, updateAbsoluteCapacities(); super.updateEffectiveResources(clusterResource); - super.updateMaxAppRelatedField(csContext.getConfiguration(), - this); + + // Update maximum applications for the queue and for users + updateMaximumApplications(csContext.getConfiguration()); updateCurrentResourceLimits(currentResourceLimits, clusterResource); @@ -2001,7 +2010,7 @@ public void incAMUsedResource(String nodeLabel, Resource resourceToInc, getUser(application.getUser()).getResourceUsage().incAMUsed(nodeLabel, resourceToInc); // ResourceUsage has its own lock, no addition lock needs here. - queueUsage.incAMUsed(nodeLabel, resourceToInc); + usageTracker.getQueueUsage().incAMUsed(nodeLabel, resourceToInc); } public void decAMUsedResource(String nodeLabel, Resource resourceToDec, @@ -2009,7 +2018,7 @@ public void decAMUsedResource(String nodeLabel, Resource resourceToDec, getUser(application.getUser()).getResourceUsage().decAMUsed(nodeLabel, resourceToDec); // ResourceUsage has its own lock, no addition lock needs here. - queueUsage.decAMUsed(nodeLabel, resourceToDec); + usageTracker.getQueueUsage().decAMUsed(nodeLabel, resourceToDec); } @Override @@ -2033,7 +2042,7 @@ public void recoverContainer(Resource clusterResource, writeLock.unlock(); } - getParent().recoverContainer(clusterResource, attempt, rmContainer); + parent.recoverContainer(clusterResource, attempt, rmContainer); } /** @@ -2159,9 +2168,9 @@ public void attachContainer(Resource clusterResource, + " resource=" + rmContainer.getContainer().getResource() + " queueMoveIn=" + this + " usedCapacity=" + getUsedCapacity() + " absoluteUsedCapacity=" + getAbsoluteUsedCapacity() + " used=" - + queueUsage.getUsed() + " cluster=" + clusterResource); + + usageTracker.getQueueUsage().getUsed() + " cluster=" + clusterResource); // Inform the parent queue - getParent().attachContainer(clusterResource, application, rmContainer); + parent.attachContainer(clusterResource, application, rmContainer); } } @@ -2179,9 +2188,9 @@ public void detachContainer(Resource clusterResource, + " resource=" + rmContainer.getContainer().getResource() + " queueMoveOut=" + this + " usedCapacity=" + getUsedCapacity() + " absoluteUsedCapacity=" + getAbsoluteUsedCapacity() + " used=" - + queueUsage.getUsed() + " cluster=" + clusterResource); + + usageTracker.getQueueUsage().getUsed() + " cluster=" + clusterResource); // Inform the parent queue - getParent().detachContainer(clusterResource, application, rmContainer); + parent.detachContainer(clusterResource, application, rmContainer); } } @@ -2326,6 +2335,58 @@ public void stopQueue() { } } + void updateMaximumApplications(CapacitySchedulerConfiguration conf) { + int maxAppsForQueue = conf.getMaximumApplicationsPerQueue(getQueuePath()); + + int maxDefaultPerQueueApps = conf.getGlobalMaximumApplicationsPerQueue(); + int maxSystemApps = conf.getMaximumSystemApplications(); + int baseMaxApplications = maxDefaultPerQueueApps > 0 ? + Math.min(maxDefaultPerQueueApps, maxSystemApps) + : maxSystemApps; + + String maxLabel = RMNodeLabelsManager.NO_LABEL; + if (maxAppsForQueue < 0) { + if (maxDefaultPerQueueApps > 0 && this.capacityConfigType + != CapacityConfigType.ABSOLUTE_RESOURCE) { + maxAppsForQueue = baseMaxApplications; + } else { + for (String label : queueNodeLabelsSettings.getConfiguredNodeLabels()) { + int maxApplicationsByLabel = (int) (baseMaxApplications + * queueCapacities.getAbsoluteCapacity(label)); + if (maxApplicationsByLabel > maxAppsForQueue) { + maxAppsForQueue = maxApplicationsByLabel; + maxLabel = label; + } + } + } + } + + setMaxApplications(maxAppsForQueue); + + updateMaxAppsPerUser(); + + LOG.info("LeafQueue:" + getQueuePath() + + "update max app related, maxApplications=" + + maxAppsForQueue + ", maxApplicationsPerUser=" + + maxApplicationsPerUser + ", Abs Cap:" + queueCapacities + .getAbsoluteCapacity(maxLabel) + ", Cap: " + queueCapacities + .getCapacity(maxLabel) + ", MaxCap : " + queueCapacities + .getMaximumCapacity(maxLabel)); + } + + private void updateMaxAppsPerUser() { + int maxAppsPerUser = maxApplications; + if (getUsersManager().getUserLimitFactor() != -1) { + int maxApplicationsWithUserLimits = (int) (maxApplications + * (getUsersManager().getUserLimit() / 100.0f) + * getUsersManager().getUserLimitFactor()); + maxAppsPerUser = Math.min(maxApplications, + maxApplicationsWithUserLimits); + } + + setMaxApplicationsPerUser(maxAppsPerUser); + } + /** * Get all valid users in this queue. * @return user list @@ -2347,6 +2408,7 @@ static class CachedUserLimit { private void updateQueuePreemptionMetrics(RMContainer rmc) { final long usedMillis = rmc.getFinishTime() - rmc.getCreationTime(); final long usedSeconds = usedMillis / DateUtils.MILLIS_PER_SECOND; + CSQueueMetrics metrics = usageTracker.getMetrics(); Resource containerResource = rmc.getAllocatedResource(); metrics.preemptContainer(); long mbSeconds = (containerResource.getMemorySize() * usedMillis) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java index 3a25a17d97500..0f302b8e73c99 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java @@ -169,10 +169,9 @@ protected void setupQueueConfigs(Resource clusterResource, } StringBuilder labelStrBuilder = new StringBuilder(); - if (accessibleLabels != null) { - for (String s : accessibleLabels) { - labelStrBuilder.append(s) - .append(","); + if (queueNodeLabelsSettings.getAccessibleNodeLabels() != null) { + for (String nodeLabel : queueNodeLabelsSettings.getAccessibleNodeLabels()) { + labelStrBuilder.append(nodeLabel).append(","); } } @@ -328,7 +327,7 @@ void setChildQueues(Collection childQueues) throws IOException { Resource resourceByLabel = labelManager.getResourceByLabel(nodeLabel, scheduler.getClusterResource()); Resource parentMinResource = - queueResourceQuotas.getConfiguredMinResource(nodeLabel); + usageTracker.getQueueResourceQuotas().getConfiguredMinResource(nodeLabel); if (!parentMinResource.equals(Resources.none()) && Resources.lessThan( resourceCalculator, resourceByLabel, parentMinResource, minRes)) { throw new IOException( @@ -475,9 +474,9 @@ public String toString() { "numChildQueue= " + childQueues.size() + ", " + getCapacityOrWeightString() + ", " + "absoluteCapacity=" + queueCapacities.getAbsoluteCapacity() + ", " + - "usedResources=" + queueUsage.getUsed() + - "usedCapacity=" + getUsedCapacity() + ", " + - "numApps=" + getNumApplications() + ", " + + "usedResources=" + usageTracker.getQueueUsage().getUsed() + + "usedCapacity=" + getUsedCapacity() + ", " + + "numApps=" + getNumApplications() + ", " + "numContainers=" + getNumContainers(); } @@ -757,7 +756,7 @@ public void submitApplication(ApplicationId applicationId, String user, try { parent.submitApplication(applicationId, user, queue); } catch (AccessControlException ace) { - LOG.info("Failed to submit application to parent-queue: " + + LOG.info("Failed to submit application to parent-queue: " + parent.getQueuePath(), ace); removeApplication(applicationId, user); throw ace; @@ -846,7 +845,7 @@ private void removeApplication(ApplicationId applicationId, } private String getParentName() { - return getParent() != null ? getParent().getQueuePath() : ""; + return parent != null ? parent.getQueuePath() : ""; } @Override @@ -857,7 +856,7 @@ public CSAssignment assignContainers(Resource clusterResource, // if our queue cannot access this node, just return if (schedulingMode == SchedulingMode.RESPECT_PARTITION_EXCLUSIVITY - && !accessibleToPartition(candidates.getPartition())) { + && !queueNodeLabelsSettings.isAccessibleToPartition(candidates.getPartition())) { if (LOG.isDebugEnabled()) { long now = System.currentTimeMillis(); // Do logging every 1 sec to avoid excessive logging. @@ -992,7 +991,7 @@ public CSAssignment assignContainers(Resource clusterResource, LOG.debug("assignedContainer reserved=" + isReserved + " queue=" + getQueuePath() + " usedCapacity=" + getUsedCapacity() + " absoluteUsedCapacity=" + getAbsoluteUsedCapacity() + " used=" - + queueUsage.getUsed() + " cluster=" + clusterResource); + + usageTracker.getQueueUsage().getUsed() + " cluster=" + clusterResource); LOG.debug( "ParentQ=" + getQueuePath() + " assignedSoFarInThisIteration=" @@ -1038,7 +1037,7 @@ private boolean canAssign(Resource clusterResource, FiCaSchedulerNode node) { // 1) Node doesn't have reserved container // 2) Node's available-resource + killable-resource should > 0 boolean accept = node.getReservedContainer() == null && - Resources.fitsIn(resourceCalculator, minimumAllocation, + Resources.fitsIn(resourceCalculator, queueAllocationSettings.getMinimumAllocation(), Resources.add(node.getUnallocatedResource(), node.getTotalKillableResources())); if (!accept) { ActivitiesLogger.QUEUE.recordQueueActivity(activitiesManager, node, @@ -1065,7 +1064,7 @@ private ResourceLimits getResourceLimitsOfChild(CSQueue child, // First, cap parent limit by parent's max parentLimits.setLimit(Resources.min(resourceCalculator, clusterResource, parentLimits.getLimit(), - queueResourceQuotas.getEffectiveMaxResource(nodePartition))); + usageTracker.getQueueResourceQuotas().getEffectiveMaxResource(nodePartition))); // Parent available resource = parent-limit - parent-used-resource Resource limit = parentLimits.getLimit(); @@ -1073,7 +1072,7 @@ private ResourceLimits getResourceLimitsOfChild(CSQueue child, limit = parentLimits.getNetLimit(); } Resource parentMaxAvailableResource = Resources.subtract( - limit, queueUsage.getUsed(nodePartition)); + limit, usageTracker.getQueueUsage().getUsed(nodePartition)); // Deduct killable from used Resources.addTo(parentMaxAvailableResource, @@ -1085,7 +1084,8 @@ private ResourceLimits getResourceLimitsOfChild(CSQueue child, // Normalize before return childLimit = - Resources.roundDown(resourceCalculator, childLimit, minimumAllocation); + Resources.roundDown(resourceCalculator, childLimit, + queueAllocationSettings.getMinimumAllocation()); return new ResourceLimits(childLimit); } @@ -1270,7 +1270,7 @@ public void updateClusterResource(Resource clusterResource, } // Update effective capacity in all parent queue. - for (String label : configuredNodeLabels) { + for (String label : queueNodeLabelsSettings.getConfiguredNodeLabels()) { calculateEffectiveResourcesAndCapacity(label, clusterResource); } @@ -1332,9 +1332,9 @@ private void calculateEffectiveResourcesAndCapacity(String label, } } else { if (Resources.lessThan(resourceCalculator, clusterResource, - queueResourceQuotas.getEffectiveMinResource(label), + usageTracker.getQueueResourceQuotas().getEffectiveMinResource(label), configuredMinResources)) { - numeratorForMinRatio = queueResourceQuotas + numeratorForMinRatio = usageTracker.getQueueResourceQuotas() .getEffectiveMinResource(label); } } @@ -1344,8 +1344,8 @@ private void calculateEffectiveResourcesAndCapacity(String label, // Update effective resources for my self; if (rootQueue) { - queueResourceQuotas.setEffectiveMinResource(label, resourceByLabel); - queueResourceQuotas.setEffectiveMaxResource(label, resourceByLabel); + usageTracker.getQueueResourceQuotas().setEffectiveMinResource(label, resourceByLabel); + usageTracker.getQueueResourceQuotas().setEffectiveMaxResource(label, resourceByLabel); } else{ super.updateEffectiveResources(clusterResource); } @@ -1442,8 +1442,8 @@ public void attachContainer(Resource clusterResource, .getResource(), node.getPartition()); LOG.info("movedContainer" + " queueMoveIn=" + getQueuePath() + " usedCapacity=" + getUsedCapacity() + " absoluteUsedCapacity=" - + getAbsoluteUsedCapacity() + " used=" + queueUsage.getUsed() + " cluster=" - + clusterResource); + + getAbsoluteUsedCapacity() + " used=" + usageTracker.getQueueUsage().getUsed() + + " cluster=" + clusterResource); // Inform the parent if (parent != null) { parent.attachContainer(clusterResource, application, rmContainer); @@ -1462,8 +1462,8 @@ public void detachContainer(Resource clusterResource, node.getPartition()); LOG.info("movedContainer" + " queueMoveOut=" + getQueuePath() + " usedCapacity=" + getUsedCapacity() + " absoluteUsedCapacity=" - + getAbsoluteUsedCapacity() + " used=" + queueUsage.getUsed() + " cluster=" - + clusterResource); + + getAbsoluteUsedCapacity() + " used=" + usageTracker.getQueueUsage().getUsed() + + " cluster=" + clusterResource); // Inform the parent if (parent != null) { parent.detachContainer(clusterResource, application, rmContainer); @@ -1507,11 +1507,11 @@ void allocateResource(Resource clusterResource, * When this happens, we have to preempt killable container (on same or different * nodes) of parent queue to avoid violating parent's max resource. */ - if (!queueResourceQuotas.getEffectiveMaxResource(nodePartition) + if (!usageTracker.getQueueResourceQuotas().getEffectiveMaxResource(nodePartition) .equals(Resources.none())) { if (Resources.lessThan(resourceCalculator, clusterResource, - queueResourceQuotas.getEffectiveMaxResource(nodePartition), - queueUsage.getUsed(nodePartition))) { + usageTracker.getQueueResourceQuotas().getEffectiveMaxResource(nodePartition), + usageTracker.getQueueUsage().getUsed(nodePartition))) { killContainersToEnforceMaxQueueCapacity(nodePartition, clusterResource); } @@ -1541,7 +1541,7 @@ private void killContainersToEnforceMaxQueueCapacity(String partition, Resource maxResource = getEffectiveMaxCapacity(partition); while (Resources.greaterThan(resourceCalculator, partitionResource, - queueUsage.getUsed(partition), maxResource)) { + usageTracker.getQueueUsage().getUsed(partition), maxResource)) { RMContainer toKillContainer = killableContainerIter.next(); FiCaSchedulerApp attempt = csContext.getApplicationAttempt( toKillContainer.getContainerId().getApplicationAttemptId()); @@ -1584,7 +1584,7 @@ public void apply(Resource cluster, LOG.info("assignedContainer" + " queue=" + getQueuePath() + " usedCapacity=" + getUsedCapacity() + " absoluteUsedCapacity=" - + getAbsoluteUsedCapacity() + " used=" + queueUsage.getUsed() + + getAbsoluteUsedCapacity() + " used=" + usageTracker.getQueueUsage().getUsed() + " cluster=" + cluster); } finally { writeLock.unlock(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/QueueAllocationSettings.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/QueueAllocationSettings.java new file mode 100644 index 0000000000000..5a19a22635d9e --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/QueueAllocationSettings.java @@ -0,0 +1,97 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity; + +import org.apache.hadoop.yarn.api.records.Resource; +import org.apache.hadoop.yarn.api.records.ResourceInformation; +import org.apache.hadoop.yarn.util.resource.ResourceUtils; +import org.apache.hadoop.yarn.util.resource.Resources; + +import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration.UNDEFINED; + +/** + * This class determines minimum and maximum allocation settings based on the + * {@link CapacitySchedulerConfiguration} and other queue + * properties. + **/ +public class QueueAllocationSettings { + private final Resource minimumAllocation; + private Resource maximumAllocation; + + public QueueAllocationSettings(CapacitySchedulerContext csContext) { + this.minimumAllocation = csContext.getMinimumResourceCapability(); + } + + void setupMaximumAllocation(CapacitySchedulerConfiguration csConf, String queuePath, + CSQueue parent, CapacitySchedulerContext csContext) { + /* YARN-10869: When using AutoCreatedLeafQueues, the passed configuration + * object is a cloned one containing only the template configs + * (see ManagedParentQueue#getLeafQueueConfigs). To ensure that the actual + * cluster maximum allocation is fetched the original config object should + * be used. + */ + Resource clusterMax = ResourceUtils + .fetchMaximumAllocationFromConfig(csContext.getConfiguration()); + Resource queueMax = csConf.getQueueMaximumAllocation(queuePath); + + maximumAllocation = Resources.clone( + parent == null ? clusterMax : parent.getMaximumAllocation()); + + String errMsg = + "Queue maximum allocation cannot be larger than the cluster setting" + + " for queue " + queuePath + + " max allocation per queue: %s" + + " cluster setting: " + clusterMax; + + if (queueMax == Resources.none()) { + // Handle backward compatibility + long queueMemory = csConf.getQueueMaximumAllocationMb(queuePath); + int queueVcores = csConf.getQueueMaximumAllocationVcores(queuePath); + if (queueMemory != UNDEFINED) { + maximumAllocation.setMemorySize(queueMemory); + } + + if (queueVcores != UNDEFINED) { + maximumAllocation.setVirtualCores(queueVcores); + } + + if ((queueMemory != UNDEFINED && queueMemory > clusterMax.getMemorySize() + || (queueVcores != UNDEFINED + && queueVcores > clusterMax.getVirtualCores()))) { + throw new IllegalArgumentException( + String.format(errMsg, maximumAllocation)); + } + } else { + // Queue level maximum-allocation can't be larger than cluster setting + for (ResourceInformation ri : queueMax.getResources()) { + if (ri.compareTo(clusterMax.getResourceInformation(ri.getName())) > 0) { + throw new IllegalArgumentException(String.format(errMsg, queueMax)); + } + + maximumAllocation.setResourceInformation(ri.getName(), ri); + } + } + } + + public Resource getMinimumAllocation() { + return minimumAllocation; + } + + public Resource getMaximumAllocation() { + return maximumAllocation; + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/QueueAppLifetimeAndLimitSettings.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/QueueAppLifetimeAndLimitSettings.java new file mode 100644 index 0000000000000..e0f4d60fe4c66 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/QueueAppLifetimeAndLimitSettings.java @@ -0,0 +1,132 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity; + +import org.apache.hadoop.yarn.exceptions.YarnRuntimeException; + +/** + * This class determines application lifetime and max parallel apps settings based on the + * {@link CapacitySchedulerConfiguration} and other queue + * properties. + **/ +public class QueueAppLifetimeAndLimitSettings { + // -1 indicates lifetime is disabled + private final long maxApplicationLifetime; + private final long defaultApplicationLifetime; + + // Indicates if this queue's default lifetime was set by a config property, + // either at this level or anywhere in the queue's hierarchy. + private boolean defaultAppLifetimeWasSpecifiedInConfig = false; + + private int maxParallelApps; + + public QueueAppLifetimeAndLimitSettings(CapacitySchedulerConfiguration configuration, + AbstractCSQueue q, String queuePath) { + // Store max parallel apps property + this.maxParallelApps = configuration.getMaxParallelAppsForQueue(queuePath); + this.maxApplicationLifetime = getInheritedMaxAppLifetime(q, configuration); + this.defaultApplicationLifetime = setupInheritedDefaultAppLifetime(q, queuePath, configuration, + maxApplicationLifetime); + } + + private long getInheritedMaxAppLifetime(CSQueue q, CapacitySchedulerConfiguration conf) { + CSQueue parentQ = q.getParent(); + long maxAppLifetime = conf.getMaximumLifetimePerQueue(q.getQueuePath()); + + // If q is the root queue, then get max app lifetime from conf. + if (parentQ == null) { + return maxAppLifetime; + } + + // If this is not the root queue, get this queue's max app lifetime + // from the conf. The parent's max app lifetime will be used if it's + // not set for this queue. + // A value of 0 will override the parent's value and means no max lifetime. + // A negative value means that the parent's max should be used. + long parentsMaxAppLifetime = parentQ.getMaximumApplicationLifetime(); + return (maxAppLifetime >= 0) ? maxAppLifetime : parentsMaxAppLifetime; + } + + private long setupInheritedDefaultAppLifetime(CSQueue q, + String queuePath, CapacitySchedulerConfiguration conf, long myMaxAppLifetime) { + CSQueue parentQ = q.getParent(); + long defaultAppLifetime = conf.getDefaultLifetimePerQueue(queuePath); + defaultAppLifetimeWasSpecifiedInConfig = + (defaultAppLifetime >= 0 + || (parentQ != null && + parentQ.getDefaultAppLifetimeWasSpecifiedInConfig())); + + // If q is the root queue, then get default app lifetime from conf. + if (parentQ == null) { + return defaultAppLifetime; + } + + // If this is not the root queue, get the parent's default app lifetime. The + // parent's default app lifetime will be used if not set for this queue. + long parentsDefaultAppLifetime = parentQ.getDefaultApplicationLifetime(); + + // Negative value indicates default lifetime was not set at this level. + // If default lifetime was not set at this level, calculate it based on + // parent's default lifetime or current queue's max lifetime. + if (defaultAppLifetime < 0) { + // If default lifetime was not set at this level but was set somewhere in + // the parent's hierarchy, set default lifetime to parent queue's default + // only if parent queue's lifetime is less than current queue's max + // lifetime. Otherwise, use current queue's max lifetime value for its + // default lifetime. + if (defaultAppLifetimeWasSpecifiedInConfig) { + defaultAppLifetime = + Math.min(parentsDefaultAppLifetime, myMaxAppLifetime); + } else { + // Default app lifetime value was not set anywhere in this queue's + // hierarchy. Use current queue's max lifetime as its default. + defaultAppLifetime = myMaxAppLifetime; + } + } // else if >= 0, default lifetime was set at this level. Just use it. + + if (myMaxAppLifetime > 0 && defaultAppLifetime > myMaxAppLifetime) { + throw new YarnRuntimeException( + "Default lifetime " + defaultAppLifetime + + " can't exceed maximum lifetime " + myMaxAppLifetime); + } + + if (defaultAppLifetime <= 0) { + defaultAppLifetime = myMaxAppLifetime; + } + return defaultAppLifetime; + } + + public int getMaxParallelApps() { + return maxParallelApps; + } + + public void setMaxParallelApps(int maxParallelApps) { + this.maxParallelApps = maxParallelApps; + } + + public long getMaxApplicationLifetime() { + return maxApplicationLifetime; + } + + public long getDefaultApplicationLifetime() { + return defaultApplicationLifetime; + } + + public boolean isDefaultAppLifetimeWasSpecifiedInConfig() { + return defaultAppLifetimeWasSpecifiedInConfig; + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/QueueCapacityVector.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/QueueCapacityVector.java new file mode 100644 index 0000000000000..9f6e0e264a320 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/QueueCapacityVector.java @@ -0,0 +1,258 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity; + +import org.apache.hadoop.yarn.api.records.ResourceInformation; + +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Iterator; +import java.util.Map; +import java.util.Set; + +/** + * Contains capacity values with calculation types associated for each + * resource. + */ +public class QueueCapacityVector implements + Iterable { + private static final String START_PARENTHESES = "["; + private static final String END_PARENTHESES = "]"; + private static final String RESOURCE_DELIMITER = ","; + private static final String VALUE_DELIMITER = "="; + + private final ResourceVector resource; + private final Map capacityTypes + = new HashMap<>(); + private final Map> capacityTypePerResource + = new HashMap<>(); + + public QueueCapacityVector() { + this.resource = new ResourceVector(); + } + + private QueueCapacityVector(ResourceVector resource) { + this.resource = resource; + } + + /** + * Creates a zero {@code QueueCapacityVector}. The resources are defined + * in absolute capacity type by default. + * + * @return zero capacity vector + */ + public static QueueCapacityVector newInstance() { + QueueCapacityVector newCapacityVector = + new QueueCapacityVector(ResourceVector.newInstance()); + for (Map.Entry resourceEntry : newCapacityVector.resource) { + newCapacityVector.storeResourceType(resourceEntry.getKey(), + QueueCapacityType.ABSOLUTE); + } + + return newCapacityVector; + } + + /** + * Creates a uniform and homogeneous {@code QueueCapacityVector}. + * The resources are defined in absolute capacity type by default. + * + * @param value value to be set for each resource + * @param capacityType capacity type to be set for each resource + * @return uniform capacity vector + */ + public static QueueCapacityVector of( + float value, QueueCapacityType capacityType) { + QueueCapacityVector newCapacityVector = + new QueueCapacityVector(ResourceVector.of(value)); + for (Map.Entry resourceEntry : newCapacityVector.resource) { + newCapacityVector.storeResourceType(resourceEntry.getKey(), capacityType); + } + + return newCapacityVector; + } + + public QueueCapacityVectorEntry getResource(String resourceName) { + return new QueueCapacityVectorEntry(capacityTypes.get(resourceName), + resourceName, resource.getValue(resourceName)); + } + + /** + * Returns the number of resources defined for this vector. + * + * @return number of resources + */ + public int getResourceCount() { + return capacityTypes.size(); + } + + /** + * Set the value and capacity type of a resource. + * + * @param resourceName name of the resource + * @param value value of the resource + * @param capacityType type of the resource + */ + public void setResource(String resourceName, float value, + QueueCapacityType capacityType) { + // Necessary due to backward compatibility (memory = memory-mb) + String convertedResourceName = resourceName; + if (resourceName.equals("memory")) { + convertedResourceName = ResourceInformation.MEMORY_URI; + } + resource.setValue(convertedResourceName, value); + storeResourceType(convertedResourceName, capacityType); + } + + /** + * A shorthand to retrieve the value stored for the memory resource. + * + * @return value of memory resource + */ + public float getMemory() { + return resource.getValue(ResourceInformation.MEMORY_URI); + } + + /** + * Returns the name of all resources that are defined in the given capacity + * type. + * + * @param capacityType the capacity type of the resources + * @return all resource names for the given capacity type + */ + public Set getResourceNamesByCapacityType( + QueueCapacityType capacityType) { + return capacityTypePerResource.getOrDefault(capacityType, + Collections.emptySet()); + } + + public boolean isResourceOfType( + String resourceName, QueueCapacityType capacityType) { + return capacityTypes.containsKey(resourceName) && + capacityTypes.get(resourceName).equals(capacityType); + } + + @Override + public Iterator iterator() { + return new Iterator() { + private final Iterator> resources = + resource.iterator(); + private int i = 0; + + @Override + public boolean hasNext() { + return resources.hasNext() && capacityTypes.size() > i; + } + + @Override + public QueueCapacityVectorEntry next() { + Map.Entry resourceInformation = resources.next(); + i++; + return new QueueCapacityVectorEntry( + capacityTypes.get(resourceInformation.getKey()), + resourceInformation.getKey(), resourceInformation.getValue()); + } + }; + } + + /** + * Returns a set of all capacity type defined for this vector. + * + * @return capacity types + */ + public Set getDefinedCapacityTypes() { + return capacityTypePerResource.keySet(); + } + + private void storeResourceType( + String resourceName, QueueCapacityType resourceType) { + if (capacityTypes.get(resourceName) != null + && !capacityTypes.get(resourceName).equals(resourceType)) { + capacityTypePerResource.get(capacityTypes.get(resourceName)) + .remove(resourceName); + } + + capacityTypePerResource.putIfAbsent(resourceType, new HashSet<>()); + capacityTypePerResource.get(resourceType).add(resourceName); + capacityTypes.put(resourceName, resourceType); + } + + @Override + public String toString() { + StringBuilder stringVector = new StringBuilder(); + stringVector.append(START_PARENTHESES); + + int resourceCount = 0; + for (Map.Entry resourceEntry : resource) { + resourceCount++; + stringVector.append(resourceEntry.getKey()) + .append(VALUE_DELIMITER) + .append(resourceEntry.getValue()) + .append(capacityTypes.get(resourceEntry.getKey()).postfix); + if (resourceCount < capacityTypes.size()) { + stringVector.append(RESOURCE_DELIMITER); + } + } + + stringVector.append(END_PARENTHESES); + + return stringVector.toString(); + } + + /** + * Represents a capacity type associated with its syntax postfix. + */ + public enum QueueCapacityType { + PERCENTAGE("%"), ABSOLUTE(""), WEIGHT("w"); + private final String postfix; + + QueueCapacityType(String postfix) { + this.postfix = postfix; + } + + public String getPostfix() { + return postfix; + } + } + + public static class QueueCapacityVectorEntry { + private final QueueCapacityType vectorResourceType; + private final float resourceValue; + private final String resourceName; + + public QueueCapacityVectorEntry(QueueCapacityType vectorResourceType, + String resourceName, float resourceValue) { + this.vectorResourceType = vectorResourceType; + this.resourceValue = resourceValue; + this.resourceName = resourceName; + } + + public QueueCapacityType getVectorResourceType() { + return vectorResourceType; + } + + public float getResourceValue() { + return resourceValue; + } + + public String getResourceName() { + return resourceName; + } + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/QueueNodeLabelsSettings.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/QueueNodeLabelsSettings.java new file mode 100644 index 0000000000000..827259f1ae97a --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/QueueNodeLabelsSettings.java @@ -0,0 +1,147 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity; + +import org.apache.commons.lang3.StringUtils; +import org.apache.hadoop.util.Sets; +import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager; +import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration.ROOT; +import java.io.IOException; +import java.util.Set; + +/** + * This class determines accessible node labels, configured node labels and the default node + * label expression based on the {@link CapacitySchedulerConfiguration} object and other queue + * properties. + */ +public class QueueNodeLabelsSettings { + private final CSQueue parent; + private final String queuePath; + private final CapacitySchedulerContext csContext; + private Set accessibleLabels; + private Set configuredNodeLabels; + private String defaultLabelExpression; + + public QueueNodeLabelsSettings(CapacitySchedulerConfiguration configuration, + CSQueue parent, + String queuePath, + CapacitySchedulerContext csContext) throws IOException { + this.parent = parent; + this.queuePath = queuePath; + this.csContext = csContext; + initializeNodeLabels(configuration); + } + + private void initializeNodeLabels(CapacitySchedulerConfiguration configuration) + throws IOException { + initializeAccessibleLabels(configuration); + initializeDefaultLabelExpression(configuration); + initializeConfiguredNodeLabels(); + validateNodeLabels(); + } + + private void initializeAccessibleLabels(CapacitySchedulerConfiguration configuration) { + this.accessibleLabels = configuration.getAccessibleNodeLabels(queuePath); + // Inherit labels from parent if not set + if (this.accessibleLabels == null && parent != null) { + this.accessibleLabels = parent.getAccessibleNodeLabels(); + } + } + + private void initializeDefaultLabelExpression(CapacitySchedulerConfiguration configuration) { + this.defaultLabelExpression = configuration.getDefaultNodeLabelExpression(queuePath); + // If the accessible labels is not null and the queue has a parent with a + // similar set of labels copy the defaultNodeLabelExpression from the parent + if (this.accessibleLabels != null && parent != null + && this.defaultLabelExpression == null && + this.accessibleLabels.containsAll(parent.getAccessibleNodeLabels())) { + this.defaultLabelExpression = parent.getDefaultNodeLabelExpression(); + } + } + + private void initializeConfiguredNodeLabels() { + if (csContext.getCapacitySchedulerQueueManager() != null + && csContext.getCapacitySchedulerQueueManager().getConfiguredNodeLabels() != null) { + if (queuePath.equals(ROOT)) { + this.configuredNodeLabels = csContext.getCapacitySchedulerQueueManager() + .getConfiguredNodeLabels().getAllConfiguredLabels(); + } else { + this.configuredNodeLabels = csContext.getCapacitySchedulerQueueManager() + .getConfiguredNodeLabels().getLabelsByQueue(queuePath); + } + } else { + // Fallback to suboptimal but correct logic + this.configuredNodeLabels = csContext.getConfiguration().getConfiguredNodeLabels(queuePath); + } + } + + private void validateNodeLabels() throws IOException { + // Check if labels of this queue is a subset of parent queue, only do this + // when the queue in question is not root + if (isNotRoot()) { + if (parent.getAccessibleNodeLabels() != null && !parent + .getAccessibleNodeLabels().contains(RMNodeLabelsManager.ANY)) { + // If parent isn't "*", child shouldn't be "*" too + if (this.getAccessibleNodeLabels().contains(RMNodeLabelsManager.ANY)) { + throw new IOException("Parent's accessible queue is not ANY(*), " + + "but child's accessible queue is " + RMNodeLabelsManager.ANY); + } else { + Set diff = Sets.difference(this.getAccessibleNodeLabels(), + parent.getAccessibleNodeLabels()); + if (!diff.isEmpty()) { + throw new IOException(String.format( + "Some labels of child queue is not a subset of parent queue, these labels=[%s]", + StringUtils.join(diff, ","))); + } + } + } + } + } + + private boolean isNotRoot() { + return parent != null && parent.getParent() != null; + } + + public boolean isAccessibleToPartition(String nodePartition) { + // if queue's label is *, it can access any node + if (accessibleLabels != null && accessibleLabels.contains(RMNodeLabelsManager.ANY)) { + return true; + } + // any queue can access to a node without label + if (nodePartition == null || nodePartition.equals(RMNodeLabelsManager.NO_LABEL)) { + return true; + } + // a queue can access to a node only if it contains any label of the node + if (accessibleLabels != null && accessibleLabels.contains(nodePartition)) { + return true; + } + // The partition cannot be accessed + return false; + } + + public Set getAccessibleNodeLabels() { + return accessibleLabels; + } + + public Set getConfiguredNodeLabels() { + return configuredNodeLabels; + } + + public String getDefaultLabelExpression() { + return defaultLabelExpression; + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ResourceVector.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ResourceVector.java new file mode 100644 index 0000000000000..88c09af6b0911 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ResourceVector.java @@ -0,0 +1,129 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity; + +import org.apache.hadoop.yarn.api.records.Resource; +import org.apache.hadoop.yarn.api.records.ResourceInformation; +import org.apache.hadoop.yarn.util.resource.ResourceUtils; + +import java.util.HashMap; +import java.util.Iterator; +import java.util.Map; + +/** + * Represents a simple resource floating point value storage + * grouped by resource names. + */ +public class ResourceVector implements Iterable> { + private final Map resourcesByName = new HashMap<>(); + + /** + * Creates a new {@code ResourceVector} with all pre-defined resources set to + * zero. + * @return zero resource vector + */ + public static ResourceVector newInstance() { + ResourceVector zeroResourceVector = new ResourceVector(); + for (ResourceInformation resource : ResourceUtils.getResourceTypesArray()) { + zeroResourceVector.setValue(resource.getName(), 0); + } + + return zeroResourceVector; + } + + /** + * Creates a new {@code ResourceVector} with all pre-defined resources set to + * the same value. + * @param value the value to set all resources to + * @return uniform resource vector + */ + public static ResourceVector of(float value) { + ResourceVector emptyResourceVector = new ResourceVector(); + for (ResourceInformation resource : ResourceUtils.getResourceTypesArray()) { + emptyResourceVector.setValue(resource.getName(), value); + } + + return emptyResourceVector; + } + + /** + * Creates a new {@code ResourceVector} with the values set in a + * {@code Resource} object. + * @param resource resource object the resource vector will be based on + * @return uniform resource vector + */ + public static ResourceVector of(Resource resource) { + ResourceVector resourceVector = new ResourceVector(); + for (ResourceInformation resourceInformation : resource.getResources()) { + resourceVector.setValue(resourceInformation.getName(), + resourceInformation.getValue()); + } + + return resourceVector; + } + + /** + * Subtract values for each resource defined in the given resource vector. + * @param otherResourceVector rhs resource vector of the subtraction + */ + public void subtract(ResourceVector otherResourceVector) { + for (Map.Entry resource : otherResourceVector) { + setValue(resource.getKey(), getValue(resource.getKey()) - resource.getValue()); + } + } + + /** + * Increments the given resource by the specified value. + * @param resourceName name of the resource + * @param value value to be added to the resource's current value + */ + public void increment(String resourceName, float value) { + setValue(resourceName, getValue(resourceName) + value); + } + + public Float getValue(String resourceName) { + return resourcesByName.get(resourceName); + } + + public void setValue(String resourceName, float value) { + resourcesByName.put(resourceName, value); + } + + @Override + public Iterator> iterator() { + return resourcesByName.entrySet().iterator(); + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + + return this.resourcesByName.equals(((ResourceVector) o).resourcesByName); + } + + @Override + public int hashCode() { + return resourcesByName.hashCode(); + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/QueueCapacityConfigParser.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/QueueCapacityConfigParser.java new file mode 100644 index 0000000000000..28eb33c553636 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/QueueCapacityConfigParser.java @@ -0,0 +1,215 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.conf; + +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.QueueCapacityVector; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.QueueCapacityVector.QueueCapacityType; +import org.apache.hadoop.yarn.util.UnitsConversionUtil; + +import java.util.ArrayList; +import java.util.List; +import java.util.function.Function; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +/** + * A class that parses {@code QueueCapacityVector} from the capacity + * configuration property set for a queue. + * + * A new syntax for capacity property could be implemented, by creating a parser + * with a regex to match the pattern and a method that creates a + * {@code QueueCapacityVector} from the matched pattern. + * Extending the parsers field with a {@code Parser} object in the constructor + * is needed in this case. + * + * A new capacity type for the existing parsers could be added by extending + * the {@code QueueCapacityVector.QueueCapacityType} with a new type and its + * associated postfix symbol. + */ +public class QueueCapacityConfigParser { + private static final String UNIFORM_REGEX = "^([0-9.]+)(.*)"; + private static final String RESOURCE_REGEX = "^\\[([\\w\\.,\\-_%\\ /]+=[\\w\\.,\\-_%\\ /]+)+\\]$"; + + private static final Pattern RESOURCE_PATTERN = Pattern.compile(RESOURCE_REGEX); + private static final Pattern UNIFORM_PATTERN = Pattern.compile(UNIFORM_REGEX); + public static final String FLOAT_DIGIT_REGEX = "[0-9.]"; + + private final List parsers = new ArrayList<>(); + + public QueueCapacityConfigParser() { + parsers.add(new Parser(RESOURCE_PATTERN, this::heterogeneousParser)); + parsers.add(new Parser(UNIFORM_PATTERN, this::uniformParser)); + } + + /** + * Creates a {@code QueueCapacityVector} parsed from the capacity configuration + * property set for a queue. + * @param conf configuration object + * @param queuePath queue for which the capacity property is parsed + * @param label node label + * @return a parsed capacity vector + */ + public QueueCapacityVector parse(CapacitySchedulerConfiguration conf, + String queuePath, String label) { + + if (queuePath.equals(CapacitySchedulerConfiguration.ROOT)) { + return QueueCapacityVector.of(100f, QueueCapacityType.PERCENTAGE); + } + + String propertyName = CapacitySchedulerConfiguration.getNodeLabelPrefix( + queuePath, label) + CapacitySchedulerConfiguration.CAPACITY; + String capacityString = conf.get(propertyName); + + if (capacityString == null) { + return new QueueCapacityVector(); + } + // Trim all spaces from capacity string + capacityString = capacityString.replaceAll(" ", ""); + + for (Parser parser : parsers) { + Matcher matcher = parser.regex.matcher(capacityString); + if (matcher.find()) { + return parser.parser.apply(matcher); + } + } + + return new QueueCapacityVector(); + } + + /** + * A parser method that is usable on uniform capacity values e.g. percentage or + * weight. + * @param matcher a regex matcher that contains parsed value and its possible + * suffix + * @return a parsed capacity vector + */ + private QueueCapacityVector uniformParser(Matcher matcher) { + QueueCapacityType capacityType = null; + String value = matcher.group(1); + if (matcher.groupCount() == 2) { + String matchedSuffix = matcher.group(2); + for (QueueCapacityType suffix : QueueCapacityType.values()) { + // Absolute uniform syntax is not supported + if (suffix.equals(QueueCapacityType.ABSOLUTE)) { + continue; + } + // when capacity is given in percentage, we do not need % symbol + String uniformSuffix = suffix.getPostfix().replaceAll("%", ""); + if (uniformSuffix.equals(matchedSuffix)) { + capacityType = suffix; + } + } + } + + if (capacityType == null) { + return new QueueCapacityVector(); + } + + return QueueCapacityVector.of(Float.parseFloat(value), capacityType); + } + + /** + * A parser method that is usable on resource capacity values e.g. mixed or + * absolute resource. + * @param matcher a regex matcher that contains the matched resource string + * @return a parsed capacity vector + */ + private QueueCapacityVector heterogeneousParser(Matcher matcher) { + QueueCapacityVector capacityVector = QueueCapacityVector.newInstance(); + + /* + * Absolute resource configuration for a queue will be grouped by "[]". + * Syntax of absolute resource config could be like below + * "memory=4Gi vcores=2". Ideally this means "4GB of memory and 2 vcores". + */ + // Get the sub-group. + String bracketedGroup = matcher.group(0); + // Get the string inside starting and closing [] + bracketedGroup = bracketedGroup.substring(1, bracketedGroup.length() - 1); + // Split by comma and equals delimiter eg. the string memory=1024,vcores=6 + // is converted to an array of array as {{memory,1024}, {vcores, 6}} + for (String kvPair : bracketedGroup.trim().split(",")) { + String[] splits = kvPair.split("="); + + // Ensure that each sub string is key value pair separated by '='. + if (splits.length > 1) { + setCapacityVector(capacityVector, splits[0], splits[1]); + } + } + + // Memory always have to be defined + if (capacityVector.getMemory() == 0L) { + return new QueueCapacityVector(); + } + + return capacityVector; + } + + private void setCapacityVector( + QueueCapacityVector resource, String resourceName, String resourceValue) { + QueueCapacityType capacityType = QueueCapacityType.ABSOLUTE; + + // Extract suffix from a value e.g. for 6w extract w + String suffix = resourceValue.replaceAll(FLOAT_DIGIT_REGEX, ""); + if (!resourceValue.endsWith(suffix)) { + return; + } + + float parsedResourceValue = Float.parseFloat(resourceValue.substring( + 0, resourceValue.length() - suffix.length())); + float convertedValue = parsedResourceValue; + + if (!suffix.isEmpty() && UnitsConversionUtil.KNOWN_UNITS.contains(suffix)) { + // Convert all incoming units to MB if units is configured. + convertedValue = UnitsConversionUtil.convert(suffix, "Mi", (long) parsedResourceValue); + } else { + for (QueueCapacityType capacityTypeSuffix : QueueCapacityType.values()) { + if (capacityTypeSuffix.getPostfix().equals(suffix)) { + capacityType = capacityTypeSuffix; + } + } + } + + resource.setResource(resourceName, convertedValue, capacityType); + } + + /** + * Checks whether the given capacity string is in a capacity vector compatible + * format. + * @param configuredCapacity capacity string + * @return true, if capacity string is in capacity vector format, + * false otherwise + */ + public boolean isCapacityVectorFormat(String configuredCapacity) { + return configuredCapacity != null + && RESOURCE_PATTERN.matcher(configuredCapacity).find(); + } + + private static class Parser { + private final Pattern regex; + private final Function parser; + + Parser(Pattern regex, Function parser) { + this.regex = regex; + this.parser = parser; + } + } + +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/queuemanagement/DeactivatedLeafQueuesByLabel.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/queuemanagement/DeactivatedLeafQueuesByLabel.java new file mode 100644 index 0000000000000..31486a159d0bc --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/queuemanagement/DeactivatedLeafQueuesByLabel.java @@ -0,0 +1,107 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.queuemanagement; + +import org.apache.hadoop.classification.VisibleForTesting; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.QueueCapacities; +import org.slf4j.Logger; + +import java.util.Map; +import java.util.Set; + +import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CSQueueUtils.EPSILON; + +public class DeactivatedLeafQueuesByLabel { + private String parentQueuePath; + private String nodeLabel; + private Map deactivatedLeafQueues; + private float sumOfChildQueueActivatedCapacity; + private float parentAbsoluteCapacity; + private float leafQueueTemplateAbsoluteCapacity; + private float availableCapacity; + private float totalDeactivatedCapacity; + + @VisibleForTesting + public DeactivatedLeafQueuesByLabel() {} + + public DeactivatedLeafQueuesByLabel( + Map deactivatedLeafQueues, + String parentQueuePath, + String nodeLabel, + float sumOfChildQueueActivatedCapacity, + float parentAbsoluteCapacity, + float leafQueueTemplateAbsoluteCapacity) { + this.parentQueuePath = parentQueuePath; + this.nodeLabel = nodeLabel; + this.deactivatedLeafQueues = deactivatedLeafQueues; + this.sumOfChildQueueActivatedCapacity = sumOfChildQueueActivatedCapacity; + this.parentAbsoluteCapacity = parentAbsoluteCapacity; + this.leafQueueTemplateAbsoluteCapacity = leafQueueTemplateAbsoluteCapacity; + + this.totalDeactivatedCapacity = getTotalDeactivatedCapacity(); + this.availableCapacity = parentAbsoluteCapacity - sumOfChildQueueActivatedCapacity + + this.totalDeactivatedCapacity + EPSILON; + } + + float getTotalDeactivatedCapacity() { + float deactivatedCapacity = 0; + for (Map.Entry deactivatedQueueCapacity : + deactivatedLeafQueues.entrySet()) { + deactivatedCapacity += deactivatedQueueCapacity.getValue().getAbsoluteCapacity(nodeLabel); + } + return deactivatedCapacity; + } + + public Set getQueues() { + return deactivatedLeafQueues.keySet(); + } + + public void printToDebug(Logger logger) { + if (logger.isDebugEnabled()) { + logger.debug("Parent queue = {}, nodeLabel = {}, absCapacity = {}, " + + "leafQueueAbsoluteCapacity = {}, deactivatedCapacity = {}, " + + "absChildActivatedCapacity = {}, availableCapacity = {}", + parentQueuePath, nodeLabel, parentAbsoluteCapacity, + leafQueueTemplateAbsoluteCapacity, getTotalDeactivatedCapacity(), + sumOfChildQueueActivatedCapacity, availableCapacity); + } + } + + @VisibleForTesting + public int getMaxLeavesToBeActivated(int numPendingApps) { + float childQueueAbsoluteCapacity = leafQueueTemplateAbsoluteCapacity; + if (childQueueAbsoluteCapacity > 0) { + int numLeafQueuesNeeded = (int) Math.floor(availableCapacity / childQueueAbsoluteCapacity); + return Math.min(numLeafQueuesNeeded, numPendingApps); + } + return 0; + } + + public boolean canActivateLeafQueues() { + return availableCapacity >= leafQueueTemplateAbsoluteCapacity; + } + + @VisibleForTesting + public void setAvailableCapacity(float availableCapacity) { + this.availableCapacity = availableCapacity; + } + + @VisibleForTesting + public void setLeafQueueTemplateAbsoluteCapacity(float leafQueueTemplateAbsoluteCapacity) { + this.leafQueueTemplateAbsoluteCapacity = leafQueueTemplateAbsoluteCapacity; + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/queuemanagement/GuaranteedOrZeroCapacityOverTimePolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/queuemanagement/GuaranteedOrZeroCapacityOverTimePolicy.java index 5647b9146ed66..76ab7cab7ac58 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/queuemanagement/GuaranteedOrZeroCapacityOverTimePolicy.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/queuemanagement/GuaranteedOrZeroCapacityOverTimePolicy.java @@ -21,12 +21,9 @@ import org.apache.hadoop.classification.VisibleForTesting; import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CSQueueUtils; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity - .QueueManagementDynamicEditPolicy; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceLimits; import org.apache.hadoop.yarn.server.resourcemanager.scheduler .SchedulerDynamicEditException; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity @@ -84,6 +81,7 @@ public class GuaranteedOrZeroCapacityOverTimePolicy implements AutoCreatedQueueManagementPolicy { + private static final int DEFAULT_QUEUE_PRINT_SIZE_LIMIT = 25; private CapacitySchedulerContext scheduler; private ManagedParentQueue managedParentQueue; @@ -345,12 +343,11 @@ private void initializeLeafQueueTemplate(ManagedParentQueue parentQueue) } /** - * Compute/Adjust child queue capacities - * for auto created leaf queues - * This computes queue entitlements but does not update LeafQueueState or - * queue capacities. Scheduler calls commitQueueManagemetChanges after - * validation after applying queue changes and commits to LeafQueueState - * are done in commitQueueManagementChanges. + * Computes / adjusts child queue capacities for auto created leaf queues. + * This method computes queue entitlements but does not update LeafQueueState or + * queue capacities. + * Scheduler calls commitQueueManagementChanges after validation after applying queue changes + * and commits to LeafQueueState are done in commitQueueManagementChanges. * * @return List of Queue Management change suggestions which could potentially * be committed/rejected by the scheduler due to validation failures @@ -367,117 +364,98 @@ public List computeQueueManagementChanges() managedParentQueue.getAutoCreatedQueueManagementPolicy()); //TODO : Add support for node labels on leaf queue template configurations - //synch/add missing leaf queue(s) if any to state + //sync / add missing leaf queue(s) if any TO state updateLeafQueueState(); readLock.lock(); try { - List queueManagementChanges = new ArrayList<>(); - List pendingApps = getSortedPendingApplications(); - - //Map of LeafQueue->QueueCapacities - keep adding the computed - // entitlements to this map and finally - // build the leaf queue configuration Template for all identified leaf - // queues - Map leafQueueEntitlements = new HashMap<>(); + LeafQueueEntitlements leafQueueEntitlements = new LeafQueueEntitlements(); for (String nodeLabel : leafQueueTemplateNodeLabels) { - // check if any leaf queues need to be deactivated based on pending - // applications - float parentAbsoluteCapacity = - managedParentQueue.getQueueCapacities().getAbsoluteCapacity( - nodeLabel); - float leafQueueTemplateAbsoluteCapacity = - leafQueueTemplateCapacities.getAbsoluteCapacity(nodeLabel); - Map deactivatedLeafQueues = - deactivateLeafQueuesIfInActive(managedParentQueue, nodeLabel, - leafQueueEntitlements); - - if (LOG.isDebugEnabled()) { - if ( deactivatedLeafQueues.size() > 0) { - LOG.debug("Parent queue = {}, " + - ", nodeLabel = {}, deactivated leaf queues = [{}] ", - managedParentQueue.getQueuePath(), nodeLabel, - deactivatedLeafQueues.size() > 25 ? deactivatedLeafQueues - .size() : deactivatedLeafQueues); - - } - } - - float deactivatedCapacity = getTotalDeactivatedCapacity( - deactivatedLeafQueues, nodeLabel); - - float sumOfChildQueueActivatedCapacity = parentQueueState. - getAbsoluteActivatedChildQueueCapacity(nodeLabel); + DeactivatedLeafQueuesByLabel deactivatedLeafQueues = + deactivateLeafQueues(nodeLabel, leafQueueEntitlements); + deactivatedLeafQueues.printToDebug(LOG); //Check if we need to activate anything at all? - float availableCapacity = - parentAbsoluteCapacity - sumOfChildQueueActivatedCapacity - + deactivatedCapacity + EPSILON; - - if (LOG.isDebugEnabled()) { - LOG.debug("Parent queue = " + managedParentQueue.getQueuePath() - + ", nodeLabel = " + nodeLabel + ", absCapacity = " - + parentAbsoluteCapacity + ", leafQueueAbsoluteCapacity = " - + leafQueueTemplateAbsoluteCapacity + ", deactivatedCapacity = " - + deactivatedCapacity + " , absChildActivatedCapacity = " - + sumOfChildQueueActivatedCapacity + ", availableCapacity = " - + availableCapacity); - } - - if (availableCapacity >= leafQueueTemplateAbsoluteCapacity) { - //sort applications across leaf queues by submit time - if (pendingApps.size() > 0) { - int maxLeafQueuesTobeActivated = getMaxLeavesToBeActivated( - availableCapacity, leafQueueTemplateAbsoluteCapacity, - pendingApps.size()); - - if (LOG.isDebugEnabled()) { - LOG.debug("Parent queue = " + managedParentQueue.getQueuePath() - + " : Found " + maxLeafQueuesTobeActivated + " leaf queues" - + " to be activated with " + pendingApps.size() + " apps "); - } - - LinkedHashSet leafQueuesToBeActivated = getSortedLeafQueues( - nodeLabel, pendingApps, maxLeafQueuesTobeActivated, - deactivatedLeafQueues.keySet()); - - //Compute entitlement changes for the identified leaf queues - // which is appended to the List of computedEntitlements - updateLeafQueueCapacitiesByLabel(nodeLabel, leafQueuesToBeActivated, - leafQueueEntitlements); - - if (LOG.isDebugEnabled()) { - if (leafQueuesToBeActivated.size() > 0) { - LOG.debug("Activated leaf queues : [{}]", - leafQueuesToBeActivated.size() < 25 ? - leafQueuesToBeActivated : leafQueuesToBeActivated.size()); - } - } - } + if (deactivatedLeafQueues.canActivateLeafQueues()) { + activateLeafQueues(leafQueueEntitlements, nodeLabel, deactivatedLeafQueues); } } //Populate new entitlements - - for (final Iterator> iterator = - leafQueueEntitlements.entrySet().iterator(); iterator.hasNext(); ) { - Map.Entry queueCapacities = iterator.next(); - String leafQueueName = queueCapacities.getKey(); + return leafQueueEntitlements.mapToQueueManagementChanges((leafQueueName, capacities) -> { AutoCreatedLeafQueue leafQueue = (AutoCreatedLeafQueue) scheduler.getCapacitySchedulerQueueManager() .getQueue(leafQueueName); - AutoCreatedLeafQueueConfig newTemplate = buildTemplate( - queueCapacities.getValue()); - queueManagementChanges.add( - new QueueManagementChange.UpdateQueue(leafQueue, newTemplate)); - - } - return queueManagementChanges; + AutoCreatedLeafQueueConfig newTemplate = buildTemplate(capacities); + return new QueueManagementChange.UpdateQueue(leafQueue, newTemplate); + }); } finally { readLock.unlock(); } } + private void activateLeafQueues(LeafQueueEntitlements leafQueueEntitlements, String nodeLabel, + DeactivatedLeafQueuesByLabel deactivatedLeafQueues) throws SchedulerDynamicEditException { + //sort applications across leaf queues by submit time + List pendingApps = getSortedPendingApplications(); + if (pendingApps.size() > 0) { + int maxLeafQueuesTobeActivated = deactivatedLeafQueues. + getMaxLeavesToBeActivated(pendingApps.size()); + + if (LOG.isDebugEnabled()) { + LOG.debug("Parent queue = {}, Found {} leaf queues to be activated with {} aps", + managedParentQueue.getQueuePath(), maxLeafQueuesTobeActivated, pendingApps.size()); + } + + Set leafQueuesToBeActivated = getSortedLeafQueues( + nodeLabel, pendingApps, maxLeafQueuesTobeActivated, + deactivatedLeafQueues.getQueues()); + + // Compute entitlement changes for the identified leaf queues + // which is appended to the List of computedEntitlements + updateLeafQueueCapacitiesByLabel(nodeLabel, leafQueuesToBeActivated, leafQueueEntitlements); + + if (LOG.isDebugEnabled() && leafQueuesToBeActivated.size() > 0) { + LOG.debug("Activated leaf queues : [{}]", + getListContentsUpToLimit(leafQueuesToBeActivated)); + } + } + } + + private Object getListContentsUpToLimit(Set leafQueuesToBeActivated) { + return leafQueuesToBeActivated.size() < DEFAULT_QUEUE_PRINT_SIZE_LIMIT ? + leafQueuesToBeActivated : leafQueuesToBeActivated.size(); + } + + private Object getMapUpToLimit(Map deactivatedLeafQueues) { + return deactivatedLeafQueues.size() > DEFAULT_QUEUE_PRINT_SIZE_LIMIT ? + deactivatedLeafQueues.size() : deactivatedLeafQueues; + } + + private DeactivatedLeafQueuesByLabel deactivateLeafQueues(String nodeLabel, + LeafQueueEntitlements leafQueueEntitlements) throws SchedulerDynamicEditException { + // check if any leaf queues need to be deactivated based on pending applications + float parentAbsoluteCapacity = + managedParentQueue.getQueueCapacities().getAbsoluteCapacity(nodeLabel); + float leafQueueTemplateAbsoluteCapacity = + leafQueueTemplateCapacities.getAbsoluteCapacity(nodeLabel); + Map deactivatedLeafQueues = + deactivateLeafQueuesIfInActive(managedParentQueue, nodeLabel, leafQueueEntitlements); + + if (LOG.isDebugEnabled() && deactivatedLeafQueues.size() > 0) { + LOG.debug("Parent queue = {}, nodeLabel = {}, deactivated leaf queues = [{}] ", + managedParentQueue.getQueuePath(), nodeLabel, + getMapUpToLimit(deactivatedLeafQueues)); + } + + return new DeactivatedLeafQueuesByLabel(deactivatedLeafQueues, + managedParentQueue.getQueuePath(), + nodeLabel, + parentQueueState.getAbsoluteActivatedChildQueueCapacity(nodeLabel), + parentAbsoluteCapacity, + leafQueueTemplateAbsoluteCapacity); + } + private void updateTemplateAbsoluteCapacities(QueueCapacities parentQueueCapacities, GuaranteedOrZeroCapacityOverTimePolicy policy) { writeLock.lock(); @@ -496,19 +474,6 @@ public void updateTemplateAbsoluteCapacities(QueueCapacities queueCapacities) { updateTemplateAbsoluteCapacities(queueCapacities, this); } - private float getTotalDeactivatedCapacity( - Map deactivatedLeafQueues, String nodeLabel) { - float deactivatedCapacity = 0; - for (Iterator> iterator = - deactivatedLeafQueues.entrySet().iterator(); iterator.hasNext(); ) { - Map.Entry deactivatedQueueCapacity = - iterator.next(); - deactivatedCapacity += - deactivatedQueueCapacity.getValue().getAbsoluteCapacity(nodeLabel); - } - return deactivatedCapacity; - } - @VisibleForTesting void updateLeafQueueState() { writeLock.lock(); @@ -608,9 +573,15 @@ public boolean isActive(final AutoCreatedLeafQueue leafQueue, } } + /** + * Map of LeafQueue -> QueueCapacities - keep adding the computed + * entitlements to this map and finally + * build the leaf queue configuration Template for all identified leaf + * queues + */ private Map deactivateLeafQueuesIfInActive( ParentQueue parentQueue, String nodeLabel, - Map leafQueueEntitlements) + LeafQueueEntitlements leafQueueEntitlements) throws SchedulerDynamicEditException { Map deactivatedQueues = new HashMap<>(); @@ -618,18 +589,11 @@ private Map deactivateLeafQueuesIfInActive( AutoCreatedLeafQueue leafQueue = (AutoCreatedLeafQueue) childQueue; if (leafQueue != null) { if (isActive(leafQueue, nodeLabel) && !hasPendingApps(leafQueue)) { - if (!leafQueueEntitlements.containsKey(leafQueue.getQueuePath())) { - leafQueueEntitlements.put(leafQueue.getQueuePath(), - new QueueCapacities(false)); - } - - QueueCapacities capacities = leafQueueEntitlements.get( - leafQueue.getQueuePath()); + QueueCapacities capacities = leafQueueEntitlements.getCapacityOfQueue(leafQueue); updateToZeroCapacity(capacities, nodeLabel, (LeafQueue)childQueue); - deactivatedQueues.put(leafQueue.getQueuePath(), - leafQueueTemplateCapacities); + deactivatedQueues.put(leafQueue.getQueuePath(), leafQueueTemplateCapacities); } - } else{ + } else { LOG.warn("Could not find queue in scheduler while trying" + " to " + "deactivate for " + parentQueue); } @@ -640,35 +604,16 @@ private Map deactivateLeafQueuesIfInActive( private void updateLeafQueueCapacitiesByLabel(String nodeLabel, Set leafQueuesToBeActivated, - Map leafQueueEntitlements) { - for (String curLeafQueue : leafQueuesToBeActivated) { - if (!leafQueueEntitlements.containsKey(curLeafQueue)) { - leafQueueEntitlements.put(curLeafQueue, new QueueCapacities(false)); - // Activate queues if capacity is available - } - - QueueCapacities capacities = leafQueueEntitlements.get(curLeafQueue); + LeafQueueEntitlements leafQueueEntitlements) { + for (String leafQueue : leafQueuesToBeActivated) { + QueueCapacities capacities = leafQueueEntitlements.getCapacityOfQueueByPath(leafQueue); updateCapacityFromTemplate(capacities, nodeLabel); } } - @VisibleForTesting - public int getMaxLeavesToBeActivated(float availableCapacity, - float childQueueAbsoluteCapacity, int numPendingApps) - throws SchedulerDynamicEditException { - - if (childQueueAbsoluteCapacity > 0) { - int numLeafQueuesNeeded = (int) Math.floor( - availableCapacity / childQueueAbsoluteCapacity); - - return Math.min(numLeafQueuesNeeded, numPendingApps); - } - return 0; - } - /** * Commit queue management changes - which involves updating required state - * on parent/underlying leaf queues + * on parent/underlying leaf queues. * * @param queueManagementChanges Queue Management changes to commit * @throws SchedulerDynamicEditException when validation fails diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/queuemanagement/LeafQueueEntitlements.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/queuemanagement/LeafQueueEntitlements.java new file mode 100644 index 0000000000000..c558716ca48e6 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/queuemanagement/LeafQueueEntitlements.java @@ -0,0 +1,52 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.queuemanagement; + +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.AutoCreatedLeafQueue; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.QueueCapacities; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.QueueManagementChange; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.function.BiFunction; +import java.util.stream.Collectors; + +public class LeafQueueEntitlements { + private final Map entitlements = new HashMap<>(); + + public QueueCapacities getCapacityOfQueue(AutoCreatedLeafQueue leafQueue) { + return getCapacityOfQueueByPath(leafQueue.getQueuePath()); + } + + public QueueCapacities getCapacityOfQueueByPath(String leafQueuePath) { + if (!entitlements.containsKey(leafQueuePath)) { + entitlements.put(leafQueuePath, new QueueCapacities(false)); + } + return entitlements.get(leafQueuePath); + } + + public Map getEntitlements() { + return entitlements; + } + + public List mapToQueueManagementChanges( + BiFunction func) { + return entitlements.entrySet().stream().map(e -> func.apply(e.getKey(), e.getValue())) + .collect(Collectors.toList()); + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestGuaranteedOrZeroCapacityOverTimePolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/queuemanagement/package-info.java similarity index 52% rename from hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestGuaranteedOrZeroCapacityOverTimePolicy.java rename to hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/queuemanagement/package-info.java index c71d2bf91343a..98bed9515506d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestGuaranteedOrZeroCapacityOverTimePolicy.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/queuemanagement/package-info.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -15,26 +15,9 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity; - -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerDynamicEditException; - -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity - .queuemanagement.GuaranteedOrZeroCapacityOverTimePolicy; -import org.junit.Test; - -import static org.junit.Assert.assertEquals; -public class TestGuaranteedOrZeroCapacityOverTimePolicy { - - @Test - public void testGetMaxLeavesToBeActivated() - throws SchedulerDynamicEditException { - GuaranteedOrZeroCapacityOverTimePolicy policy = - new GuaranteedOrZeroCapacityOverTimePolicy(); - - assertEquals(1, policy.getMaxLeavesToBeActivated(0.17f, 0.03f, 1)); - assertEquals(5, policy.getMaxLeavesToBeActivated(0.17f, 0.03f, 7)); - assertEquals(0, policy.getMaxLeavesToBeActivated(0, 0.03f, 10)); - } -} +/** + * Contains classes that are related to the newer + * version of CS placement engine. + */ +package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.queuemanagement; \ No newline at end of file diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/proto/yarn_server_resourcemanager_recovery.proto b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/proto/yarn_server_resourcemanager_recovery.proto index 61f4a8a57a6a1..2d1cf1c5a53ec 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/proto/yarn_server_resourcemanager_recovery.proto +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/proto/yarn_server_resourcemanager_recovery.proto @@ -72,6 +72,7 @@ message ApplicationStateDataProto { optional hadoop.common.RPCCallerContextProto caller_context = 8; repeated ApplicationTimeoutMapProto application_timeouts = 9; optional int64 launch_time = 10; + optional string real_user = 11; } message ApplicationAttemptStateDataProto { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/AppManagerTestBase.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/AppManagerTestBase.java index 33fe02824513a..2e43aa44c5508 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/AppManagerTestBase.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/AppManagerTestBase.java @@ -19,6 +19,7 @@ package org.apache.hadoop.yarn.server.resourcemanager; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext; import org.apache.hadoop.yarn.exceptions.YarnException; @@ -71,7 +72,7 @@ public void submitApplication( ApplicationSubmissionContext submissionContext, String user) throws YarnException { super.submitApplication(submissionContext, System.currentTimeMillis(), - user); + UserGroupInformation.createRemoteUser(user)); } public String getUserNameForPlacement(final String user, diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/RMHATestBase.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/RMHATestBase.java index 439a4490947d7..ca76c24b87818 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/RMHATestBase.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/RMHATestBase.java @@ -26,6 +26,7 @@ import org.apache.hadoop.ha.HAServiceProtocol; import org.apache.hadoop.ha.HAServiceProtocol.HAServiceState; import org.apache.hadoop.ha.HAServiceProtocol.StateChangeRequestInfo; +import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext; import org.apache.hadoop.yarn.conf.HAUtil; import org.apache.hadoop.yarn.conf.YarnConfiguration; @@ -169,7 +170,8 @@ public MyRMAppManager(RMContext context, YarnScheduler scheduler, @Override protected void submitApplication( ApplicationSubmissionContext submissionContext, long submitTime, - String user) throws YarnException { + UserGroupInformation userUgi) throws YarnException { + String user = userUgi.getShortUserName(); //Do nothing, just add the application to RMContext RMAppImpl application = new RMAppImpl(submissionContext.getApplicationId(), this.rmContext, diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/MockAsm.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/MockAsm.java index 4640952016d7f..e6c159d67d2c3 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/MockAsm.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/MockAsm.java @@ -67,6 +67,10 @@ public String getUser() { throw new UnsupportedOperationException("Not supported yet."); } + @Override + public String getRealUser() { + throw new UnsupportedOperationException("Not supported yet."); + } @Override public ApplicationSubmissionContext getApplicationSubmissionContext() { throw new UnsupportedOperationException("Not supported yet."); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/placement/csmappingrule/TestCSMappingPlacementRule.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/placement/csmappingrule/TestCSMappingPlacementRule.java index 0cf10595ee9dd..3e614bcbc967a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/placement/csmappingrule/TestCSMappingPlacementRule.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/placement/csmappingrule/TestCSMappingPlacementRule.java @@ -18,6 +18,8 @@ package org.apache.hadoop.yarn.server.resourcemanager.placement.csmappingrule; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.security.GroupMappingServiceProvider; import org.apache.hadoop.thirdparty.com.google.common.collect.ImmutableMap; import org.apache.hadoop.thirdparty.com.google.common.collect.ImmutableSet; import org.apache.hadoop.security.Groups; @@ -51,6 +53,7 @@ import static junit.framework.TestCase.assertNull; import static junit.framework.TestCase.assertTrue; import static junit.framework.TestCase.fail; +import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_SECURITY_GROUP_MAPPING; import static org.mockito.ArgumentMatchers.isNull; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; @@ -771,4 +774,138 @@ public void testSecondaryGroupNameCleanup() throws IOException { "Application should have been placed to root.groups.sec_dot_test_dot_grp", engine, app, "test.user", "root.groups.sec_dot_test_dot_grp"); } + + /** + * 1. Invoke Groups.reset(). This make sure that the underlying singleton {@link Groups#GROUPS} + * is set to null.
+ * 2. Create a Configuration object in which the property "hadoop.security.group.mapping" + * refers to an existing a test implementation.
+ * 3. Create a mock CapacityScheduler where getConf() and getConfiguration() contain different + * settings for "hadoop.security.group.mapping". Since getConf() is the service config, this + * should return the config object created in step #2.
+ * 4. Create an instance of CSMappingPlacementRule with a single primary group rule.
+ * 5. Run the placement evaluation.
+ * 6. Expected: The returned queue matches what is supposed to be coming from the test group + * mapping service ("testuser" --> "testGroup1").
+ * 7. Modify "hadoop.security.group.mapping" in the config object created in step #2. + * This step is required to guarantee that the CSMappingPlacementRule doesn't try to recreate + * the group mapping implementation and uses the one that was previously created.
+ * 8. Call Groups.refresh() which changes the group mapping ("testuser" --> "testGroup0"). This + * requires that the test group mapping service implement GroupMappingServiceProvider + * .cacheGroupsRefresh().
+ * 9. Create a new instance of CSMappingPlacementRule. This is important as we want to test + * that even this new {@link CSMappingPlacementRule} instance uses the same group mapping + * instance.
+ * 10. Run the placement evaluation again
+ * 11. Expected: with the same user, the target queue has changed to 'testGroup0'.
+ *

+ * These all looks convoluted, but the steps above make sure all the following conditions are met: + *

+ * 1. CSMappingPlacementRule will force the initialization of groups.
+ * 2. We select the correct configuration for group service init.
+ * 3. We don't create a new Groups instance if the singleton is initialized, so we cover the + * original problem described in YARN-10597.
+ */ + @Test + public void testPlacementEngineSelectsCorrectConfigurationForGroupMapping() throws IOException { + Groups.reset(); + final String user = "testuser"; + + //Create service-wide configuration object + Configuration yarnConf = new Configuration(); + yarnConf.setClass(HADOOP_SECURITY_GROUP_MAPPING, MockUnixGroupsMapping.class, + GroupMappingServiceProvider.class); + + //Create CS configuration object with a single, primary group mapping rule + List mappingRules = new ArrayList<>(); + mappingRules.add( + new MappingRule( + MappingRuleMatchers.createUserMatcher(user), + (new MappingRuleActions.PlaceToQueueAction( + "root.man.%primary_group", true)) + .setFallbackReject())); + CapacitySchedulerConfiguration csConf = new CapacitySchedulerConfiguration() { + @Override + public List getMappingRules() { + return mappingRules; + } + }; + csConf.setOverrideWithQueueMappings(true); + //Intentionally add a dummy implementation class - + // The "HADOOP_SECURITY_GROUP_MAPPING" should not be read from the + // CapacitySchedulerConfiguration instance! + csConf.setClass(HADOOP_SECURITY_GROUP_MAPPING, String.class, Object.class); + + CapacityScheduler cs = createMockCS(yarnConf, csConf); + + //Create app, submit to placement engine, expecting queue=testGroup1 + CSMappingPlacementRule engine = initPlacementEngine(cs); + ApplicationSubmissionContext app = createApp("app"); + assertPlace(engine, app, user, "root.man.testGroup1"); + + //Intentionally add a dummy implementation class! + // The "HADOOP_SECURITY_GROUP_MAPPING" should not be read from the + // CapacitySchedulerConfiguration instance! + //This makes sure that the Groups instance is not recreated by CSMappingPlacementRule + yarnConf.setClass(HADOOP_SECURITY_GROUP_MAPPING, String.class, Object.class); + + //Refresh the groups, this makes testGroup0 as primary group for "testUser" + engine.getGroups().refresh(); + + //Create app, submit to placement engine, expecting queue=testGroup0 (the new primary group) + engine = initPlacementEngine(cs); + assertPlace(engine, app, user, "root.man.testGroup0"); + } + + private CSMappingPlacementRule initPlacementEngine(CapacityScheduler cs) throws IOException { + CSMappingPlacementRule engine = new CSMappingPlacementRule(); + engine.setFailOnConfigError(true); + engine.initialize(cs); + return engine; + } + + private CapacityScheduler createMockCS(Configuration conf, + CapacitySchedulerConfiguration csConf) { + CapacitySchedulerQueueManager qm = + mock(CapacitySchedulerQueueManager.class); + createQueueHierarchy(qm); + + CapacityScheduler cs = mock(CapacityScheduler.class); + when(cs.getConfiguration()).thenReturn(csConf); + when(cs.getConf()).thenReturn(conf); + when(cs.getCapacitySchedulerQueueManager()).thenReturn(qm); + return cs; + } + + public static class MockUnixGroupsMapping implements GroupMappingServiceProvider { + + public MockUnixGroupsMapping() { + GROUP.clear(); + GROUP.add("testGroup1"); + GROUP.add("testGroup2"); + GROUP.add("testGroup3"); + } + + private static final List GROUP = new ArrayList<>(); + + @Override + public List getGroups(String user) throws IOException { + return GROUP; + } + + @Override + public void cacheGroupsRefresh() { + GROUP.add(0, "testGroup0"); + } + + @Override + public void cacheGroupsAdd(List groups) { + // Do nothing + } + + @Override + public Set getGroupsSet(String user) { + return ImmutableSet.copyOf(GROUP); + } + } } \ No newline at end of file diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/MockRMApp.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/MockRMApp.java index ad29d274a4f5b..96cff0f02894d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/MockRMApp.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/MockRMApp.java @@ -119,6 +119,11 @@ public String getUser() { return user; } + @Override + public String getRealUser() { + return null; + } + public void setUser(String user) { this.user = user; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestAbsoluteResourceConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestAbsoluteResourceConfiguration.java index 97f9a652c1be7..7114728fc6b7a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestAbsoluteResourceConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestAbsoluteResourceConfiguration.java @@ -222,13 +222,13 @@ public void testSimpleMinMaxResourceConfigurartionPerQueue() new ResourceLimits(cs.getClusterResource())); Assert.assertEquals(QUEUE_D_TEMPL_MINRES, - d1.queueResourceQuotas.getConfiguredMinResource()); + d1.usageTracker.getQueueResourceQuotas().getConfiguredMinResource()); Assert.assertEquals(QUEUE_D_TEMPL_MINRES, - d1.queueResourceQuotas.getEffectiveMinResource()); + d1.usageTracker.getQueueResourceQuotas().getEffectiveMinResource()); Assert.assertEquals(QUEUE_D_TEMPL_MAXRES, - d1.queueResourceQuotas.getConfiguredMaxResource()); + d1.usageTracker.getQueueResourceQuotas().getConfiguredMaxResource()); Assert.assertEquals(QUEUE_D_TEMPL_MAXRES, - d1.queueResourceQuotas.getEffectiveMaxResource()); + d1.usageTracker.getQueueResourceQuotas().getEffectiveMaxResource()); /** * After adding child queue d2, d1 + d2 > resource @@ -243,22 +243,22 @@ public void testSimpleMinMaxResourceConfigurartionPerQueue() new ResourceLimits(cs.getClusterResource())); Assert.assertEquals(Resource.newInstance(0, 0), - d2.queueResourceQuotas.getConfiguredMinResource()); + d2.usageTracker.getQueueResourceQuotas().getConfiguredMinResource()); Assert.assertEquals(Resource.newInstance(0, 0), - d2.queueResourceQuotas.getEffectiveMinResource()); + d2.usageTracker.getQueueResourceQuotas().getEffectiveMinResource()); Assert.assertEquals(QUEUE_D_TEMPL_MAXRES, - d2.queueResourceQuotas.getConfiguredMaxResource()); + d2.usageTracker.getQueueResourceQuotas().getConfiguredMaxResource()); Assert.assertEquals(QUEUE_D_TEMPL_MAXRES, - d2.queueResourceQuotas.getEffectiveMaxResource()); + d2.usageTracker.getQueueResourceQuotas().getEffectiveMaxResource()); Assert.assertEquals(QUEUE_D_TEMPL_MINRES, - d1.queueResourceQuotas.getConfiguredMinResource()); + d1.usageTracker.getQueueResourceQuotas().getConfiguredMinResource()); Assert.assertEquals(QUEUE_D_TEMPL_MINRES, - d1.queueResourceQuotas.getEffectiveMinResource()); + d1.usageTracker.getQueueResourceQuotas().getEffectiveMinResource()); Assert.assertEquals(QUEUE_D_TEMPL_MAXRES, - d1.queueResourceQuotas.getConfiguredMaxResource()); + d1.usageTracker.getQueueResourceQuotas().getConfiguredMaxResource()); Assert.assertEquals(QUEUE_D_TEMPL_MAXRES, - d1.queueResourceQuotas.getEffectiveMaxResource()); + d1.usageTracker.getQueueResourceQuotas().getEffectiveMaxResource()); rm.close(); } @@ -287,35 +287,35 @@ public void testEffectiveMinMaxResourceConfigurartionPerQueue() LeafQueue qA = (LeafQueue) cs.getQueue(QUEUEA); Assert.assertNotNull(qA); Assert.assertEquals("Min resource configured for QUEUEA is not correct", - QUEUE_A_MINRES, qA.queueResourceQuotas.getConfiguredMinResource()); + QUEUE_A_MINRES, qA.usageTracker.getQueueResourceQuotas().getConfiguredMinResource()); Assert.assertEquals("Max resource configured for QUEUEA is not correct", - QUEUE_A_MAXRES, qA.queueResourceQuotas.getConfiguredMaxResource()); + QUEUE_A_MAXRES, qA.usageTracker.getQueueResourceQuotas().getConfiguredMaxResource()); Assert.assertEquals("Effective Min resource for QUEUEA is not correct", - QUEUE_A_MINRES, qA.queueResourceQuotas.getEffectiveMinResource()); + QUEUE_A_MINRES, qA.usageTracker.getQueueResourceQuotas().getEffectiveMinResource()); Assert.assertEquals("Effective Max resource for QUEUEA is not correct", - QUEUE_A_MAXRES, qA.queueResourceQuotas.getEffectiveMaxResource()); + QUEUE_A_MAXRES, qA.usageTracker.getQueueResourceQuotas().getEffectiveMaxResource()); LeafQueue qB = (LeafQueue) cs.getQueue(QUEUEB); Assert.assertNotNull(qB); Assert.assertEquals("Min resource configured for QUEUEB is not correct", - QUEUE_B_MINRES, qB.queueResourceQuotas.getConfiguredMinResource()); + QUEUE_B_MINRES, qB.usageTracker.getQueueResourceQuotas().getConfiguredMinResource()); Assert.assertEquals("Max resource configured for QUEUEB is not correct", - QUEUE_B_MAXRES, qB.queueResourceQuotas.getConfiguredMaxResource()); + QUEUE_B_MAXRES, qB.usageTracker.getQueueResourceQuotas().getConfiguredMaxResource()); Assert.assertEquals("Effective Min resource for QUEUEB is not correct", - QUEUE_B_MINRES, qB.queueResourceQuotas.getEffectiveMinResource()); + QUEUE_B_MINRES, qB.usageTracker.getQueueResourceQuotas().getEffectiveMinResource()); Assert.assertEquals("Effective Max resource for QUEUEB is not correct", - QUEUE_B_MAXRES, qB.queueResourceQuotas.getEffectiveMaxResource()); + QUEUE_B_MAXRES, qB.usageTracker.getQueueResourceQuotas().getEffectiveMaxResource()); LeafQueue qC = (LeafQueue) cs.getQueue(QUEUEC); Assert.assertNotNull(qC); Assert.assertEquals("Min resource configured for QUEUEC is not correct", - QUEUE_C_MINRES, qC.queueResourceQuotas.getConfiguredMinResource()); + QUEUE_C_MINRES, qC.usageTracker.getQueueResourceQuotas().getConfiguredMinResource()); Assert.assertEquals("Max resource configured for QUEUEC is not correct", - QUEUE_C_MAXRES, qC.queueResourceQuotas.getConfiguredMaxResource()); + QUEUE_C_MAXRES, qC.usageTracker.getQueueResourceQuotas().getConfiguredMaxResource()); Assert.assertEquals("Effective Min resource for QUEUEC is not correct", - QUEUE_C_MINRES, qC.queueResourceQuotas.getEffectiveMinResource()); + QUEUE_C_MINRES, qC.usageTracker.getQueueResourceQuotas().getEffectiveMinResource()); Assert.assertEquals("Effective Max resource for QUEUEC is not correct", - QUEUE_C_MAXRES, qC.queueResourceQuotas.getEffectiveMaxResource()); + QUEUE_C_MAXRES, qC.usageTracker.getQueueResourceQuotas().getEffectiveMaxResource()); rm.stop(); } @@ -378,37 +378,37 @@ public void testSimpleValidateAbsoluteResourceConfig() throws Exception { LeafQueue qA1 = (LeafQueue) cs.getQueue(QUEUEA1); Assert.assertEquals("Effective Min resource for QUEUEA1 is not correct", - QUEUE_A1_MINRES, qA1.queueResourceQuotas.getEffectiveMinResource()); + QUEUE_A1_MINRES, qA1.usageTracker.getQueueResourceQuotas().getEffectiveMinResource()); Assert.assertEquals("Effective Max resource for QUEUEA1 is not correct", - QUEUE_A_MAXRES, qA1.queueResourceQuotas.getEffectiveMaxResource()); + QUEUE_A_MAXRES, qA1.usageTracker.getQueueResourceQuotas().getEffectiveMaxResource()); LeafQueue qA2 = (LeafQueue) cs.getQueue(QUEUEA2); Assert.assertEquals("Effective Min resource for QUEUEA2 is not correct", - QUEUE_A2_MINRES, qA2.queueResourceQuotas.getEffectiveMinResource()); + QUEUE_A2_MINRES, qA2.usageTracker.getQueueResourceQuotas().getEffectiveMinResource()); Assert.assertEquals("Effective Max resource for QUEUEA2 is not correct", - QUEUE_A_MAXRES, qA2.queueResourceQuotas.getEffectiveMaxResource()); + QUEUE_A_MAXRES, qA2.usageTracker.getQueueResourceQuotas().getEffectiveMaxResource()); LeafQueue qB1 = (LeafQueue) cs.getQueue(QUEUEB1); Assert.assertNotNull(qB1); Assert.assertEquals("Min resource configured for QUEUEB1 is not correct", - QUEUE_B1_MINRES, qB1.queueResourceQuotas.getConfiguredMinResource()); + QUEUE_B1_MINRES, qB1.usageTracker.getQueueResourceQuotas().getConfiguredMinResource()); Assert.assertEquals("Max resource configured for QUEUEB1 is not correct", - QUEUE_B_MAXRES, qB1.queueResourceQuotas.getConfiguredMaxResource()); + QUEUE_B_MAXRES, qB1.usageTracker.getQueueResourceQuotas().getConfiguredMaxResource()); Assert.assertEquals("Effective Min resource for QUEUEB1 is not correct", - QUEUE_B1_MINRES, qB1.queueResourceQuotas.getEffectiveMinResource()); + QUEUE_B1_MINRES, qB1.usageTracker.getQueueResourceQuotas().getEffectiveMinResource()); Assert.assertEquals("Effective Max resource for QUEUEB1 is not correct", - QUEUE_B_MAXRES, qB1.queueResourceQuotas.getEffectiveMaxResource()); + QUEUE_B_MAXRES, qB1.usageTracker.getQueueResourceQuotas().getEffectiveMaxResource()); LeafQueue qC = (LeafQueue) cs.getQueue(QUEUEC); Assert.assertNotNull(qC); Assert.assertEquals("Min resource configured for QUEUEC is not correct", - QUEUE_C_MINRES, qC.queueResourceQuotas.getConfiguredMinResource()); + QUEUE_C_MINRES, qC.usageTracker.getQueueResourceQuotas().getConfiguredMinResource()); Assert.assertEquals("Max resource configured for QUEUEC is not correct", - QUEUE_C_MAXRES, qC.queueResourceQuotas.getConfiguredMaxResource()); + QUEUE_C_MAXRES, qC.usageTracker.getQueueResourceQuotas().getConfiguredMaxResource()); Assert.assertEquals("Effective Min resource for QUEUEC is not correct", - QUEUE_C_MINRES, qC.queueResourceQuotas.getEffectiveMinResource()); + QUEUE_C_MINRES, qC.usageTracker.getQueueResourceQuotas().getEffectiveMinResource()); Assert.assertEquals("Effective Max resource for QUEUEC is not correct", - QUEUE_C_MAXRES, qC.queueResourceQuotas.getEffectiveMaxResource()); + QUEUE_C_MAXRES, qC.usageTracker.getQueueResourceQuotas().getEffectiveMaxResource()); // 3. Create a new config and make sure one queue's min resource is more // than its max resource configured. @@ -597,35 +597,35 @@ public void testEffectiveResourceAfterReducingClusterResource() LeafQueue qA = (LeafQueue) cs.getQueue(QUEUEA); Assert.assertNotNull(qA); Assert.assertEquals("Min resource configured for QUEUEA is not correct", - QUEUE_A_MINRES, qA.queueResourceQuotas.getConfiguredMinResource()); + QUEUE_A_MINRES, qA.usageTracker.getQueueResourceQuotas().getConfiguredMinResource()); Assert.assertEquals("Max resource configured for QUEUEA is not correct", - QUEUE_A_MAXRES, qA.queueResourceQuotas.getConfiguredMaxResource()); + QUEUE_A_MAXRES, qA.usageTracker.getQueueResourceQuotas().getConfiguredMaxResource()); Assert.assertEquals("Effective Min resource for QUEUEA is not correct", - QUEUE_A_MINRES, qA.queueResourceQuotas.getEffectiveMinResource()); + QUEUE_A_MINRES, qA.usageTracker.getQueueResourceQuotas().getEffectiveMinResource()); Assert.assertEquals("Effective Max resource for QUEUEA is not correct", - QUEUE_A_MAXRES, qA.queueResourceQuotas.getEffectiveMaxResource()); + QUEUE_A_MAXRES, qA.usageTracker.getQueueResourceQuotas().getEffectiveMaxResource()); LeafQueue qB = (LeafQueue) cs.getQueue(QUEUEB); Assert.assertNotNull(qB); Assert.assertEquals("Min resource configured for QUEUEB is not correct", - QUEUE_B_MINRES, qB.queueResourceQuotas.getConfiguredMinResource()); + QUEUE_B_MINRES, qB.usageTracker.getQueueResourceQuotas().getConfiguredMinResource()); Assert.assertEquals("Max resource configured for QUEUEB is not correct", - QUEUE_B_MAXRES, qB.queueResourceQuotas.getConfiguredMaxResource()); + QUEUE_B_MAXRES, qB.usageTracker.getQueueResourceQuotas().getConfiguredMaxResource()); Assert.assertEquals("Effective Min resource for QUEUEB is not correct", - QUEUE_B_MINRES, qB.queueResourceQuotas.getEffectiveMinResource()); + QUEUE_B_MINRES, qB.usageTracker.getQueueResourceQuotas().getEffectiveMinResource()); Assert.assertEquals("Effective Max resource for QUEUEB is not correct", - QUEUE_B_MAXRES, qB.queueResourceQuotas.getEffectiveMaxResource()); + QUEUE_B_MAXRES, qB.usageTracker.getQueueResourceQuotas().getEffectiveMaxResource()); LeafQueue qC = (LeafQueue) cs.getQueue(QUEUEC); Assert.assertNotNull(qC); Assert.assertEquals("Min resource configured for QUEUEC is not correct", - QUEUE_C_MINRES, qC.queueResourceQuotas.getConfiguredMinResource()); + QUEUE_C_MINRES, qC.usageTracker.getQueueResourceQuotas().getConfiguredMinResource()); Assert.assertEquals("Max resource configured for QUEUEC is not correct", - QUEUE_C_MAXRES, qC.queueResourceQuotas.getConfiguredMaxResource()); + QUEUE_C_MAXRES, qC.usageTracker.getQueueResourceQuotas().getConfiguredMaxResource()); Assert.assertEquals("Effective Min resource for QUEUEC is not correct", - QUEUE_C_MINRES, qC.queueResourceQuotas.getEffectiveMinResource()); + QUEUE_C_MINRES, qC.usageTracker.getQueueResourceQuotas().getEffectiveMinResource()); Assert.assertEquals("Effective Max resource for QUEUEC is not correct", - QUEUE_C_MAXRES, qC.queueResourceQuotas.getEffectiveMaxResource()); + QUEUE_C_MAXRES, qC.usageTracker.getQueueResourceQuotas().getEffectiveMaxResource()); // unregister one NM. rm.unRegisterNode(nm1); @@ -634,19 +634,19 @@ public void testEffectiveResourceAfterReducingClusterResource() // above half. Hence A's min will be 60Gi and 6 cores and max will be // 128GB and 20 cores. Assert.assertEquals("Effective Min resource for QUEUEA is not correct", - QUEUEA_REDUCED, qA.queueResourceQuotas.getEffectiveMinResource()); + QUEUEA_REDUCED, qA.usageTracker.getQueueResourceQuotas().getEffectiveMinResource()); Assert.assertEquals("Effective Max resource for QUEUEA is not correct", - QUEUEMAX_REDUCED, qA.queueResourceQuotas.getEffectiveMaxResource()); + QUEUEMAX_REDUCED, qA.usageTracker.getQueueResourceQuotas().getEffectiveMaxResource()); Assert.assertEquals("Effective Min resource for QUEUEB is not correct", - QUEUEB_REDUCED, qB.queueResourceQuotas.getEffectiveMinResource()); + QUEUEB_REDUCED, qB.usageTracker.getQueueResourceQuotas().getEffectiveMinResource()); Assert.assertEquals("Effective Max resource for QUEUEB is not correct", - QUEUEMAX_REDUCED, qB.queueResourceQuotas.getEffectiveMaxResource()); + QUEUEMAX_REDUCED, qB.usageTracker.getQueueResourceQuotas().getEffectiveMaxResource()); Assert.assertEquals("Effective Min resource for QUEUEC is not correct", - QUEUEC_REDUCED, qC.queueResourceQuotas.getEffectiveMinResource()); + QUEUEC_REDUCED, qC.usageTracker.getQueueResourceQuotas().getEffectiveMinResource()); Assert.assertEquals("Effective Max resource for QUEUEC is not correct", - QUEUEMAX_REDUCED, qC.queueResourceQuotas.getEffectiveMaxResource()); + QUEUEMAX_REDUCED, qC.usageTracker.getQueueResourceQuotas().getEffectiveMaxResource()); rm.stop(); } @@ -676,13 +676,13 @@ public void testEffectiveResourceAfterIncreasingClusterResource() ParentQueue qA = (ParentQueue) cs.getQueue(QUEUEA); Assert.assertNotNull(qA); Assert.assertEquals("Min resource configured for QUEUEA is not correct", - QUEUE_A_MINRES, qA.queueResourceQuotas.getConfiguredMinResource()); + QUEUE_A_MINRES, qA.usageTracker.getQueueResourceQuotas().getConfiguredMinResource()); Assert.assertEquals("Max resource configured for QUEUEA is not correct", - QUEUE_A_MAXRES, qA.queueResourceQuotas.getConfiguredMaxResource()); + QUEUE_A_MAXRES, qA.usageTracker.getQueueResourceQuotas().getConfiguredMaxResource()); Assert.assertEquals("Effective Min resource for QUEUEA is not correct", - QUEUE_A_MINRES, qA.queueResourceQuotas.getEffectiveMinResource()); + QUEUE_A_MINRES, qA.usageTracker.getQueueResourceQuotas().getEffectiveMinResource()); Assert.assertEquals("Effective Max resource for QUEUEA is not correct", - QUEUE_A_MAXRES, qA.queueResourceQuotas.getEffectiveMaxResource()); + QUEUE_A_MAXRES, qA.usageTracker.getQueueResourceQuotas().getEffectiveMaxResource()); Assert.assertEquals("Absolute capacity for QUEUEA is not correct", 0.4, qA.getAbsoluteCapacity(), DELTA); Assert.assertEquals("Absolute Max capacity for QUEUEA is not correct", @@ -691,13 +691,13 @@ public void testEffectiveResourceAfterIncreasingClusterResource() ParentQueue qB = (ParentQueue) cs.getQueue(QUEUEB); Assert.assertNotNull(qB); Assert.assertEquals("Min resource configured for QUEUEB is not correct", - QUEUE_B_MINRES, qB.queueResourceQuotas.getConfiguredMinResource()); + QUEUE_B_MINRES, qB.usageTracker.getQueueResourceQuotas().getConfiguredMinResource()); Assert.assertEquals("Max resource configured for QUEUEB is not correct", - QUEUE_B_MAXRES, qB.queueResourceQuotas.getConfiguredMaxResource()); + QUEUE_B_MAXRES, qB.usageTracker.getQueueResourceQuotas().getConfiguredMaxResource()); Assert.assertEquals("Effective Min resource for QUEUEB is not correct", - QUEUE_B_MINRES, qB.queueResourceQuotas.getEffectiveMinResource()); + QUEUE_B_MINRES, qB.usageTracker.getQueueResourceQuotas().getEffectiveMinResource()); Assert.assertEquals("Effective Max resource for QUEUEB is not correct", - QUEUE_B_MAXRES, qB.queueResourceQuotas.getEffectiveMaxResource()); + QUEUE_B_MAXRES, qB.usageTracker.getQueueResourceQuotas().getEffectiveMaxResource()); Assert.assertEquals("Absolute capacity for QUEUEB is not correct", 0.2, qB.getAbsoluteCapacity(), DELTA); Assert.assertEquals("Absolute Max capacity for QUEUEB is not correct", @@ -706,13 +706,13 @@ public void testEffectiveResourceAfterIncreasingClusterResource() LeafQueue qC = (LeafQueue) cs.getQueue(QUEUEC); Assert.assertNotNull(qC); Assert.assertEquals("Min resource configured for QUEUEC is not correct", - QUEUE_C_MINRES, qC.queueResourceQuotas.getConfiguredMinResource()); + QUEUE_C_MINRES, qC.usageTracker.getQueueResourceQuotas().getConfiguredMinResource()); Assert.assertEquals("Max resource configured for QUEUEC is not correct", - QUEUE_C_MAXRES, qC.queueResourceQuotas.getConfiguredMaxResource()); + QUEUE_C_MAXRES, qC.usageTracker.getQueueResourceQuotas().getConfiguredMaxResource()); Assert.assertEquals("Effective Min resource for QUEUEC is not correct", - QUEUE_C_MINRES, qC.queueResourceQuotas.getEffectiveMinResource()); + QUEUE_C_MINRES, qC.usageTracker.getQueueResourceQuotas().getEffectiveMinResource()); Assert.assertEquals("Effective Max resource for QUEUEC is not correct", - QUEUE_C_MAXRES, qC.queueResourceQuotas.getEffectiveMaxResource()); + QUEUE_C_MAXRES, qC.usageTracker.getQueueResourceQuotas().getEffectiveMaxResource()); Assert.assertEquals("Absolute capacity for QUEUEC is not correct", 0.1, qC.getAbsoluteCapacity(), DELTA); Assert.assertEquals("Absolute Max capacity for QUEUEC is not correct", @@ -720,9 +720,9 @@ public void testEffectiveResourceAfterIncreasingClusterResource() LeafQueue qA1 = (LeafQueue) cs.getQueue(QUEUEA1); Assert.assertEquals("Effective Min resource for QUEUEA1 is not correct", - QUEUE_A1_MINRES, qA1.queueResourceQuotas.getEffectiveMinResource()); + QUEUE_A1_MINRES, qA1.usageTracker.getQueueResourceQuotas().getEffectiveMinResource()); Assert.assertEquals("Effective Max resource for QUEUEA1 is not correct", - QUEUE_A_MAXRES, qA1.queueResourceQuotas.getEffectiveMaxResource()); + QUEUE_A_MAXRES, qA1.usageTracker.getQueueResourceQuotas().getEffectiveMaxResource()); Assert.assertEquals("Absolute capacity for QUEUEA1 is not correct", 0.2, qA1.getAbsoluteCapacity(), DELTA); Assert.assertEquals("Absolute Max capacity for QUEUEA1 is not correct", @@ -730,9 +730,9 @@ public void testEffectiveResourceAfterIncreasingClusterResource() LeafQueue qA2 = (LeafQueue) cs.getQueue(QUEUEA2); Assert.assertEquals("Effective Min resource for QUEUEA2 is not correct", - QUEUE_A2_MINRES, qA2.queueResourceQuotas.getEffectiveMinResource()); + QUEUE_A2_MINRES, qA2.usageTracker.getQueueResourceQuotas().getEffectiveMinResource()); Assert.assertEquals("Effective Max resource for QUEUEA2 is not correct", - QUEUE_A_MAXRES, qA2.queueResourceQuotas.getEffectiveMaxResource()); + QUEUE_A_MAXRES, qA2.usageTracker.getQueueResourceQuotas().getEffectiveMaxResource()); Assert.assertEquals("Absolute capacity for QUEUEA2 is not correct", 0.2, qA2.getAbsoluteCapacity(), DELTA); Assert.assertEquals("Absolute Max capacity for QUEUEA2 is not correct", @@ -740,13 +740,13 @@ public void testEffectiveResourceAfterIncreasingClusterResource() LeafQueue qB1 = (LeafQueue) cs.getQueue(QUEUEB1); Assert.assertEquals("Min resource configured for QUEUEB1 is not correct", - QUEUE_B1_MINRES, qB1.queueResourceQuotas.getConfiguredMinResource()); + QUEUE_B1_MINRES, qB1.usageTracker.getQueueResourceQuotas().getConfiguredMinResource()); Assert.assertEquals("Max resource configured for QUEUEB1 is not correct", - QUEUE_B_MAXRES, qB1.queueResourceQuotas.getConfiguredMaxResource()); + QUEUE_B_MAXRES, qB1.usageTracker.getQueueResourceQuotas().getConfiguredMaxResource()); Assert.assertEquals("Effective Min resource for QUEUEB1 is not correct", - QUEUE_B1_MINRES, qB1.queueResourceQuotas.getEffectiveMinResource()); + QUEUE_B1_MINRES, qB1.usageTracker.getQueueResourceQuotas().getEffectiveMinResource()); Assert.assertEquals("Effective Max resource for QUEUEB1 is not correct", - QUEUE_B_MAXRES, qB1.queueResourceQuotas.getEffectiveMaxResource()); + QUEUE_B_MAXRES, qB1.usageTracker.getQueueResourceQuotas().getEffectiveMaxResource()); Assert.assertEquals("Absolute capacity for QUEUEB1 is not correct", 0.16, qB1.getAbsoluteCapacity(), DELTA); Assert.assertEquals("Absolute Max capacity for QUEUEB1 is not correct", @@ -759,54 +759,54 @@ public void testEffectiveResourceAfterIncreasingClusterResource() // Since configured capacity was based on initial node capacity, a // re configurations is needed to use this added capacity. Assert.assertEquals("Effective Min resource for QUEUEA is not correct", - QUEUE_A_MINRES, qA.queueResourceQuotas.getEffectiveMinResource()); + QUEUE_A_MINRES, qA.usageTracker.getQueueResourceQuotas().getEffectiveMinResource()); Assert.assertEquals("Effective Max resource for QUEUEA is not correct", - QUEUE_A_MAXRES, qA.queueResourceQuotas.getEffectiveMaxResource()); + QUEUE_A_MAXRES, qA.usageTracker.getQueueResourceQuotas().getEffectiveMaxResource()); Assert.assertEquals("Absolute capacity for QUEUEA is not correct", 0.266, qA.getAbsoluteCapacity(), DELTA); Assert.assertEquals("Absolute Max capacity for QUEUEA is not correct", 0.533, qA.getAbsoluteMaximumCapacity(), DELTA); Assert.assertEquals("Effective Min resource for QUEUEB is not correct", - QUEUE_B_MINRES, qB.queueResourceQuotas.getEffectiveMinResource()); + QUEUE_B_MINRES, qB.usageTracker.getQueueResourceQuotas().getEffectiveMinResource()); Assert.assertEquals("Effective Max resource for QUEUEB is not correct", - QUEUE_B_MAXRES, qB.queueResourceQuotas.getEffectiveMaxResource()); + QUEUE_B_MAXRES, qB.usageTracker.getQueueResourceQuotas().getEffectiveMaxResource()); Assert.assertEquals("Absolute capacity for QUEUEB is not correct", 0.133, qB.getAbsoluteCapacity(), DELTA); Assert.assertEquals("Absolute Max capacity for QUEUEB is not correct", 0.4, qB.getAbsoluteMaximumCapacity(), DELTA); Assert.assertEquals("Effective Min resource for QUEUEC is not correct", - QUEUE_C_MINRES, qC.queueResourceQuotas.getEffectiveMinResource()); + QUEUE_C_MINRES, qC.usageTracker.getQueueResourceQuotas().getEffectiveMinResource()); Assert.assertEquals("Effective Max resource for QUEUEC is not correct", - QUEUE_C_MAXRES, qC.queueResourceQuotas.getEffectiveMaxResource()); + QUEUE_C_MAXRES, qC.usageTracker.getQueueResourceQuotas().getEffectiveMaxResource()); Assert.assertEquals("Absolute capacity for QUEUEC is not correct", 0.066, qC.getAbsoluteCapacity(), DELTA); Assert.assertEquals("Absolute Max capacity for QUEUEC is not correct", 0.4, qC.getAbsoluteMaximumCapacity(), DELTA); Assert.assertEquals("Effective Min resource for QUEUEB1 is not correct", - QUEUE_B1_MINRES, qB1.queueResourceQuotas.getEffectiveMinResource()); + QUEUE_B1_MINRES, qB1.usageTracker.getQueueResourceQuotas().getEffectiveMinResource()); Assert.assertEquals("Effective Max resource for QUEUEB1 is not correct", - QUEUE_B_MAXRES, qB1.queueResourceQuotas.getEffectiveMaxResource()); + QUEUE_B_MAXRES, qB1.usageTracker.getQueueResourceQuotas().getEffectiveMaxResource()); Assert.assertEquals("Absolute capacity for QUEUEB1 is not correct", 0.106, qB1.getAbsoluteCapacity(), DELTA); Assert.assertEquals("Absolute Max capacity for QUEUEB1 is not correct", 0.4, qB1.getAbsoluteMaximumCapacity(), DELTA); Assert.assertEquals("Effective Min resource for QUEUEA1 is not correct", - QUEUE_A1_MINRES, qA1.queueResourceQuotas.getEffectiveMinResource()); + QUEUE_A1_MINRES, qA1.usageTracker.getQueueResourceQuotas().getEffectiveMinResource()); Assert.assertEquals("Effective Max resource for QUEUEA1 is not correct", - QUEUE_A_MAXRES, qA1.queueResourceQuotas.getEffectiveMaxResource()); + QUEUE_A_MAXRES, qA1.usageTracker.getQueueResourceQuotas().getEffectiveMaxResource()); Assert.assertEquals("Absolute capacity for QUEUEA1 is not correct", 0.133, qA1.getAbsoluteCapacity(), DELTA); Assert.assertEquals("Absolute Max capacity for QUEUEA1 is not correct", 0.533, qA1.getAbsoluteMaximumCapacity(), DELTA); Assert.assertEquals("Effective Min resource for QUEUEA2 is not correct", - QUEUE_A2_MINRES, qA2.queueResourceQuotas.getEffectiveMinResource()); + QUEUE_A2_MINRES, qA2.usageTracker.getQueueResourceQuotas().getEffectiveMinResource()); Assert.assertEquals("Effective Max resource for QUEUEA2 is not correct", - QUEUE_A_MAXRES, qA2.queueResourceQuotas.getEffectiveMaxResource()); + QUEUE_A_MAXRES, qA2.usageTracker.getQueueResourceQuotas().getEffectiveMaxResource()); Assert.assertEquals("Absolute capacity for QUEUEA2 is not correct", 0.133, qA2.getAbsoluteCapacity(), DELTA); Assert.assertEquals("Absolute Max capacity for QUEUEA2 is not correct", diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimits.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimits.java index a42e4af5dbf0f..a1252cfade79c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimits.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimits.java @@ -870,8 +870,8 @@ public RMNodeLabelsManager createNodeLabelManager() { + "submission of application: " + app3.getApplicationId(), app3.getDiagnostics().toString()); - // based on Global limit of queue usert application is rejected - RMApp app11 = MockRMAppSubmitter.submit(rm, + // based on per user max app settings, app should be rejected instantly + RMApp app13 = MockRMAppSubmitter.submit(rm, MockRMAppSubmissionData.Builder.createWithMemory(GB, rm) .withAppName("app") .withUser("user") @@ -880,36 +880,36 @@ public RMNodeLabelsManager createNodeLabelManager() { .withWaitForAppAcceptedState(false) .build()); rm.drainEvents(); - rm.waitForState(app11.getApplicationId(), RMAppState.ACCEPTED); - assertEquals(RMAppState.ACCEPTED, app11.getState()); - RMApp app12 = MockRMAppSubmitter.submit(rm, + rm.waitForState(app13.getApplicationId(), RMAppState.FAILED); + assertEquals(RMAppState.FAILED, app13.getState()); + assertEquals( + "org.apache.hadoop.security.AccessControlException: Queue" + + " root.d already has 0 applications from user user cannot" + + " accept submission of application: " + app13.getApplicationId(), + app13.getDiagnostics().toString()); + + RMApp app11 = MockRMAppSubmitter.submit(rm, MockRMAppSubmissionData.Builder.createWithMemory(GB, rm) .withAppName("app") - .withUser("user") + .withUser("user2") .withAcls(null) - .withQueue("d") + .withQueue("a2") .withWaitForAppAcceptedState(false) .build()); rm.drainEvents(); - rm.waitForState(app12.getApplicationId(), RMAppState.ACCEPTED); - assertEquals(RMAppState.ACCEPTED, app12.getState()); - RMApp app13 = MockRMAppSubmitter.submit(rm, + rm.waitForState(app11.getApplicationId(), RMAppState.ACCEPTED); + assertEquals(RMAppState.ACCEPTED, app11.getState()); + RMApp app12 = MockRMAppSubmitter.submit(rm, MockRMAppSubmissionData.Builder.createWithMemory(GB, rm) .withAppName("app") - .withUser("user") + .withUser("user2") .withAcls(null) - .withQueue("d") + .withQueue("a2") .withWaitForAppAcceptedState(false) .build()); rm.drainEvents(); - rm.waitForState(app13.getApplicationId(), RMAppState.FAILED); - assertEquals(RMAppState.FAILED, app13.getState()); - assertEquals( - "org.apache.hadoop.security.AccessControlException: Queue" - + " root.d already has 2 applications from user user cannot" - + " accept submission of application: " + app13.getApplicationId(), - app13.getDiagnostics().toString()); - + rm.waitForState(app12.getApplicationId(), RMAppState.ACCEPTED); + assertEquals(RMAppState.ACCEPTED, app12.getState()); // based on system max limit application is rejected RMApp app14 = MockRMAppSubmitter.submit(rm, MockRMAppSubmissionData.Builder.createWithMemory(GB, rm) @@ -938,7 +938,6 @@ public RMNodeLabelsManager createNodeLabelManager() { app15.getDiagnostics().toString()); rm.killApp(app2.getApplicationId()); - rm.killApp(app11.getApplicationId()); rm.killApp(app13.getApplicationId()); rm.killApp(app14.getApplicationId()); rm.stop(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerConfiguration.java new file mode 100644 index 0000000000000..2cbb11fc9bdbe --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerConfiguration.java @@ -0,0 +1,160 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.security.authorize.AccessControlList; +import org.apache.hadoop.util.Sets; +import org.apache.hadoop.yarn.api.records.QueueACL; +import org.junit.Test; + +import java.util.Set; + +import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration.ROOT; +import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration.getQueuePrefix; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +public class TestCapacitySchedulerConfiguration { + + private static final String ROOT_TEST = ROOT + ".test"; + private static final String EMPTY_ACL = ""; + private static final String SPACE_ACL = " "; + private static final String USER1 = "user1"; + private static final String USER2 = "user2"; + private static final String GROUP1 = "group1"; + private static final String GROUP2 = "group2"; + public static final String ONE_USER_ONE_GROUP_ACL = USER1 + " " + GROUP1; + public static final String TWO_USERS_TWO_GROUPS_ACL = + USER1 + "," + USER2 + " " + GROUP1 + ", " + GROUP2; + + private CapacitySchedulerConfiguration createDefaultCsConf() { + return new CapacitySchedulerConfiguration(new Configuration(false), false); + } + + private AccessControlList getSubmitAcl(CapacitySchedulerConfiguration csConf, String queue) { + return csConf.getAcl(queue, QueueACL.SUBMIT_APPLICATIONS); + } + + private void setSubmitAppsConfig(CapacitySchedulerConfiguration csConf, String queue, + String value) { + csConf.set(getSubmitAppsConfigKey(queue), value); + } + + private String getSubmitAppsConfigKey(String queue) { + return getQueuePrefix(queue) + "acl_submit_applications"; + } + + private void testWithGivenAclNoOneHasAccess(String queue, String aclValue) { + testWithGivenAclNoOneHasAccessInternal(queue, queue, aclValue); + } + + private void testWithGivenAclNoOneHasAccess(String queueToSet, String queueToVerify, + String aclValue) { + testWithGivenAclNoOneHasAccessInternal(queueToSet, queueToVerify, aclValue); + } + + private void testWithGivenAclNoOneHasAccessInternal(String queueToSet, String queueToVerify, + String aclValue) { + CapacitySchedulerConfiguration csConf = createDefaultCsConf(); + setSubmitAppsConfig(csConf, queueToSet, aclValue); + AccessControlList acl = getSubmitAcl(csConf, queueToVerify); + assertTrue(acl.getUsers().isEmpty()); + assertTrue(acl.getGroups().isEmpty()); + assertFalse(acl.isAllAllowed()); + } + + private void testWithGivenAclCorrectUserAndGroupHasAccess(String queue, String aclValue, + Set expectedUsers, Set expectedGroups) { + testWithGivenAclCorrectUserAndGroupHasAccessInternal(queue, queue, aclValue, expectedUsers, + expectedGroups); + } + + private void testWithGivenAclCorrectUserAndGroupHasAccessInternal(String queueToSet, + String queueToVerify, String aclValue, Set expectedUsers, + Set expectedGroups) { + CapacitySchedulerConfiguration csConf = createDefaultCsConf(); + setSubmitAppsConfig(csConf, queueToSet, aclValue); + AccessControlList acl = getSubmitAcl(csConf, queueToVerify); + assertFalse(acl.getUsers().isEmpty()); + assertFalse(acl.getGroups().isEmpty()); + assertEquals(expectedUsers, acl.getUsers()); + assertEquals(expectedGroups, acl.getGroups()); + assertFalse(acl.isAllAllowed()); + } + + @Test + public void testDefaultSubmitACLForRootAllAllowed() { + CapacitySchedulerConfiguration csConf = createDefaultCsConf(); + AccessControlList acl = getSubmitAcl(csConf, ROOT); + assertTrue(acl.getUsers().isEmpty()); + assertTrue(acl.getGroups().isEmpty()); + assertTrue(acl.isAllAllowed()); + } + + @Test + public void testDefaultSubmitACLForRootChildNoneAllowed() { + CapacitySchedulerConfiguration csConf = createDefaultCsConf(); + AccessControlList acl = getSubmitAcl(csConf, ROOT_TEST); + assertTrue(acl.getUsers().isEmpty()); + assertTrue(acl.getGroups().isEmpty()); + assertFalse(acl.isAllAllowed()); + } + + @Test + public void testSpecifiedEmptySubmitACLForRoot() { + testWithGivenAclNoOneHasAccess(ROOT, EMPTY_ACL); + } + + @Test + public void testSpecifiedEmptySubmitACLForRootIsNotInherited() { + testWithGivenAclNoOneHasAccess(ROOT, ROOT_TEST, EMPTY_ACL); + } + + @Test + public void testSpecifiedSpaceSubmitACLForRoot() { + testWithGivenAclNoOneHasAccess(ROOT, SPACE_ACL); + } + + @Test + public void testSpecifiedSpaceSubmitACLForRootIsNotInherited() { + testWithGivenAclNoOneHasAccess(ROOT, ROOT_TEST, SPACE_ACL); + } + + @Test + public void testSpecifiedSubmitACLForRoot() { + Set expectedUsers = Sets.newHashSet(USER1); + Set expectedGroups = Sets.newHashSet(GROUP1); + testWithGivenAclCorrectUserAndGroupHasAccess(ROOT, ONE_USER_ONE_GROUP_ACL, expectedUsers, + expectedGroups); + } + + @Test + public void testSpecifiedSubmitACLForRootIsNotInherited() { + testWithGivenAclNoOneHasAccess(ROOT, ROOT_TEST, ONE_USER_ONE_GROUP_ACL); + } + + @Test + public void testSpecifiedSubmitACLTwoUsersTwoGroupsForRoot() { + Set expectedUsers = Sets.newHashSet(USER1, USER2); + Set expectedGroups = Sets.newHashSet(GROUP1, GROUP2); + testWithGivenAclCorrectUserAndGroupHasAccess(ROOT, TWO_USERS_TWO_GROUPS_ACL, expectedUsers, + expectedGroups); + } + +} \ No newline at end of file diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java index 351fdbe3a53e7..d3545bd41f57a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java @@ -57,12 +57,15 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.security.AccessControlException; import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.security.authorize.AccessControlList; import org.apache.hadoop.util.Time; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext; import org.apache.hadoop.yarn.api.records.ContainerExitStatus; +import org.apache.hadoop.yarn.api.records.ContainerLaunchContext; import org.apache.hadoop.yarn.api.records.ContainerState; import org.apache.hadoop.yarn.api.records.ContainerStatus; import org.apache.hadoop.yarn.api.records.NodeId; @@ -74,9 +77,13 @@ import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.api.records.ResourceRequest; import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.factories.RecordFactory; import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider; +import org.apache.hadoop.yarn.security.AccessType; +import org.apache.hadoop.yarn.server.resourcemanager.ApplicationMasterService; import org.apache.hadoop.yarn.server.resourcemanager.MockRM; +import org.apache.hadoop.yarn.server.resourcemanager.RMAppManager; import org.apache.hadoop.yarn.server.resourcemanager.RMContext; import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp; @@ -94,8 +101,10 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueStateManager; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceLimits; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceUsage; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.YarnScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.policy.FifoOrderingPolicyWithExclusivePartitions; import org.apache.hadoop.yarn.server.scheduler.SchedulerRequestKey; +import org.apache.hadoop.yarn.server.security.ApplicationACLsManager; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.UsersManager.User; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.preemption.PreemptionManager; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.ResourceCommitRequest; @@ -5137,6 +5146,13 @@ public void testSetupQueueConfigsWithSpecifiedConfiguration() assertEquals(1.0, leafQueue.getAbsoluteMaximumCapacity(), EPSILON); + // limit maximum apps by max system apps + csConf.setMaximumSystemApplications(15); + leafQueue.updateClusterResource(Resource.newInstance(0, 0), + new ResourceLimits(Resource.newInstance(0, 0))); + + assertEquals(15, leafQueue.getMaxApplications()); + } finally { //revert config changes csConf.setNodeLocalityDelay( @@ -5245,7 +5261,7 @@ public void testRootHasAllNodeLabelsOfItsDescendants() throws IOException { ParentQueue rootQueue = (ParentQueue) cs.getRootQueue(); Assert.assertEquals(Sets.newHashSet("", "test", "test2"), - rootQueue.configuredNodeLabels); + rootQueue.queueNodeLabelsSettings.getConfiguredNodeLabels()); } @After @@ -5254,4 +5270,139 @@ public void tearDown() throws Exception { cs.stop(); } } + + private static class TestRMAppManager extends RMAppManager { + TestRMAppManager(RMContext context, YarnScheduler scheduler, + ApplicationMasterService masterService, + ApplicationACLsManager applicationACLsManager, Configuration conf) { + super(context, scheduler, masterService, applicationACLsManager, conf); + } + + @Override + public void submitApplication( + ApplicationSubmissionContext submissionContext, long submitTime, + UserGroupInformation userUgi) throws YarnException { + super.submitApplication(submissionContext, submitTime, userUgi); + } + } + + @Test + public void testSubmitUsingRealUserAcls() throws Exception { + final String realUser = "AdminUser"; + final String user0 = "user0"; + final String user1 = "user1"; + final String queue = "default"; + + YarnConfiguration conf = new YarnConfiguration(); + MockRM rm = new MockRM(); + rm.init(conf); + rm.start(); + rm.getConfig().setBoolean(YarnConfiguration.YARN_ACL_ENABLE, true); + + UserGroupInformation realUserUgi = + UserGroupInformation.createRemoteUser(realUser); + UserGroupInformation ugi0 = + UserGroupInformation.createProxyUserForTesting("user0", realUserUgi, + new String[] {"group1"}); + UserGroupInformation ugi1 = + UserGroupInformation.createProxyUserForTesting("user1", realUserUgi, + new String[] {"group1"}); + ApplicationId applicationId0 = TestUtils.getMockApplicationId(0); + ApplicationId applicationId1 = TestUtils.getMockApplicationId(1); + CapacityScheduler cSched = (CapacityScheduler) rm.getResourceScheduler(); + + ParentQueue rootQueue = (ParentQueue) cSched.getRootQueue(); + Map rootAcls = rootQueue.acls; + rootAcls.put(AccessType.SUBMIT_APP, new AccessControlList(realUser)); + rootAcls.put(AccessType.ADMINISTER_QUEUE, new AccessControlList(realUser)); + + LeafQueue defaultQueue = (LeafQueue)cSched.getQueue(queue); + Map a = defaultQueue.acls; + a.put(AccessType.SUBMIT_APP, new AccessControlList(realUser)); + a.put(AccessType.ADMINISTER_QUEUE, new AccessControlList(realUser)); + + TestRMAppManager testRmAppManager = + new TestRMAppManager(rmContext, cSched, rm.getApplicationMasterService(), + rm.getApplicationACLsManager(), rm.getConfig()); + ContainerLaunchContext clc = + mock(ContainerLaunchContext.class); + ApplicationSubmissionContext asc = + ApplicationSubmissionContext.newInstance( + applicationId0, "myApp0", "default", Priority.newInstance(0), + clc, false, false, 1, Resource.newInstance(1024, 1)); + + // Each of the following test cases has a proxied user and a real user. + // The proxied users are user0 and user1, depending on the test. The real + // user is always AdminUser. + + // Ensure that user0 is not allowed to submit to the default queue when only + // the admin user is in the submit ACL and the admin user does not have the + // USE_REAL_ACLS character prepended. + try { + testRmAppManager.submitApplication(asc, System.currentTimeMillis(), ugi0); + Assert.fail(user0 + " should not be allowed to submit to the " + + queue + " queue when only admin user is in submit ACL."); + } catch (YarnException e) { + // This is the expected behavior. + assertTrue("Should have received an AccessControlException.", + e.getCause() instanceof AccessControlException); + } + + // With only user0 in the list of users authorized to submit apps to the + // queue, ensure that user0 is allowed to submit to the default queue. + a.put(AccessType.SUBMIT_APP, new AccessControlList(user0)); + a.put(AccessType.ADMINISTER_QUEUE, new AccessControlList(realUser)); + try { + testRmAppManager.submitApplication(asc, System.currentTimeMillis(), ugi0); + } catch (YarnException e) { + Assert.fail(user0 + " should be allowed to submit to the " + + queue + " queue."); + } + + // With only user0 in the list of users authorized to submit apps to the + // queue, ensure that user1 is NOT allowed to submit to the default queue + try { + testRmAppManager.submitApplication(asc, System.currentTimeMillis(), ugi1); + Assert.fail(user1 + " should not be allowed to submit to the " + + queue + " queue."); + } catch (YarnException e) { + // This is the expected behavior. + assertTrue("Should have received an AccessControlException.", + e.getCause() instanceof AccessControlException); + } + + // Even though the admin user is in the list of users allowed to submit to + // the default queue and user1's real user is the admin user, user1 should + // not be allowed to submit to queue because the ACL entry does not have the + // special character prepended in the list. + a.put(AccessType.SUBMIT_APP, + new AccessControlList(user0 + "," + realUser)); + try { + testRmAppManager.submitApplication(asc, System.currentTimeMillis(), ugi1); + Assert.fail(user1 + " should not be allowed to submit to the " + + queue + " queue."); + } catch (YarnException e) { + // This is the expected behavior. + assertTrue("Should have received an AccessControlException.", + e.getCause() instanceof AccessControlException); + } + + // user1 should now be allowed to submit to the default queue because the + // admin user is in the ACL list and has the USE_REAL_ACLS character + // prepended. + a.put(AccessType.SUBMIT_APP, + new AccessControlList(user0 + "," + + AccessControlList.USE_REAL_ACLS + realUser)); + asc.setApplicationId(applicationId1); + try { + testRmAppManager.submitApplication(asc, System.currentTimeMillis(), ugi1); + } catch (YarnException e) { + LOG.error("failed to submit", e); + Assert.fail(user1 + " should be allowed to submit to the " + + queue + " queue when real user is" + realUser + "."); + } + + rm.stop(); + rm.close(); + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestParentQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestParentQueue.java index 8b75a9c126fbb..fe90ca844359f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestParentQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestParentQueue.java @@ -1137,7 +1137,7 @@ public void testDeriveCapacityFromAbsoluteConfigurations() throws Exception { assertEquals(b.getMaxApplications(), b.getMaxApplicationsPerUser()); // Set GlobalMaximumApplicationsPerQueue in csConf - csConf.setGlobalMaximumApplicationsPerQueue(20000); + csConf.setGlobalMaximumApplicationsPerQueue(8000); root.updateClusterResource(clusterResource, new ResourceLimits(clusterResource)); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestQueueCapacityVector.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestQueueCapacityVector.java new file mode 100644 index 0000000000000..058e14bfaf272 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestQueueCapacityVector.java @@ -0,0 +1,111 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity; + +import org.apache.hadoop.util.Lists; +import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.QueueCapacityVector.QueueCapacityType; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.QueueCapacityVector.QueueCapacityVectorEntry; +import org.apache.hadoop.yarn.util.resource.ResourceUtils; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +import java.util.List; + +import static org.apache.hadoop.yarn.api.records.ResourceInformation.MEMORY_URI; +import static org.apache.hadoop.yarn.api.records.ResourceInformation.VCORES_URI; +import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CSQueueUtils.EPSILON; + +public class TestQueueCapacityVector { + private static final String CUSTOM_RESOURCE = "custom"; + public static final String MIXED_CAPACITY_VECTOR_STRING = + "[custom=3.0,memory-mb=10.0w,vcores=6.0%]"; + + private final YarnConfiguration conf = new YarnConfiguration(); + + @Before + public void setUp() { + conf.set(YarnConfiguration.RESOURCE_TYPES, CUSTOM_RESOURCE); + ResourceUtils.resetResourceTypes(conf); + } + + @Test + public void getResourceNamesByCapacityType() { + QueueCapacityVector capacityVector = QueueCapacityVector.newInstance(); + + capacityVector.setResource(MEMORY_URI, 10, QueueCapacityType.PERCENTAGE); + capacityVector.setResource(VCORES_URI, 6, QueueCapacityType.PERCENTAGE); + + // custom is not set, defaults to 0 + Assert.assertEquals(1, capacityVector.getResourceNamesByCapacityType( + QueueCapacityType.ABSOLUTE).size()); + Assert.assertTrue(capacityVector.getResourceNamesByCapacityType( + QueueCapacityType.ABSOLUTE).contains(CUSTOM_RESOURCE)); + + Assert.assertEquals(2, capacityVector.getResourceNamesByCapacityType( + QueueCapacityType.PERCENTAGE).size()); + Assert.assertTrue(capacityVector.getResourceNamesByCapacityType( + QueueCapacityType.PERCENTAGE).contains(VCORES_URI)); + Assert.assertTrue(capacityVector.getResourceNamesByCapacityType( + QueueCapacityType.PERCENTAGE).contains(MEMORY_URI)); + Assert.assertEquals(10, capacityVector.getResource(MEMORY_URI).getResourceValue(), EPSILON); + Assert.assertEquals(6, capacityVector.getResource(VCORES_URI).getResourceValue(), EPSILON); + } + + @Test + public void isResourceOfType() { + QueueCapacityVector capacityVector = QueueCapacityVector.newInstance(); + + capacityVector.setResource(MEMORY_URI, 10, QueueCapacityType.WEIGHT); + capacityVector.setResource(VCORES_URI, 6, QueueCapacityType.PERCENTAGE); + capacityVector.setResource(CUSTOM_RESOURCE, 3, QueueCapacityType.ABSOLUTE); + + Assert.assertTrue(capacityVector.isResourceOfType(MEMORY_URI, QueueCapacityType.WEIGHT)); + Assert.assertTrue(capacityVector.isResourceOfType(VCORES_URI, QueueCapacityType.PERCENTAGE)); + Assert.assertTrue(capacityVector.isResourceOfType(CUSTOM_RESOURCE, QueueCapacityType.ABSOLUTE)); + } + + @Test + public void testIterator() { + QueueCapacityVector capacityVector = QueueCapacityVector.newInstance(); + List entries = Lists.newArrayList(capacityVector); + + Assert.assertEquals(3, entries.size()); + + QueueCapacityVector emptyCapacityVector = new QueueCapacityVector(); + List emptyEntries = Lists.newArrayList(emptyCapacityVector); + + Assert.assertEquals(0, emptyEntries.size()); + } + + @Test + public void testToString() { + QueueCapacityVector capacityVector = QueueCapacityVector.newInstance(); + + capacityVector.setResource(MEMORY_URI, 10, QueueCapacityType.WEIGHT); + capacityVector.setResource(VCORES_URI, 6, QueueCapacityType.PERCENTAGE); + capacityVector.setResource(CUSTOM_RESOURCE, 3, QueueCapacityType.ABSOLUTE); + + Assert.assertEquals(MIXED_CAPACITY_VECTOR_STRING, capacityVector.toString()); + + QueueCapacityVector emptyCapacityVector = new QueueCapacityVector(); + Assert.assertEquals("[]", emptyCapacityVector.toString()); + } +} \ No newline at end of file diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestResourceVector.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestResourceVector.java new file mode 100644 index 0000000000000..fd6edb1fa5d49 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestResourceVector.java @@ -0,0 +1,118 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity; + +import org.apache.hadoop.yarn.api.records.Resource; +import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.util.resource.ResourceUtils; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +import java.util.HashMap; +import java.util.Map; + +import static org.apache.hadoop.yarn.api.records.ResourceInformation.MEMORY_URI; +import static org.apache.hadoop.yarn.api.records.ResourceInformation.VCORES_URI; +import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CSQueueUtils.EPSILON; + +public class TestResourceVector { + private final static String CUSTOM_RESOURCE = "custom"; + + private final YarnConfiguration conf = new YarnConfiguration(); + + @Before + public void setUp() { + conf.set(YarnConfiguration.RESOURCE_TYPES, CUSTOM_RESOURCE); + ResourceUtils.resetResourceTypes(conf); + } + + @Test + public void testCreation() { + ResourceVector zeroResourceVector = ResourceVector.newInstance(); + Assert.assertEquals(0, zeroResourceVector.getValue(MEMORY_URI), EPSILON); + Assert.assertEquals(0, zeroResourceVector.getValue(VCORES_URI), EPSILON); + Assert.assertEquals(0, zeroResourceVector.getValue(CUSTOM_RESOURCE), EPSILON); + + ResourceVector uniformResourceVector = ResourceVector.of(10); + Assert.assertEquals(10, uniformResourceVector.getValue(MEMORY_URI), EPSILON); + Assert.assertEquals(10, uniformResourceVector.getValue(VCORES_URI), EPSILON); + Assert.assertEquals(10, uniformResourceVector.getValue(CUSTOM_RESOURCE), EPSILON); + + Map customResources = new HashMap<>(); + customResources.put(CUSTOM_RESOURCE, 2L); + Resource resource = Resource.newInstance(10, 5, customResources); + ResourceVector resourceVectorFromResource = ResourceVector.of(resource); + Assert.assertEquals(10, resourceVectorFromResource.getValue(MEMORY_URI), EPSILON); + Assert.assertEquals(5, resourceVectorFromResource.getValue(VCORES_URI), EPSILON); + Assert.assertEquals(2, resourceVectorFromResource.getValue(CUSTOM_RESOURCE), EPSILON); + } + + @Test + public void testSubtract() { + ResourceVector lhsResourceVector = ResourceVector.of(13); + ResourceVector rhsResourceVector = ResourceVector.of(5); + lhsResourceVector.subtract(rhsResourceVector); + + Assert.assertEquals(8, lhsResourceVector.getValue(MEMORY_URI), EPSILON); + Assert.assertEquals(8, lhsResourceVector.getValue(VCORES_URI), EPSILON); + Assert.assertEquals(8, lhsResourceVector.getValue(CUSTOM_RESOURCE), EPSILON); + + ResourceVector negativeResourceVector = ResourceVector.of(-100); + + // Check whether overflow causes any issues + negativeResourceVector.subtract(ResourceVector.of(Float.MAX_VALUE)); + Assert.assertEquals(-Float.MAX_VALUE, negativeResourceVector.getValue(MEMORY_URI), EPSILON); + Assert.assertEquals(-Float.MAX_VALUE, negativeResourceVector.getValue(VCORES_URI), EPSILON); + Assert.assertEquals(-Float.MAX_VALUE, negativeResourceVector.getValue(CUSTOM_RESOURCE), + EPSILON); + + } + + @Test + public void testIncrement() { + ResourceVector resourceVector = ResourceVector.of(13); + resourceVector.increment(MEMORY_URI, 5); + + Assert.assertEquals(18, resourceVector.getValue(MEMORY_URI), EPSILON); + Assert.assertEquals(13, resourceVector.getValue(VCORES_URI), EPSILON); + Assert.assertEquals(13, resourceVector.getValue(CUSTOM_RESOURCE), EPSILON); + + // Check whether overflow causes any issues + ResourceVector maxFloatResourceVector = ResourceVector.of(Float.MAX_VALUE); + maxFloatResourceVector.increment(MEMORY_URI, 100); + Assert.assertEquals(Float.MAX_VALUE, maxFloatResourceVector.getValue(MEMORY_URI), EPSILON); + } + + @Test + public void testEquals() { + ResourceVector resourceVector = ResourceVector.of(13); + ResourceVector resourceVectorOther = ResourceVector.of(14); + Resource resource = Resource.newInstance(13, 13); + + Assert.assertNotEquals(null, resourceVector); + Assert.assertNotEquals(resourceVectorOther, resourceVector); + Assert.assertNotEquals(resource, resourceVector); + + ResourceVector resourceVectorOne = ResourceVector.of(1); + resourceVectorOther.subtract(resourceVectorOne); + + Assert.assertEquals(resourceVectorOther, resourceVector); + } +} \ No newline at end of file diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestQueueCapacityConfigParser.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestQueueCapacityConfigParser.java new file mode 100644 index 0000000000000..1aba816abd27a --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestQueueCapacityConfigParser.java @@ -0,0 +1,241 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.conf; + +import org.apache.hadoop.util.Lists; +import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.QueueCapacityVector; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.QueueCapacityVector.QueueCapacityVectorEntry; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.QueueCapacityVector.QueueCapacityType; +import org.apache.hadoop.yarn.util.resource.ResourceUtils; +import org.junit.Assert; +import org.junit.Test; + +import java.util.List; + +import static org.apache.hadoop.yarn.api.records.ResourceInformation.GPU_URI; +import static org.apache.hadoop.yarn.api.records.ResourceInformation.MEMORY_URI; +import static org.apache.hadoop.yarn.api.records.ResourceInformation.VCORES_URI; +import static org.apache.hadoop.yarn.nodelabels.CommonNodeLabelsManager.NO_LABEL; +import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.TestQueueMetricsForCustomResources.GB; +import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CSQueueUtils.EPSILON; + +public class TestQueueCapacityConfigParser { + + private static final String ALL_RESOURCE_TEMPLATE = "[memory-mb=%s, vcores=%s, yarn.io/gpu=%s]"; + private static final String MEMORY_VCORE_TEMPLATE = "[memory-mb=%s, vcores=%s]"; + + private static final String MEMORY_ABSOLUTE = "12Gi"; + private static final float VCORE_ABSOLUTE = 6; + private static final float GPU_ABSOLUTE = 10; + + private static final float PERCENTAGE_VALUE = 50f; + private static final float MEMORY_MIXED = 1024; + private static final float WEIGHT_VALUE = 6; + + private static final String QUEUE = "root.test"; + + private static final String ABSOLUTE_RESOURCE = String.format( + ALL_RESOURCE_TEMPLATE, MEMORY_ABSOLUTE, VCORE_ABSOLUTE, GPU_ABSOLUTE); + private static final String ABSOLUTE_RESOURCE_MEMORY_VCORE = String.format( + MEMORY_VCORE_TEMPLATE, MEMORY_ABSOLUTE, VCORE_ABSOLUTE); + private static final String MIXED_RESOURCE = String.format( + ALL_RESOURCE_TEMPLATE, MEMORY_MIXED, PERCENTAGE_VALUE + "%", WEIGHT_VALUE + "w"); + private static final String RESOURCE_TYPES = GPU_URI; + + public static final String NONEXISTINGSUFFIX = "50nonexistingsuffix"; + public static final String EMPTY_BRACKET = "[]"; + public static final String INVALID_CAPACITY_BRACKET = "[invalid]"; + public static final String INVALID_CAPACITY_FORMAT = "[memory-100,vcores-60]"; + + private final QueueCapacityConfigParser capacityConfigParser + = new QueueCapacityConfigParser(); + + @Test + public void testPercentageCapacityConfig() { + CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration(); + conf.setCapacity(QUEUE, PERCENTAGE_VALUE); + + QueueCapacityVector percentageCapacityVector = capacityConfigParser.parse(conf, QUEUE, + NO_LABEL); + QueueCapacityVectorEntry memory = percentageCapacityVector.getResource(MEMORY_URI); + QueueCapacityVectorEntry vcore = percentageCapacityVector.getResource(VCORES_URI); + + Assert.assertEquals(QueueCapacityType.PERCENTAGE, memory.getVectorResourceType()); + Assert.assertEquals(PERCENTAGE_VALUE, memory.getResourceValue(), EPSILON); + + Assert.assertEquals(QueueCapacityType.PERCENTAGE, vcore.getVectorResourceType()); + Assert.assertEquals(PERCENTAGE_VALUE, vcore.getResourceValue(), EPSILON); + + QueueCapacityVector rootCapacityVector = capacityConfigParser.parse(conf, + CapacitySchedulerConfiguration.ROOT, NO_LABEL); + + QueueCapacityVectorEntry memoryRoot = rootCapacityVector.getResource(MEMORY_URI); + QueueCapacityVectorEntry vcoreRoot = rootCapacityVector.getResource(VCORES_URI); + + Assert.assertEquals(QueueCapacityType.PERCENTAGE, memoryRoot.getVectorResourceType()); + Assert.assertEquals(100f, memoryRoot.getResourceValue(), EPSILON); + + Assert.assertEquals(QueueCapacityType.PERCENTAGE, vcoreRoot.getVectorResourceType()); + Assert.assertEquals(100f, vcoreRoot.getResourceValue(), EPSILON); + } + + @Test + public void testWeightCapacityConfig() { + CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration(); + conf.setNonLabeledQueueWeight(QUEUE, WEIGHT_VALUE); + + QueueCapacityVector weightCapacityVector = capacityConfigParser.parse(conf, QUEUE, NO_LABEL); + + QueueCapacityVectorEntry memory = weightCapacityVector.getResource(MEMORY_URI); + QueueCapacityVectorEntry vcore = weightCapacityVector.getResource(VCORES_URI); + + Assert.assertEquals(QueueCapacityType.WEIGHT, memory.getVectorResourceType()); + Assert.assertEquals(WEIGHT_VALUE, memory.getResourceValue(), EPSILON); + + Assert.assertEquals(QueueCapacityType.WEIGHT, vcore.getVectorResourceType()); + Assert.assertEquals(WEIGHT_VALUE, vcore.getResourceValue(), EPSILON); + } + + @Test + public void testAbsoluteCapacityVectorConfig() { + CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration(); + conf.set(CapacitySchedulerConfiguration.getQueuePrefix(QUEUE) + + CapacitySchedulerConfiguration.CAPACITY, ABSOLUTE_RESOURCE); + conf.set(YarnConfiguration.RESOURCE_TYPES, RESOURCE_TYPES); + ResourceUtils.resetResourceTypes(conf); + + QueueCapacityVector absoluteCapacityVector = capacityConfigParser.parse(conf, QUEUE, NO_LABEL); + + Assert.assertEquals(QueueCapacityType.ABSOLUTE, absoluteCapacityVector.getResource(MEMORY_URI) + .getVectorResourceType()); + Assert.assertEquals(12 * GB, absoluteCapacityVector.getResource(MEMORY_URI) + .getResourceValue(), EPSILON); + + Assert.assertEquals(QueueCapacityType.ABSOLUTE, absoluteCapacityVector.getResource(VCORES_URI) + .getVectorResourceType()); + Assert.assertEquals(VCORE_ABSOLUTE, absoluteCapacityVector.getResource(VCORES_URI) + .getResourceValue(), EPSILON); + + Assert.assertEquals(QueueCapacityType.ABSOLUTE, absoluteCapacityVector.getResource(GPU_URI) + .getVectorResourceType()); + Assert.assertEquals(GPU_ABSOLUTE, absoluteCapacityVector.getResource(GPU_URI) + .getResourceValue(), EPSILON); + + conf.set(CapacitySchedulerConfiguration.getQueuePrefix(QUEUE) + + CapacitySchedulerConfiguration.CAPACITY, ABSOLUTE_RESOURCE_MEMORY_VCORE); + QueueCapacityVector withoutGpuVector = capacityConfigParser.parse(conf, QUEUE, NO_LABEL); + + Assert.assertEquals(3, withoutGpuVector.getResourceCount()); + Assert.assertEquals(0f, withoutGpuVector.getResource(GPU_URI).getResourceValue(), EPSILON); + } + + @Test + public void testMixedCapacityConfig() { + CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration(); + conf.set(CapacitySchedulerConfiguration.getQueuePrefix(QUEUE) + + CapacitySchedulerConfiguration.CAPACITY, MIXED_RESOURCE); + conf.set(YarnConfiguration.RESOURCE_TYPES, RESOURCE_TYPES); + ResourceUtils.resetResourceTypes(conf); + + QueueCapacityVector mixedCapacityVector = + capacityConfigParser.parse(conf, QUEUE, NO_LABEL); + + Assert.assertEquals(QueueCapacityType.ABSOLUTE, + mixedCapacityVector.getResource(MEMORY_URI).getVectorResourceType()); + Assert.assertEquals(MEMORY_MIXED, mixedCapacityVector.getResource(MEMORY_URI) + .getResourceValue(), EPSILON); + + Assert.assertEquals(QueueCapacityType.PERCENTAGE, + mixedCapacityVector.getResource(VCORES_URI).getVectorResourceType()); + Assert.assertEquals(PERCENTAGE_VALUE, + mixedCapacityVector.getResource(VCORES_URI).getResourceValue(), EPSILON); + + Assert.assertEquals(QueueCapacityType.WEIGHT, + mixedCapacityVector.getResource(GPU_URI).getVectorResourceType()); + Assert.assertEquals(WEIGHT_VALUE, + mixedCapacityVector.getResource(GPU_URI).getResourceValue(), EPSILON); + + // Test undefined capacity type default value + conf.set(CapacitySchedulerConfiguration.getQueuePrefix(QUEUE) + + CapacitySchedulerConfiguration.CAPACITY, ABSOLUTE_RESOURCE_MEMORY_VCORE); + + QueueCapacityVector mixedCapacityVectorWithGpuUndefined = + capacityConfigParser.parse(conf, QUEUE, NO_LABEL); + Assert.assertEquals(QueueCapacityType.ABSOLUTE, + mixedCapacityVectorWithGpuUndefined.getResource(MEMORY_URI).getVectorResourceType()); + Assert.assertEquals(0, mixedCapacityVectorWithGpuUndefined.getResource(GPU_URI) + .getResourceValue(), EPSILON); + + } + + @Test + public void testInvalidCapacityConfigs() { + CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration(); + + conf.set(CapacitySchedulerConfiguration.getQueuePrefix(QUEUE) + + CapacitySchedulerConfiguration.CAPACITY, NONEXISTINGSUFFIX); + QueueCapacityVector capacityVectorWithInvalidSuffix = + capacityConfigParser.parse(conf, QUEUE, NO_LABEL); + List entriesWithInvalidSuffix = + Lists.newArrayList(capacityVectorWithInvalidSuffix.iterator()); + Assert.assertEquals(0, entriesWithInvalidSuffix.size()); + + conf.set(CapacitySchedulerConfiguration.getQueuePrefix(QUEUE) + + CapacitySchedulerConfiguration.CAPACITY, INVALID_CAPACITY_FORMAT); + QueueCapacityVector invalidDelimiterCapacityVector = + capacityConfigParser.parse(conf, QUEUE, NO_LABEL); + List invalidDelimiterEntries = + Lists.newArrayList(invalidDelimiterCapacityVector.iterator()); + Assert.assertEquals(0, invalidDelimiterEntries.size()); + + conf.set(CapacitySchedulerConfiguration.getQueuePrefix(QUEUE) + + CapacitySchedulerConfiguration.CAPACITY, INVALID_CAPACITY_BRACKET); + QueueCapacityVector invalidCapacityVector = + capacityConfigParser.parse(conf, QUEUE, NO_LABEL); + List resources = + Lists.newArrayList(invalidCapacityVector.iterator()); + Assert.assertEquals(0, resources.size()); + + conf.set(CapacitySchedulerConfiguration.getQueuePrefix(QUEUE) + + CapacitySchedulerConfiguration.CAPACITY, EMPTY_BRACKET); + QueueCapacityVector emptyBracketCapacityVector = + capacityConfigParser.parse(conf, QUEUE, NO_LABEL); + List emptyEntries = + Lists.newArrayList(emptyBracketCapacityVector.iterator()); + Assert.assertEquals(0, emptyEntries.size()); + + conf.set(CapacitySchedulerConfiguration.getQueuePrefix(QUEUE) + + CapacitySchedulerConfiguration.CAPACITY, ""); + QueueCapacityVector emptyCapacity = + capacityConfigParser.parse(conf, QUEUE, NO_LABEL); + List emptyResources = + Lists.newArrayList(emptyCapacity.iterator()); + Assert.assertEquals(emptyResources.size(), 0); + + conf.unset(CapacitySchedulerConfiguration.getQueuePrefix(QUEUE) + + CapacitySchedulerConfiguration.CAPACITY); + QueueCapacityVector nonSetCapacity = + capacityConfigParser.parse(conf, QUEUE, NO_LABEL); + List nonSetResources = + Lists.newArrayList(nonSetCapacity.iterator()); + Assert.assertEquals(nonSetResources.size(), 0); + } +} \ No newline at end of file diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/queuemanagement/TestDeactivatedLeafQueuesByLabel.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/queuemanagement/TestDeactivatedLeafQueuesByLabel.java new file mode 100644 index 0000000000000..d886641408e78 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/queuemanagement/TestDeactivatedLeafQueuesByLabel.java @@ -0,0 +1,42 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.queuemanagement; + +import org.junit.Test; + +import static org.junit.Assert.*; +import static org.mockito.Mockito.spy; + +public class TestDeactivatedLeafQueuesByLabel { + @Test + public void testGetMaxLeavesToBeActivated() { + DeactivatedLeafQueuesByLabel d1 = spy(DeactivatedLeafQueuesByLabel.class); + d1.setAvailableCapacity(0.17f); + d1.setLeafQueueTemplateAbsoluteCapacity(0.03f); + assertEquals(1, d1.getMaxLeavesToBeActivated(1)); + + DeactivatedLeafQueuesByLabel d2 = spy(DeactivatedLeafQueuesByLabel.class); + d2.setAvailableCapacity(0.17f); + d2.setLeafQueueTemplateAbsoluteCapacity(0.03f); + assertEquals(5, d2.getMaxLeavesToBeActivated(7)); + + DeactivatedLeafQueuesByLabel d3 = spy(DeactivatedLeafQueuesByLabel.class); + d3.setAvailableCapacity(0f); + d3.setLeafQueueTemplateAbsoluteCapacity(0.03f); + assertEquals(0, d3.getMaxLeavesToBeActivated(10)); + } +} \ No newline at end of file diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/CapacityScheduler.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/CapacityScheduler.md index 6d87d4febb65c..a945e14aba70f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/CapacityScheduler.md +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/CapacityScheduler.md @@ -176,8 +176,8 @@ Configuration | Property | Description | |:---- |:---- | | `yarn.scheduler.capacity..state` | The *state* of the queue. Can be one of `RUNNING` or `STOPPED`. If a queue is in `STOPPED` state, new applications cannot be submitted to *itself* or *any of its child queues*. Thus, if the *root* queue is `STOPPED` no applications can be submitted to the entire cluster. Existing applications continue to completion, thus the queue can be *drained* gracefully. Value is specified as Enumeration. | -| `yarn.scheduler.capacity.root..acl_submit_applications` | The *ACL* which controls who can *submit* applications to the given queue. If the given user/group has necessary ACLs on the given queue or *one of the parent queues in the hierarchy* they can submit applications. *ACLs* for this property *are* inherited from the parent queue if not specified. | -| `yarn.scheduler.capacity.root..acl_administer_queue` | The *ACL* which controls who can *administer* applications on the given queue. If the given user/group has necessary ACLs on the given queue or *one of the parent queues in the hierarchy* they can administer applications. *ACLs* for this property *are* inherited from the parent queue if not specified. | +| `yarn.scheduler.capacity.root..acl_submit_applications` | The *ACL* which controls who can *submit* applications to the given queue. If the given user/group has necessary ACLs on the given queue or *one of the parent queues in the hierarchy* they can submit applications. *ACLs* for this property *are* inherited from the parent queue if not specified. If a tilde (~) is prepended to a user name in this list, the real user's ACLs will allow the proxied user to submit to the queue. | +| `yarn.scheduler.capacity.root..acl_administer_queue` | The *ACL* which controls who can *administer* applications on the given queue. If the given user/group has necessary ACLs on the given queue or *one of the parent queues in the hierarchy* they can administer applications. *ACLs* for this property *are* inherited from the parent queue if not specified. If a tilde (~) is prepended to a user name in this list, the real user's ACLs will allow the proxied user to administer apps the queue. | **Note:** An *ACL* is of the form *user1*,*user2* *space* *group1*,*group2*. The special value of * implies *anyone*. The special value of *space* implies *no one*. The default is * for the root queue if not specified. diff --git a/hadoop-yarn-project/hadoop-yarn/pom.xml b/hadoop-yarn-project/hadoop-yarn/pom.xml index e9c3bccc618ce..e97e35608553a 100644 --- a/hadoop-yarn-project/hadoop-yarn/pom.xml +++ b/hadoop-yarn-project/hadoop-yarn/pom.xml @@ -78,38 +78,6 @@ org.apache.hadoop.yarn.proto:org.apache.hadoop.yarn.federation.proto:org.apache.hadoop.yarn.service - - org.apache.maven.plugins - maven-enforcer-plugin - - - de.skuzzle.enforcer - restrict-imports-enforcer-rule - ${restrict-imports.enforcer.version} - - - - - banned-illegal-imports - process-sources - - enforce - - - - - true - Use hadoop-common provided VisibleForTesting rather than the one provided by Guava - - org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting - com.google.common.annotations.VisibleForTesting - - - - - - - diff --git a/pom.xml b/pom.xml index 7ec3ade62ab51..62f011f7082b4 100644 --- a/pom.xml +++ b/pom.xml @@ -189,7 +189,6 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/x Use hadoop-common provided Sets rather than Guava provided Sets org.apache.hadoop.thirdparty.com.google.common.collect.Sets - com.google.common.collect.Sets @@ -197,7 +196,62 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/x Use hadoop-common provided Lists rather than Guava provided Lists org.apache.hadoop.thirdparty.com.google.common.collect.Lists - com.google.common.collect.Lists + + + + true + Use hadoop-annotation provided VisibleForTesting rather than the one provided by Guava + + org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting + + + + true + Use alternatives to Guava common classes + + com.google.common.** + + + + true + Use alternative to Guava provided BaseEncoding + + org.apache.hadoop.thirdparty.com.google.common.io.BaseEncoding + + + + true + Use alternative to Guava provided Optional + + org.apache.hadoop.thirdparty.com.google.common.base.Optional + + + + true + Use alternative to Guava provided Function + + org.apache.hadoop.thirdparty.com.google.common.base.Function + + + + true + Use alternative to Guava provided Predicate + + org.apache.hadoop.thirdparty.com.google.common.base.Predicate + + + + true + Use alternative to Guava provided Supplier + + org.apache.hadoop.thirdparty.com.google.common.base.Supplier + + + + true + Use alternative to Guava provided ImmutableListMultimap + + org.apache.hadoop.thirdparty.com.google.common.collect.ImmutableListMultimap